From c78e2aba7f90fe58e349a739bd603429810c3452 Mon Sep 17 00:00:00 2001 From: user Date: Wed, 22 Jul 2026 13:00:18 +0700 Subject: [PATCH 1/2] frames --- README.md | 13 + controller/bin/remove-repeated-frames.js | 9 + .../run-repeated-frame-removal-examples.js | 115 ++++ controller/src/action-handler.js | 5 +- controller/src/browser-session.js | 44 +- controller/src/protocol.js | 21 + controller/src/repeated-frame-remover.js | 624 ++++++++++++++++++ ops/remote/run-playtest.js | 1 + test/browser-session.test.js | 33 + test/repeated-frame-remover.test.js | 86 +++ test/session-cleanup.test.js | 23 + 11 files changed, 972 insertions(+), 2 deletions(-) create mode 100644 controller/bin/remove-repeated-frames.js create mode 100644 controller/bin/run-repeated-frame-removal-examples.js create mode 100644 controller/src/repeated-frame-remover.js create mode 100644 test/repeated-frame-remover.test.js diff --git a/README.md b/README.md index fe0eefe..81ba09b 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,11 @@ Useful `start` options: capture the rendered viewport. Override capture sources with `videoSource` and `audioSource`. (`recordAudio` is accepted as a legacy alias for `record`; they mean the same thing — audio is always captured when recording.) +- `repeatedFrameRemoval`: when `record` is enabled, post-process the final WebM + to remove repeated-frame runs before returning the stop response. Pass `true` + for defaults, or an options object such as + `{ "similarityThreshold": 0.98, "edgeFrameCount": 10 }`. The raw capture is + renamed with a `_raw` suffix. - `headless`: defaults to `true`; set `false` to watch the browser. - `channel`: optional Playwright browser channel, such as `chrome` or `msedge`. - `executablePath`: optional explicit browser executable path. @@ -287,6 +292,12 @@ the same recorded audio/video WebM. All recording goes through gstreamer — see the [Requirements](#requirements) section for the mandatory environment (Linux, gstreamer, X server/Xvfb, PulseAudio). +When `repeatedFrameRemoval` was set on `start`, `video` and `audioVideo` +point at the cut WebM at the normal recording path. The original uncut capture +is preserved as `rawVideo` / `rawAudioVideo`, typically +`video/000-runwave-with-audio_raw.webm`, and `repeatedFrameRemoval` reports the +removed frame ranges. + ## State Every response includes generic browser state: @@ -317,6 +328,8 @@ Each turn writes: - `*.png`: screenshots captured during that operation. - `NNN-.json`: detailed sequence log for `step` operations. - `video/000-runwave-with-audio.webm`: final gstreamer audio+video recording. +- `video/000-runwave-with-audio_raw.webm`: original recording when repeated + frame cutting was enabled. Active sessions are tracked as JSON files in `.runwave-sessions/` by default. The matching session file is removed by `stop`. diff --git a/controller/bin/remove-repeated-frames.js b/controller/bin/remove-repeated-frames.js new file mode 100644 index 0000000..a7260df --- /dev/null +++ b/controller/bin/remove-repeated-frames.js @@ -0,0 +1,9 @@ +module.exports = require('../src/repeated-frame-remover'); + +if (require.main === module) { + const { main } = module.exports; + main(process.argv.slice(2)).catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }); +} diff --git a/controller/bin/run-repeated-frame-removal-examples.js b/controller/bin/run-repeated-frame-removal-examples.js new file mode 100644 index 0000000..9dcf33b --- /dev/null +++ b/controller/bin/run-repeated-frame-removal-examples.js @@ -0,0 +1,115 @@ +const fs = require('fs'); +const path = require('path'); +const { removeRepeatedFrames } = require('../src/repeated-frame-remover'); + +const SOURCE_DIRS = [ + '/Users/plato/code/mm-sapphire/cruft/af4809-playtests/videos', + '/Users/plato/code/mm-sapphire/cruft/pr501-playtests/videos', +]; + +const OUTPUT_ROOT = path.resolve(__dirname, '..', '..', 'cruft', 'repeated-frame-removal-examples'); +const SIMILARITY_THRESHOLD = 0.98; +const EDGE_FRAME_COUNT = 10; +const CONCURRENCY = Number(process.env.REPEATED_FRAME_REMOVAL_CONCURRENCY || 4); + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function safeName(value) { + return String(value) + .replace(/[^a-zA-Z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') || 'video'; +} + +function listSourceVideos() { + return SOURCE_DIRS.flatMap((dir) => { + const group = safeName(path.basename(path.dirname(dir))); + return fs.readdirSync(dir) + .filter((entry) => /\.(webm|mp4|mov|mkv)$/i.test(entry)) + .sort() + .map((entry) => ({ + group, + name: path.basename(entry, path.extname(entry)), + ext: path.extname(entry), + sourcePath: path.join(dir, entry), + })); + }); +} + +function elapsedMs(startedAt) { + return Number((process.hrtime.bigint() - startedAt) / 1000000n); +} + +async function processVideo(video) { + const startedAt = process.hrtime.bigint(); + const videoDir = ensureDir(path.join(OUTPUT_ROOT, `${video.group}-${safeName(video.name)}`)); + const inputPath = path.join(videoDir, `input${video.ext}`); + const outputPath = path.join(videoDir, 'output.mp4'); + const reportPath = path.join(videoDir, 'report.json'); + + fs.copyFileSync(video.sourcePath, inputPath); + console.log(`Processing ${video.group}/${video.name}${video.ext}`); + const summary = await removeRepeatedFrames(inputPath, outputPath, { + similarityThreshold: SIMILARITY_THRESHOLD, + edgeFrameCount: EDGE_FRAME_COUNT, + }); + const durationMs = elapsedMs(startedAt); + const report = { + sourcePath: video.sourcePath, + durationMs, + ...summary, + }; + + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); + console.log(` removed ${summary.removedFrameCount}/${summary.frameCount} frames in ${durationMs}ms -> ${outputPath}`); + return report; +} + +async function mapWithConcurrency(items, concurrency, fn) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await fn(items[index], index); + } + } + + const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency) || 1)); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + +async function main() { + ensureDir(OUTPUT_ROOT); + + const startedAt = process.hrtime.bigint(); + const startedAtIso = new Date().toISOString(); + const videos = listSourceVideos(); + const summaries = await mapWithConcurrency(videos, CONCURRENCY, (video) => processVideo(video)); + const durationMs = elapsedMs(startedAt); + + const indexPath = path.join(OUTPUT_ROOT, 'summary.json'); + fs.writeFileSync(indexPath, JSON.stringify({ + startedAt: startedAtIso, + finishedAt: new Date().toISOString(), + durationMs, + videoCount: summaries.length, + concurrency: Math.max(1, Math.min(videos.length, Math.floor(CONCURRENCY) || 1)), + similarityThreshold: SIMILARITY_THRESHOLD, + edgeFrameCount: EDGE_FRAME_COUNT, + videos: summaries, + }, null, 2)); + console.log(`Wrote ${indexPath} after ${durationMs}ms`); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }); +} diff --git a/controller/src/action-handler.js b/controller/src/action-handler.js index d31d49e..c4247ce 100644 --- a/controller/src/action-handler.js +++ b/controller/src/action-handler.js @@ -130,7 +130,7 @@ async function writeStopResponse(runtime, input) { : await runtime.profiler.time('action.stop.screenshot', { action_name: input.action_name }, () => browser.screenshot(outputDir, '999-final') ); - const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close()); + const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close(input)); const video = typeof recording === 'string' ? recording : recording.video; runtime.profiler.timeSync('action.stop.remove_session_file', { sessionFile: runtime.sessionFile }, () => @@ -147,6 +147,9 @@ async function writeStopResponse(runtime, input) { }); if (recording && typeof recording === 'object') { if (recording.audioVideo) payload.audioVideo = recording.audioVideo; + if (recording.rawVideo) payload.rawVideo = recording.rawVideo; + if (recording.rawAudioVideo) payload.rawAudioVideo = recording.rawAudioVideo; + if (recording.repeatedFrameRemoval) payload.repeatedFrameRemoval = recording.repeatedFrameRemoval; } if (screenshot) payload.screenshot = screenshot; const response = runtime.profiler.timeSync('action.stop.write_response', { action_name: input.action_name }, () => diff --git a/controller/src/browser-session.js b/controller/src/browser-session.js index ab1fab7..e13b50d 100644 --- a/controller/src/browser-session.js +++ b/controller/src/browser-session.js @@ -5,6 +5,7 @@ const { AudioVideoRecorder } = require('./audio-recorder'); const { ensureDir, safeName, sleep, timestamp } = require('./file-utils'); const { drawGridOnScreenshot } = require('./grid-overlay'); const { parseArgList, targetUrl } = require('./protocol'); +const { removeRepeatedFramesInPlace } = require('./repeated-frame-remover'); const { readPageState } = require('./state-reader'); const DEFAULT_CHROMIUM_ARGS = ['--no-sandbox', '--enable-unsafe-swiftshader', '--autoplay-policy=no-user-gesture-required']; @@ -30,6 +31,24 @@ function isRecording(config = {}) { return Boolean(config.record || config.recordAudio); } +function isRepeatedFrameRemovalEnabled(config = {}) { + return Boolean(config.repeatedFrameRemoval); +} + +function repeatedFrameRemovalOptions(config = {}) { + const options = config.repeatedFrameRemoval && typeof config.repeatedFrameRemoval === 'object' + ? config.repeatedFrameRemoval + : {}; + return { + edgeFrameCount: options.edgeFrameCount, + similarityThreshold: options.similarityThreshold, + pixelTolerance: options.pixelTolerance, + comparisonWidth: options.comparisonWidth, + ffmpegPath: options.ffmpegPath || config.ffmpegPath, + ffprobePath: options.ffprobePath || config.ffprobePath, + }; +} + function chromiumLaunchArgs(config = {}, env = process.env) { const args = chromiumArgs(config, env); if (!isRecording(config)) return args; @@ -327,16 +346,37 @@ class BrowserSession { ); } - async close() { + async close(overrides = {}) { + const hasRepeatedFrameRemovalOverride = Object.prototype.hasOwnProperty.call(overrides, 'repeatedFrameRemoval'); + const closeConfig = { + ...this.config, + ...overrides, + repeatedFrameRemoval: hasRepeatedFrameRemovalOverride + ? overrides.repeatedFrameRemoval + : this.config.repeatedFrameRemoval, + }; let audioVideoPath = null; + let rawVideoPath = null; + let repeatedFrameRemoval = null; if (this.audioRecorder) { audioVideoPath = await this.time('browser.close.audio_video_stop', () => this.audioRecorder.stop()); } + if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) { + const processed = await this.time('browser.close.remove_repeated_frames', { video: audioVideoPath }, () => + removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions(closeConfig)) + ); + audioVideoPath = processed.video; + rawVideoPath = processed.rawVideo; + repeatedFrameRemoval = processed.repeatedFrameRemoval; + } if (this.context) await this.time('browser.close.context_close', () => this.context.close()); if (this.browser) await this.time('browser.close.browser_close', () => this.browser.close()); return { video: audioVideoPath, audioVideo: audioVideoPath || undefined, + rawVideo: rawVideoPath || undefined, + rawAudioVideo: rawVideoPath || undefined, + repeatedFrameRemoval: repeatedFrameRemoval || undefined, }; } } @@ -346,4 +386,6 @@ module.exports = { browserViewportStabilizerScript, chromiumArgs, chromiumLaunchArgs, + isRepeatedFrameRemovalEnabled, + repeatedFrameRemovalOptions, }; diff --git a/controller/src/protocol.js b/controller/src/protocol.js index 565e4ea..13b9adf 100644 --- a/controller/src/protocol.js +++ b/controller/src/protocol.js @@ -19,6 +19,7 @@ function usage() { session_id: 'playtest-001', file: 'sunnyland-platformer/index.html', record: true, + repeatedFrameRemoval: true, keyAliases: { right: 'd', left: 'a', jump: 'w' }, }, { @@ -115,6 +116,23 @@ function optionalNumber(value, fallback) { return Number.isFinite(number) ? number : fallback; } +function repeatedFrameRemovalEnabled(input) { + return Boolean(input.repeatedFrameRemoval); +} + +function repeatedFrameRemovalConfig(input) { + const options = input.repeatedFrameRemoval && typeof input.repeatedFrameRemoval === 'object' + ? input.repeatedFrameRemoval + : {}; + return { + enabled: repeatedFrameRemovalEnabled(input), + edgeFrameCount: optionalNumber(options.edgeFrameCount, 10), + similarityThreshold: optionalNumber(options.similarityThreshold, 0.98), + pixelTolerance: optionalNumber(options.pixelTolerance, 3), + comparisonWidth: optionalNumber(options.comparisonWidth, 160), + }; +} + function startSessionConfig(input, options = {}) { const viewport = normalizeSize(input.viewport, { width: 1024, height: 620 }); const record = Boolean(input.record || input.recordAudio); @@ -132,6 +150,7 @@ function startSessionConfig(input, options = {}) { deviceScaleFactor: optionalNumber(input.deviceScaleFactor, 1), record, videoSize: record ? normalizeSize(input.videoSize || input.viewport, viewport) : null, + repeatedFrameRemoval: record ? repeatedFrameRemovalConfig(input) : { enabled: false }, }, navigation: { waitUntil: String(input.waitUntil || 'load'), @@ -179,6 +198,8 @@ module.exports = { sessionId, targetUrl, parseArgList, + repeatedFrameRemovalConfig, + repeatedFrameRemovalEnabled, startSessionConfig, diffStartSessionConfig, isListSessionsAction, diff --git a/controller/src/repeated-frame-remover.js b/controller/src/repeated-frame-remover.js new file mode 100644 index 0000000..9ccc248 --- /dev/null +++ b/controller/src/repeated-frame-remover.js @@ -0,0 +1,624 @@ +const fs = require('fs'); +const path = require('path'); +const { spawn } = require('child_process'); + +const DEFAULT_EDGE_FRAME_COUNT = 10; +const DEFAULT_SIMILARITY_THRESHOLD = 0.98; +const DEFAULT_PIXEL_TOLERANCE = 3; + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function spawnChecked(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + stdio: ['ignore', 'ignore', 'pipe'], + ...options, + }); + const stderr = []; + + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (code === 0) { + resolve({ code, signal, stderr: Buffer.concat(stderr).toString('utf8') }); + return; + } + + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${command} failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + }); + }); +} + +function parseFrameMd5Line(line) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) return null; + + const parts = trimmed.split(',').map((part) => part.trim()); + const hash = parts[parts.length - 1]; + if (!/^[a-f0-9]{32}$/i.test(hash)) return null; + + return { + streamIndex: Number(parts[0]), + dts: Number(parts[1]), + pts: Number(parts[2]), + duration: Number(parts[3]), + size: Number(parts[4]), + hash: hash.toLowerCase(), + }; +} + +function collectFrameHashes(inputPath, options = {}) { + const ffmpegPath = options.ffmpegPath || 'ffmpeg'; + const args = [ + '-hide_banner', + '-v', + 'error', + '-i', + inputPath, + '-map', + '0:v:0', + '-an', + '-f', + 'framemd5', + '-', + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffmpegPath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const frames = []; + const stderr = []; + let buffer = ''; + + child.stdout.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() || ''; + + for (const line of lines) { + const frame = parseFrameMd5Line(line); + if (frame) frames.push(frame); + } + }); + + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (buffer) { + const frame = parseFrameMd5Line(buffer); + if (frame) frames.push(frame); + } + + if (code === 0) { + resolve(frames); + return; + } + + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${ffmpegPath} framemd5 failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + }); + }); +} + +function findRepeatedFrameRemovalRanges(frames, options = {}) { + const edgeFrameCount = Number(options.edgeFrameCount ?? DEFAULT_EDGE_FRAME_COUNT); + if (!Number.isInteger(edgeFrameCount) || edgeFrameCount < 1) { + throw new Error(`edgeFrameCount must be a positive integer, got ${options.edgeFrameCount}`); + } + + const hashes = frames.map((frame) => (typeof frame === 'string' ? frame : frame.hash)); + const ranges = []; + let runStart = 0; + + for (let index = 1; index <= hashes.length; index += 1) { + if (index < hashes.length && hashes[index] === hashes[runStart]) continue; + + pushRemovalRangeForRun(ranges, runStart, index - 1, edgeFrameCount, { hash: hashes[runStart] }); + + runStart = index; + } + + return ranges; +} + +function pushRemovalRangeForRun(ranges, runStart, runEnd, edgeFrameCount, extra = {}) { + const runLength = runEnd - runStart + 1; + const removeStart = runStart + edgeFrameCount; + const removeEnd = runEnd - edgeFrameCount; + + if (removeStart <= removeEnd) { + ranges.push({ + start: removeStart, + end: removeEnd, + removedFrames: removeEnd - removeStart + 1, + runStart, + runEnd, + runLength, + ...extra, + }); + } +} + +function normalizeSimilarityThreshold(value) { + const threshold = Number(value ?? DEFAULT_SIMILARITY_THRESHOLD); + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + throw new Error(`similarityThreshold must be between 0 and 1, got ${value}`); + } + return threshold; +} + +function comparisonSize(mediaInfo, options = {}) { + const width = Number(mediaInfo.width); + const height = Number(mediaInfo.height); + if (!Number.isFinite(width) || width < 1 || !Number.isFinite(height) || height < 1) { + throw new Error(`could not determine video dimensions`); + } + + const requestedWidth = Number(options.comparisonWidth ?? 160); + const comparisonWidth = Math.max(1, Math.min(width, Math.round(requestedWidth))); + const comparisonHeight = Math.max(1, Math.round(height * (comparisonWidth / width))); + + return { + width: comparisonWidth, + height: comparisonHeight, + frameByteLength: comparisonWidth * comparisonHeight, + }; +} + +function normalizePixelTolerance(value) { + const tolerance = Number(value ?? DEFAULT_PIXEL_TOLERANCE); + if (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 255) { + throw new Error(`pixelTolerance must be between 0 and 255, got ${value}`); + } + return tolerance; +} + +function frameSimilarity(a, b, options = {}) { + if (a.length !== b.length) throw new Error(`cannot compare frames with different sizes`); + + const pixelTolerance = normalizePixelTolerance(options.pixelTolerance); + let matchingPixels = 0; + for (let index = 0; index < a.length; index += 1) { + if (Math.abs(a[index] - b[index]) <= pixelTolerance) matchingPixels += 1; + } + + return matchingPixels / a.length; +} + +async function findSimilarFrameRemovalRanges(inputPath, options = {}) { + const ffmpegPath = options.ffmpegPath || 'ffmpeg'; + const mediaInfo = options.mediaInfo || await readMediaInfo(inputPath, options); + const edgeFrameCount = Number(options.edgeFrameCount ?? DEFAULT_EDGE_FRAME_COUNT); + if (!Number.isInteger(edgeFrameCount) || edgeFrameCount < 1) { + throw new Error(`edgeFrameCount must be a positive integer, got ${options.edgeFrameCount}`); + } + + const similarityThreshold = normalizeSimilarityThreshold(options.similarityThreshold); + const pixelTolerance = normalizePixelTolerance(options.pixelTolerance); + const size = comparisonSize(mediaInfo, options); + const args = [ + '-hide_banner', + '-v', + 'error', + '-i', + inputPath, + '-map', + '0:v:0', + '-an', + '-vf', + `scale=${size.width}:${size.height},format=gray`, + '-f', + 'rawvideo', + '-', + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffmpegPath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const stderr = []; + const removalRanges = []; + let buffered = Buffer.alloc(0); + let anchor = null; + let frameIndex = 0; + let runStart = 0; + let weakestSimilarity = 1; + + function finishRun(runEnd) { + pushRemovalRangeForRun(removalRanges, runStart, runEnd, edgeFrameCount, { + weakestSimilarity: Number(weakestSimilarity.toFixed(6)), + }); + } + + function processFrame(frame) { + if (!anchor) { + anchor = Buffer.from(frame); + frameIndex += 1; + return; + } + + const similarity = frameSimilarity(anchor, frame, { pixelTolerance }); + if (similarity >= similarityThreshold) { + weakestSimilarity = Math.min(weakestSimilarity, similarity); + frameIndex += 1; + return; + } + + finishRun(frameIndex - 1); + anchor = Buffer.from(frame); + runStart = frameIndex; + weakestSimilarity = 1; + frameIndex += 1; + } + + child.stdout.on('data', (chunk) => { + buffered = buffered.length ? Buffer.concat([buffered, chunk]) : chunk; + while (buffered.length >= size.frameByteLength) { + processFrame(buffered.subarray(0, size.frameByteLength)); + buffered = buffered.subarray(size.frameByteLength); + } + }); + + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (code !== 0) { + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${ffmpegPath} raw frame decode failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + return; + } + + if (buffered.length) { + reject(new Error(`raw frame decode ended with ${buffered.length} trailing bytes; expected whole ${size.frameByteLength}-byte frames`)); + return; + } + + if (frameIndex > 0) finishRun(frameIndex - 1); + resolve({ + frameCount: frameIndex, + mediaInfo, + removalRanges, + detection: { + mode: 'similarity', + similarityThreshold, + pixelTolerance, + comparisonWidth: size.width, + comparisonHeight: size.height, + pixelFormat: 'gray', + }, + }); + }); + }); +} + +function escapeSelectFunction(name, values) { + return `${name}(${values.join('\\,')})`; +} + +function buildVideoRemovalExpression(removalRanges) { + return removalRanges + .map((range) => escapeSelectFunction('between', ['n', range.start, range.end])) + .join('+'); +} + +function buildVideoFilter(removalRanges, fps) { + const safeFps = String(fps || '').trim(); + if (!safeFps || safeFps === '0/0') throw new Error(`invalid fps for setpts: ${fps}`); + + const setPts = `setpts=N/(${safeFps})/TB`; + if (!removalRanges.length) return setPts; + + return `select=not(${buildVideoRemovalExpression(removalRanges)}),${setPts}`; +} + +function fpsToNumber(fps) { + const text = String(fps || '').trim(); + const match = text.match(/^(\d+)\/(\d+)$/); + if (match) { + const denominator = Number(match[2]); + return denominator ? Number(match[1]) / denominator : NaN; + } + return Number(text); +} + +function fixedSeconds(value) { + return Number(value).toFixed(6).replace(/0+$/, '').replace(/\.$/, '.0'); +} + +function buildAudioRemovalExpression(removalRanges, fps) { + const fpsNumber = fpsToNumber(fps); + if (!Number.isFinite(fpsNumber) || fpsNumber <= 0) { + throw new Error(`invalid fps for audio trimming: ${fps}`); + } + + return removalRanges + .map((range) => { + const start = fixedSeconds(range.start / fpsNumber); + const end = fixedSeconds((range.end + 1) / fpsNumber); + return `${escapeSelectFunction('gte', ['t', start])}*${escapeSelectFunction('lt', ['t', end])}`; + }) + .join('+'); +} + +function buildAudioFilter(removalRanges, fps) { + const setPts = 'asetpts=N/SR/TB'; + if (!removalRanges.length) return setPts; + return `aselect=not(${buildAudioRemovalExpression(removalRanges, fps)}),${setPts}`; +} + +function parseRate(rate) { + const text = String(rate || '').trim(); + const match = text.match(/^(\d+)\/(\d+)$/); + if (!match) return text; + + const numerator = Number(match[1]); + const denominator = Number(match[2]); + if (!denominator) return ''; + return denominator === 1 ? String(numerator) : `${numerator}/${denominator}`; +} + +async function readVideoFps(inputPath, options = {}) { + const mediaInfo = await readMediaInfo(inputPath, options); + return mediaInfo.fps; +} + +async function readMediaInfo(inputPath, options = {}) { + const ffprobePath = options.ffprobePath || 'ffprobe'; + const args = [ + '-v', + 'error', + '-show_entries', + 'stream=codec_type,width,height,avg_frame_rate,r_frame_rate', + '-of', + 'json', + inputPath, + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffprobePath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const stdout = []; + const stderr = []; + + child.stdout.on('data', (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (code !== 0) { + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${ffprobePath} failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + return; + } + + const parsed = JSON.parse(Buffer.concat(stdout).toString('utf8')); + const streams = parsed.streams || []; + const videoStream = streams.find((stream) => stream.codec_type === 'video'); + const fps = parseRate(videoStream?.avg_frame_rate) || parseRate(videoStream?.r_frame_rate); + if (!fps) reject(new Error(`could not determine video frame rate for ${inputPath}`)); + else { + resolve({ + fps, + width: videoStream.width, + height: videoStream.height, + hasAudio: streams.some((stream) => stream.codec_type === 'audio'), + }); + } + }); + }); +} + +async function encodeWithoutRepeatedFrames(inputPath, outputPath, removalRanges, options = {}) { + const ffmpegPath = options.ffmpegPath || 'ffmpeg'; + const mediaInfo = options.mediaInfo || await readMediaInfo(inputPath, options); + const fps = options.fps || mediaInfo.fps; + const videoFilter = buildVideoFilter(removalRanges, fps); + const shouldIncludeAudio = options.includeAudio ?? mediaInfo.hasAudio; + const outputExt = path.extname(outputPath).toLowerCase(); + const isMp4 = outputExt === '.mp4' || outputExt === '.m4v' || outputExt === '.mov'; + const videoCodec = options.videoCodec || (isMp4 ? 'libx264' : 'libvpx'); + ensureDir(path.dirname(outputPath)); + + const args = [ + '-hide_banner', + '-y', + '-v', + options.logLevel || 'error', + '-i', + inputPath, + ]; + + if (shouldIncludeAudio && mediaInfo.hasAudio) { + args.push( + '-filter_complex', + `[0:v:0]${videoFilter}[v];[0:a:0]${buildAudioFilter(removalRanges, fps)}[a]`, + '-map', + '[v]', + '-map', + '[a]' + ); + } else { + args.push( + '-map', + '0:v:0', + '-an', + '-vf', + videoFilter + ); + } + + args.push( + '-fps_mode', + 'cfr', + '-r', + fps, + '-c:v', + videoCodec + ); + + if (videoCodec === 'libvpx') { + args.push( + '-deadline', + options.deadline || 'realtime', + '-cpu-used', + String(options.cpuUsed ?? 8), + '-crf', + String(options.crf ?? 30), + '-b:v', + String(options.videoBitrate ?? '0') + ); + } else if (videoCodec === 'libx264') { + args.push( + '-preset', + options.preset || 'veryfast', + '-crf', + String(options.crf ?? 23), + '-pix_fmt', + 'yuv420p' + ); + } else { + args.push( + '-crf', + String(options.crf ?? 30) + ); + } + + if (shouldIncludeAudio && mediaInfo.hasAudio) { + args.push( + '-c:a', + options.audioCodec || (isMp4 ? 'aac' : 'libopus'), + '-b:a', + String(options.audioBitrate || '64k') + ); + } + + args.push( + outputPath, + ); + + await spawnChecked(ffmpegPath, args); + return { + outputPath, + fps, + videoFilter, + }; +} + +async function removeRepeatedFrames(inputPath, outputPath, options = {}) { + const analysis = await findSimilarFrameRemovalRanges(inputPath, options); + const removalRanges = analysis.removalRanges; + const removedFrameCount = removalRanges.reduce((sum, range) => sum + range.removedFrames, 0); + const encode = await encodeWithoutRepeatedFrames(inputPath, outputPath, removalRanges, { + ...options, + mediaInfo: analysis.mediaInfo, + }); + + return { + inputPath, + outputPath, + frameCount: analysis.frameCount, + keptFrameCount: analysis.frameCount - removedFrameCount, + removedFrameCount, + removalRanges, + detection: analysis.detection, + fps: encode.fps, + videoFilter: encode.videoFilter, + }; +} + +function rawVideoPath(videoPath, options = {}) { + const parsed = path.parse(videoPath); + const suffix = options.rawSuffix || '_raw'; + const preferred = path.join(parsed.dir, `${parsed.name}${suffix}${parsed.ext}`); + if (!fs.existsSync(preferred)) return preferred; + + for (let index = 2; index < 1000; index += 1) { + const candidate = path.join(parsed.dir, `${parsed.name}${suffix}-${index}${parsed.ext}`); + if (!fs.existsSync(candidate)) return candidate; + } + + throw new Error(`could not find available raw video name for ${videoPath}`); +} + +async function removeRepeatedFramesInPlace(videoPath, options = {}) { + if (!videoPath || !fs.existsSync(videoPath)) { + return null; + } + + const rawPath = options.rawPath || rawVideoPath(videoPath, options); + fs.renameSync(videoPath, rawPath); + + try { + const summary = await removeRepeatedFrames(rawPath, videoPath, options); + return { + video: videoPath, + rawVideo: rawPath, + repeatedFrameRemoval: summary, + }; + } catch (error) { + if (!fs.existsSync(videoPath) && fs.existsSync(rawPath)) { + fs.renameSync(rawPath, videoPath); + } + throw error; + } +} + +function printUsage() { + console.error('Usage: node controller/bin/remove-repeated-frames.js [--edge-frames 10] [--similarity-threshold 0.98] [--pixel-tolerance 3] [--comparison-width 160]'); +} + +async function main(argv) { + const args = [...argv]; + const inputPath = args.shift(); + const outputPath = args.shift(); + if (!inputPath || !outputPath) { + printUsage(); + process.exitCode = 2; + return; + } + + const options = {}; + while (args.length) { + const arg = args.shift(); + if (arg === '--edge-frames') { + options.edgeFrameCount = Number(args.shift()); + } else if (arg === '--similarity-threshold') { + options.similarityThreshold = Number(args.shift()); + } else if (arg === '--pixel-tolerance') { + options.pixelTolerance = Number(args.shift()); + } else if (arg === '--comparison-width') { + options.comparisonWidth = Number(args.shift()); + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + const summary = await removeRepeatedFrames(inputPath, outputPath, options); + console.log(JSON.stringify(summary, null, 2)); +} + +if (require.main === module) { + main(process.argv.slice(2)).catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }); +} + +module.exports = { + DEFAULT_EDGE_FRAME_COUNT, + DEFAULT_PIXEL_TOLERANCE, + DEFAULT_SIMILARITY_THRESHOLD, + buildAudioFilter, + buildVideoFilter, + collectFrameHashes, + encodeWithoutRepeatedFrames, + findRepeatedFrameRemovalRanges, + findSimilarFrameRemovalRanges, + frameSimilarity, + main, + parseFrameMd5Line, + rawVideoPath, + readMediaInfo, + readVideoFps, + removeRepeatedFrames, + removeRepeatedFramesInPlace, +}; diff --git a/ops/remote/run-playtest.js b/ops/remote/run-playtest.js index 3beeac1..6546323 100755 --- a/ops/remote/run-playtest.js +++ b/ops/remote/run-playtest.js @@ -583,6 +583,7 @@ async function main() { chromiumArgsMode: job.chromiumArgsMode, keyAliases: job.keyAliases, gridScreenshots: job.gridScreenshots, + repeatedFrameRemoval: job.repeatedFrameRemoval, }; if (job.videoSize) startOverrides.videoSize = job.videoSize; diff --git a/test/browser-session.test.js b/test/browser-session.test.js index 69029a5..c79585a 100644 --- a/test/browser-session.test.js +++ b/test/browser-session.test.js @@ -4,7 +4,10 @@ const test = require('node:test'); const { browserViewportStabilizerScript, chromiumLaunchArgs, + isRepeatedFrameRemovalEnabled, + repeatedFrameRemovalOptions, } = require('../controller/src/browser-session'); +const { rawVideoPath } = require('../controller/src/repeated-frame-remover'); test('chromium launch args leave non-recording runs unchanged', () => { const args = chromiumLaunchArgs({ record: false }, {}); @@ -39,3 +42,33 @@ test('browser viewport stabilizer hides overflow and prevents scrolling keys', ( assert.match(source, /ArrowDown/); assert.match(source, /preventDefault/); }); + +test('repeated frame removal config is opt-in', () => { + assert.equal(isRepeatedFrameRemovalEnabled({ record: true }), false); + assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: true }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: { similarityThreshold: 0.98 } }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: false }), false); +}); + +test('repeated frame removal options pass through supported settings', () => { + assert.deepEqual(repeatedFrameRemovalOptions({ + ffmpegPath: '/opt/ffmpeg', + repeatedFrameRemoval: { + edgeFrameCount: 10, + similarityThreshold: 0.98, + pixelTolerance: 3, + comparisonWidth: 160, + }, + }), { + edgeFrameCount: 10, + similarityThreshold: 0.98, + pixelTolerance: 3, + comparisonWidth: 160, + ffmpegPath: '/opt/ffmpeg', + ffprobePath: undefined, + }); +}); + +test('rawVideoPath appends raw suffix before extension', () => { + assert.equal(rawVideoPath('/tmp/video/000-runwave-with-audio.webm'), '/tmp/video/000-runwave-with-audio_raw.webm'); +}); diff --git a/test/repeated-frame-remover.test.js b/test/repeated-frame-remover.test.js new file mode 100644 index 0000000..9638956 --- /dev/null +++ b/test/repeated-frame-remover.test.js @@ -0,0 +1,86 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { + DEFAULT_SIMILARITY_THRESHOLD, + DEFAULT_EDGE_FRAME_COUNT, + DEFAULT_PIXEL_TOLERANCE, + buildVideoFilter, + buildAudioFilter, + findRepeatedFrameRemovalRanges, + frameSimilarity, + parseFrameMd5Line, +} = require('../controller/src/repeated-frame-remover'); + +test('parseFrameMd5Line reads frame hash rows', () => { + assert.deepEqual( + parseFrameMd5Line('0, 7, 7, 1, 1382400, 4ac1f0bdba53f0f96e5cb4aaa499dfac'), + { + streamIndex: 0, + dts: 7, + pts: 7, + duration: 1, + size: 1382400, + hash: '4ac1f0bdba53f0f96e5cb4aaa499dfac', + } + ); + assert.equal(parseFrameMd5Line('#stream#, dts, pts, duration, size, hash'), null); +}); + +test('findRepeatedFrameRemovalRanges honors explicit edge frame counts', () => { + const hashes = [ + ...Array(5).fill('a'), + ...Array(6).fill('b'), + ...Array(10).fill('c'), + ...Array(11).fill('d'), + ...Array(14).fill('e'), + 'f', + ]; + + assert.deepEqual(findRepeatedFrameRemovalRanges(hashes, { edgeFrameCount: 5 }), [ + { + start: 26, + end: 26, + removedFrames: 1, + runStart: 21, + runEnd: 31, + runLength: 11, + hash: 'd', + }, + { + start: 37, + end: 40, + removedFrames: 4, + runStart: 32, + runEnd: 45, + runLength: 14, + hash: 'e', + }, + ]); +}); + +test('buildVideoFilter removes frame ranges and closes timestamps at the original fps', () => { + assert.equal( + buildVideoFilter([ + { start: 5, end: 9 }, + { start: 20, end: 22 }, + ], '25'), + 'select=not(between(n\\,5\\,9)+between(n\\,20\\,22)),setpts=N/(25)/TB' + ); + assert.equal(buildVideoFilter([], '25'), 'setpts=N/(25)/TB'); +}); + +test('frameSimilarity scores normalized byte similarity', () => { + assert.equal(DEFAULT_EDGE_FRAME_COUNT, 10); + assert.equal(DEFAULT_SIMILARITY_THRESHOLD, 0.98); + assert.equal(DEFAULT_PIXEL_TOLERANCE, 3); + assert.equal(frameSimilarity(Buffer.from([0, 100]), Buffer.from([0, 110])), 0.5); + assert.equal(frameSimilarity(Buffer.from([0, 100]), Buffer.from([0, 110]), { pixelTolerance: 10 }), 1); +}); + +test('buildAudioFilter removes matching time spans for removed video frames', () => { + assert.equal( + buildAudioFilter([{ start: 5, end: 9 }], '25'), + 'aselect=not(gte(t\\,0.2)*lt(t\\,0.4)),asetpts=N/SR/TB' + ); +}); diff --git a/test/session-cleanup.test.js b/test/session-cleanup.test.js index 5ebaa64..0fc978b 100644 --- a/test/session-cleanup.test.js +++ b/test/session-cleanup.test.js @@ -56,6 +56,29 @@ test('start reuses a live session only when the start configuration matches', as } }); +test('start configuration records repeated frame removal settings', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'start-cut-video', + url: 'http://127.0.0.1:3000/', + record: true, + repeatedFrameRemoval: { + edgeFrameCount: 10, + similarityThreshold: 0.98, + pixelTolerance: 3, + comparisonWidth: 160, + }, + }); + + assert.deepEqual(config.context.repeatedFrameRemoval, { + enabled: true, + edgeFrameCount: 10, + similarityThreshold: 0.98, + pixelTolerance: 3, + comparisonWidth: 160, + }); +}); + test('start rejects a live session with a different target instead of silently reusing it', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-target-mismatch-')); const originalStart = { From 94498ea4a48a69472f19bec1bdd795cd3831a7a4 Mon Sep 17 00:00:00 2001 From: user Date: Wed, 22 Jul 2026 14:37:07 +0700 Subject: [PATCH 2/2] deafult on --- README.md | 6 +- runwave/controller/README.md | 24 ++++--- runwave/controller/src/browser-session.js | 28 ++------ runwave/controller/src/linux-session.js | 66 ++++++++++++++----- runwave/controller/src/protocol.js | 9 +-- .../controller/src/repeated-frame-remover.js | 35 +++++----- .../controller/test/browser-session.test.js | 29 ++++---- .../controller/test/session-cleanup.test.js | 36 ++++++---- stress-test/README.md | 6 +- stress-test/remote/playtest-runner.Dockerfile | 1 + 10 files changed, 137 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 6b415a1..b73a5f7 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,10 @@ Runwave's recording pipeline is **gstreamer-only** as other methods lead to audi - **gstreamer 1.x** with `ximagesrc`, `pulsesrc`, `vp8enc`, `opusenc`, `webmmux`, and `filesink` available on `PATH` as `gst-launch-1.0` (override via the `RUNWAVE_GSTREAMER` env var or the `gstreamerPath` start option). +- **ffmpeg and ffprobe** available on `PATH` for the default repeated-frame + removal pass that produces the shorter final recording. Pass + `repeatedFrameRemoval: false` to keep the raw gstreamer recording without + post-processing. - **An X server or Xvfb.** `DISPLAY` must be set to a display that Chromium or a native Linux game can render into and that `ximagesrc` can read. - **PulseAudio running.** `pactl info` must succeed. Chromium's audio must be @@ -61,7 +65,7 @@ From a private GitHub repo in a task Dockerfile: RUN apt-get update && apt-get install -y \ gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-bad \ gstreamer1.0-plugins-ugly gstreamer1.0-x gstreamer1.0-pulseaudio \ - pulseaudio xvfb xdotool + ffmpeg pulseaudio xvfb xdotool RUN npm install -g https://github.com/parsewave/runwave.git RUN npx playwright install --with-deps chromium ``` diff --git a/runwave/controller/README.md b/runwave/controller/README.md index 8e2306d..23c6bef 100644 --- a/runwave/controller/README.md +++ b/runwave/controller/README.md @@ -6,7 +6,8 @@ executes timed input sequences, records WebM output, and writes per-action artifacts. See the top-level [Requirements](../../README.md#requirements) section for the -Linux, gstreamer, X server/Xvfb, and PulseAudio setup required by `record: true`. +Linux, gstreamer, ffmpeg/ffprobe, X server/Xvfb, and PulseAudio setup required +by `record: true`. ## Start @@ -87,9 +88,8 @@ Useful `start` options: legacy alias for `record`; they mean the same thing - audio is always captured when recording.) - `repeatedFrameRemoval`: when `record` is enabled, post-process the final WebM - to remove repeated-frame runs before returning the stop response. Pass `true` - for defaults, or an options object such as - `{ "similarityThreshold": 0.98, "edgeFrameCount": 10 }`. The raw capture is + by default to remove repeated-frame runs before returning the stop response. + Pass `false` to keep only the raw recording. When enabled, the raw capture is renamed with a `_raw` suffix. - `headless`: defaults to `true`; set `false` to watch the browser. - `channel`: optional Playwright browser channel, such as `chrome` or `msedge`. @@ -243,13 +243,15 @@ runwave-controller '{"action":"sessions"}' When `record: true` is set, `stop` returns `video` and `audioVideo` pointing at the same recorded audio/video WebM. All recording goes through gstreamer - see the top-level [Requirements](../../README.md#requirements) section for the -mandatory environment (Linux, gstreamer, X server/Xvfb, PulseAudio). +mandatory environment (Linux, gstreamer, ffmpeg/ffprobe, X server/Xvfb, +PulseAudio). -When `repeatedFrameRemoval` was set on `start`, `video` and `audioVideo` -point at the cut WebM at the normal recording path. The original uncut capture +By default, `video` and `audioVideo` point at the shorter WebM at the normal +recording path after repeated-frame runs are removed. The original uncut capture is preserved as `rawVideo` / `rawAudioVideo`, typically `video/000-runwave-with-audio_raw.webm`, and `repeatedFrameRemoval` reports the -removed frame ranges. +removed frame ranges. Pass `repeatedFrameRemoval: false` on `start` to return +the raw recording path without post-processing. ## State @@ -288,7 +290,11 @@ Each turn writes: - `response.json`: the main CLI response. - `*.png`: screenshots captured during that operation. - `NNN-.json`: detailed sequence log for `step` operations. -- `video/000-runwave-with-audio.webm`: final gstreamer audio+video recording. +- `video/000-runwave-with-audio.webm`: final shorter audio/video recording when + repeated-frame removal is enabled, or the raw gstreamer recording when it is + disabled. +- `video/000-runwave-with-audio_raw.webm`: original recording when repeated + frame removal is enabled. Active sessions are tracked as JSON files in `.runwave-sessions/` by default. The matching session file is removed by `stop`. diff --git a/runwave/controller/src/browser-session.js b/runwave/controller/src/browser-session.js index f1c6248..ad8faab 100644 --- a/runwave/controller/src/browser-session.js +++ b/runwave/controller/src/browser-session.js @@ -7,7 +7,11 @@ const { AudioVideoRecorder } = require('./audio-recorder'); const { ensureDir, safeName, sleep, timestamp } = require('./file-utils'); const { drawGridOnScreenshot } = require('./grid-overlay'); const { parseArgList, targetUrl } = require('./protocol'); -const { removeRepeatedFramesInPlace } = require('./repeated-frame-remover'); +const { + isRepeatedFrameRemovalEnabled, + removeRepeatedFramesInPlace, + repeatedFrameRemovalOptions, +} = require('./repeated-frame-remover'); const { readPageState } = require('./state-reader'); const DEFAULT_CHROMIUM_ARGS = [ @@ -43,24 +47,6 @@ function isRecording(config = {}) { return Boolean(config.record || config.recordAudio); } -function isRepeatedFrameRemovalEnabled(config = {}) { - return Boolean(config.repeatedFrameRemoval); -} - -function repeatedFrameRemovalOptions(config = {}) { - const options = config.repeatedFrameRemoval && typeof config.repeatedFrameRemoval === 'object' - ? config.repeatedFrameRemoval - : {}; - return { - edgeFrameCount: options.edgeFrameCount, - similarityThreshold: options.similarityThreshold, - pixelTolerance: options.pixelTolerance, - comparisonWidth: options.comparisonWidth, - ffmpegPath: options.ffmpegPath || config.ffmpegPath, - ffprobePath: options.ffprobePath || config.ffprobePath, - }; -} - function chromiumLaunchArgs(config = {}, env = process.env) { const args = chromiumArgs(config, env); if (!isRecording(config)) return args; @@ -522,7 +508,7 @@ class BrowserSession { } if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) { const processed = await this.time('browser.close.remove_repeated_frames', { video: audioVideoPath }, () => - removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions(closeConfig)) + removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions()) ); audioVideoPath = processed.video; rawVideoPath = processed.rawVideo; @@ -562,8 +548,6 @@ module.exports = { browserViewportStabilizerScript, chromiumArgs, chromiumLaunchArgs, - isRepeatedFrameRemovalEnabled, - repeatedFrameRemovalOptions, launchHeadless, pageViewportVideoSource, webLaunchConfig, diff --git a/runwave/controller/src/linux-session.js b/runwave/controller/src/linux-session.js index 49bc8e5..c180c0a 100644 --- a/runwave/controller/src/linux-session.js +++ b/runwave/controller/src/linux-session.js @@ -5,6 +5,11 @@ const { AudioVideoRecorder, defaultVideoSource, parseX11VideoSource } = require( const { ensureDir, safeName, sleep, timestamp } = require('./file-utils'); const { drawGridOnScreenshot } = require('./grid-overlay'); const { parseArgList } = require('./protocol'); +const { + isRepeatedFrameRemovalEnabled, + removeRepeatedFramesInPlace, + repeatedFrameRemovalOptions, +} = require('./repeated-frame-remover'); const DEFAULT_WINDOW_WAIT_MS = 15000; const DEFAULT_LINUX_LAUNCH_SETTLE_MS = 30000; @@ -597,27 +602,58 @@ class LinuxSession { }; } - async close() { + async close(overrides = {}) { + const hasRepeatedFrameRemovalOverride = Object.prototype.hasOwnProperty.call(overrides, 'repeatedFrameRemoval'); + const closeConfig = { + ...this.config, + ...overrides, + repeatedFrameRemoval: hasRepeatedFrameRemovalOverride + ? overrides.repeatedFrameRemoval + : this.config.repeatedFrameRemoval, + }; let audioVideoPath = null; - if (this.audioRecorder) { - audioVideoPath = await this.time('linux.close.audio_video_stop', () => this.audioRecorder.stop()); + let rawVideoPath = null; + let repeatedFrameRemoval = null; + let closeError = null; + try { + if (this.audioRecorder) { + audioVideoPath = await this.time('linux.close.audio_video_stop', () => this.audioRecorder.stop()); + } + if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) { + const processed = await this.time('linux.close.remove_repeated_frames', { video: audioVideoPath }, () => + removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions()) + ); + audioVideoPath = processed.video; + rawVideoPath = processed.rawVideo; + repeatedFrameRemoval = processed.repeatedFrameRemoval; + } + } catch (error) { + closeError = error; } - if (this.process && !processHasClosed(this.process)) { - await this.time('linux.close.terminate_process', async () => { - signalProcessGroup(this.process, 'SIGTERM'); - if (!(await waitForProcessClose(this.process, 5000))) { - signalProcessGroup(this.process, 'SIGKILL'); - await waitForProcessClose(this.process, 5000); - } - }); - } else if (this.windowId) { - try { - this.timeSync('linux.close.window_close', () => this.xdotool(['windowclose', this.windowId], { timeoutMs: 1000 })); - } catch {} + try { + if (this.process && !processHasClosed(this.process)) { + await this.time('linux.close.terminate_process', async () => { + signalProcessGroup(this.process, 'SIGTERM'); + if (!(await waitForProcessClose(this.process, 5000))) { + signalProcessGroup(this.process, 'SIGKILL'); + await waitForProcessClose(this.process, 5000); + } + }); + } else if (this.windowId) { + try { + this.timeSync('linux.close.window_close', () => this.xdotool(['windowclose', this.windowId], { timeoutMs: 1000 })); + } catch {} + } + } catch (error) { + if (!closeError) closeError = error; } + if (closeError) throw closeError; return { video: audioVideoPath, audioVideo: audioVideoPath || undefined, + rawVideo: rawVideoPath || undefined, + rawAudioVideo: rawVideoPath || undefined, + repeatedFrameRemoval: repeatedFrameRemoval || undefined, }; } } diff --git a/runwave/controller/src/protocol.js b/runwave/controller/src/protocol.js index 25424d9..621549f 100644 --- a/runwave/controller/src/protocol.js +++ b/runwave/controller/src/protocol.js @@ -144,19 +144,12 @@ function optionalPositiveInteger(value) { } function repeatedFrameRemovalEnabled(input) { - return Boolean(input.repeatedFrameRemoval); + return Boolean(input.record || input.recordAudio) && input.repeatedFrameRemoval !== false; } function repeatedFrameRemovalConfig(input) { - const options = input.repeatedFrameRemoval && typeof input.repeatedFrameRemoval === 'object' - ? input.repeatedFrameRemoval - : {}; return { enabled: repeatedFrameRemovalEnabled(input), - edgeFrameCount: optionalNumber(options.edgeFrameCount, 10), - similarityThreshold: optionalNumber(options.similarityThreshold, 0.98), - pixelTolerance: optionalNumber(options.pixelTolerance, 3), - comparisonWidth: optionalNumber(options.comparisonWidth, 160), }; } diff --git a/runwave/controller/src/repeated-frame-remover.js b/runwave/controller/src/repeated-frame-remover.js index e2b72f7..6ed86ae 100644 --- a/runwave/controller/src/repeated-frame-remover.js +++ b/runwave/controller/src/repeated-frame-remover.js @@ -6,6 +6,19 @@ const DEFAULT_EDGE_FRAME_COUNT = 10; const DEFAULT_SIMILARITY_THRESHOLD = 0.98; const DEFAULT_PIXEL_TOLERANCE = 3; +function isRepeatedFrameRemovalEnabled(config = {}) { + return Boolean(config.record || config.recordAudio) && config.repeatedFrameRemoval !== false; +} + +function repeatedFrameRemovalOptions() { + return { + edgeFrameCount: DEFAULT_EDGE_FRAME_COUNT, + similarityThreshold: DEFAULT_SIMILARITY_THRESHOLD, + pixelTolerance: DEFAULT_PIXEL_TOLERANCE, + comparisonWidth: 160, + }; +} + function ensureDir(dir) { fs.mkdirSync(dir, { recursive: true }); return dir; @@ -563,7 +576,7 @@ async function removeRepeatedFramesInPlace(videoPath, options = {}) { } function printUsage() { - console.error('Usage: node runwave/controller/bin/remove-repeated-frames.js [--edge-frames 10] [--similarity-threshold 0.98] [--pixel-tolerance 3] [--comparison-width 160]'); + console.error('Usage: node runwave/controller/bin/remove-repeated-frames.js '); } async function main(argv) { @@ -576,23 +589,11 @@ async function main(argv) { return; } - const options = {}; - while (args.length) { - const arg = args.shift(); - if (arg === '--edge-frames') { - options.edgeFrameCount = Number(args.shift()); - } else if (arg === '--similarity-threshold') { - options.similarityThreshold = Number(args.shift()); - } else if (arg === '--pixel-tolerance') { - options.pixelTolerance = Number(args.shift()); - } else if (arg === '--comparison-width') { - options.comparisonWidth = Number(args.shift()); - } else { - throw new Error(`unknown argument: ${arg}`); - } + if (args.length) { + throw new Error(`unknown argument: ${args[0]}`); } - const summary = await removeRepeatedFrames(inputPath, outputPath, options); + const summary = await removeRepeatedFrames(inputPath, outputPath, repeatedFrameRemovalOptions()); console.log(JSON.stringify(summary, null, 2)); } @@ -614,11 +615,13 @@ module.exports = { findRepeatedFrameRemovalRanges, findSimilarFrameRemovalRanges, frameSimilarity, + isRepeatedFrameRemovalEnabled, main, parseFrameMd5Line, rawVideoPath, readMediaInfo, readVideoFps, + repeatedFrameRemovalOptions, removeRepeatedFrames, removeRepeatedFramesInPlace, }; diff --git a/runwave/controller/test/browser-session.test.js b/runwave/controller/test/browser-session.test.js index 4334e14..182c7f6 100644 --- a/runwave/controller/test/browser-session.test.js +++ b/runwave/controller/test/browser-session.test.js @@ -6,13 +6,15 @@ const { BrowserSession, browserViewportStabilizerScript, chromiumLaunchArgs, - isRepeatedFrameRemovalEnabled, launchHeadless, pageViewportVideoSource, - repeatedFrameRemovalOptions, webLaunchConfig, } = require('../src/browser-session'); -const { rawVideoPath } = require('../src/repeated-frame-remover'); +const { + isRepeatedFrameRemovalEnabled, + rawVideoPath, + repeatedFrameRemovalOptions, +} = require('../src/repeated-frame-remover'); test('chromium launch args leave non-recording runs unchanged', () => { const args = chromiumLaunchArgs({ record: false }, {}); @@ -81,29 +83,20 @@ test('browser viewport stabilizer hides overflow and prevents scrolling keys', ( assert.match(source, /preventDefault/); }); -test('repeated frame removal config is opt-in', () => { - assert.equal(isRepeatedFrameRemovalEnabled({ record: true }), false); +test('repeated frame removal config defaults on for recordings', () => { + assert.equal(isRepeatedFrameRemovalEnabled({ record: true }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ recordAudio: true }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ record: false }), false); assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: true }), true); - assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: { similarityThreshold: 0.98 } }), true); assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: false }), false); }); -test('repeated frame removal options pass through supported settings', () => { - assert.deepEqual(repeatedFrameRemovalOptions({ - ffmpegPath: '/opt/ffmpeg', - repeatedFrameRemoval: { - edgeFrameCount: 10, - similarityThreshold: 0.98, - pixelTolerance: 3, - comparisonWidth: 160, - }, - }), { +test('repeated frame removal options are hard-coded', () => { + assert.deepEqual(repeatedFrameRemovalOptions(), { edgeFrameCount: 10, similarityThreshold: 0.98, pixelTolerance: 3, comparisonWidth: 160, - ffmpegPath: '/opt/ffmpeg', - ffprobePath: undefined, }); }); diff --git a/runwave/controller/test/session-cleanup.test.js b/runwave/controller/test/session-cleanup.test.js index d504444..eabf4bf 100644 --- a/runwave/controller/test/session-cleanup.test.js +++ b/runwave/controller/test/session-cleanup.test.js @@ -62,21 +62,33 @@ test('start configuration records repeated frame removal settings', () => { action_name: 'start-cut-video', url: 'http://127.0.0.1:3000/', record: true, - repeatedFrameRemoval: { - edgeFrameCount: 10, - similarityThreshold: 0.98, - pixelTolerance: 3, - comparisonWidth: 160, - }, + repeatedFrameRemoval: true, }); - assert.deepEqual(config.context.repeatedFrameRemoval, { - enabled: true, - edgeFrameCount: 10, - similarityThreshold: 0.98, - pixelTolerance: 3, - comparisonWidth: 160, + assert.deepEqual(config.context.repeatedFrameRemoval, { enabled: true }); +}); + +test('start configuration enables repeated frame removal by default for recordings', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'start-short-video', + url: 'http://127.0.0.1:3000/', + record: true, }); + + assert.deepEqual(config.context.repeatedFrameRemoval, { enabled: true }); +}); + +test('start configuration can disable repeated frame removal', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'start-raw-video', + url: 'http://127.0.0.1:3000/', + record: true, + repeatedFrameRemoval: false, + }); + + assert.deepEqual(config.context.repeatedFrameRemoval, { enabled: false }); }); test('start rejects a live session with a different target instead of silently reusing it', async () => { diff --git a/stress-test/README.md b/stress-test/README.md index cd6ceb6..6822f48 100644 --- a/stress-test/README.md +++ b/stress-test/README.md @@ -69,8 +69,10 @@ one GStreamer process to record the cropped X display plus `runwave_sink.monitor into `video/000-runwave-with-audio.webm`. This keeps audio from concurrent games on the same worker out of each other's recordings and avoids post-mux sync offsets. -Recorded fleet jobs use this GStreamer path; there is no Playwright video-only -fallback for production playtests. +Recorded fleet jobs use this GStreamer path and post-process it by default to +remove repeated frames, leaving the shorter video at the normal path and the raw +capture with a `_raw` suffix. There is no Playwright video-only fallback for +production playtests. The default game source is `s3://pw-cruft/games`. Override it with: diff --git a/stress-test/remote/playtest-runner.Dockerfile b/stress-test/remote/playtest-runner.Dockerfile index c040fd5..7b93d5f 100644 --- a/stress-test/remote/playtest-runner.Dockerfile +++ b/stress-test/remote/playtest-runner.Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + ffmpeg \ git \ gstreamer1.0-plugins-base \ gstreamer1.0-plugins-good \