From 6b2179c8b7de2cbc528131d8c127e6720dc73c98 Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 18:29:39 +0800 Subject: [PATCH 1/9] docs: add implementation guide for line-caption shorts Co-Authored-By: Claude Sonnet 5 --- .../LINE_CAPTION_SHORTS.md | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 docs/implementation-guides/LINE_CAPTION_SHORTS.md diff --git a/docs/implementation-guides/LINE_CAPTION_SHORTS.md b/docs/implementation-guides/LINE_CAPTION_SHORTS.md new file mode 100644 index 0000000..a3c2710 --- /dev/null +++ b/docs/implementation-guides/LINE_CAPTION_SHORTS.md @@ -0,0 +1,280 @@ +# Line-caption Shorts — Implementation Plan + +## How agents use this document + +This document is the authoritative implementation guide for the line-caption +shorts feature. Work is broken into isolated commits. Each commit is +independently testable. + +**To resume interrupted work:** +1. Run `git log --oneline` to see which commits are complete. +2. Match the last commit message against the slugs below. +3. Continue from the next unstarted step. + +**Rules:** +- Implement commits in order — each step depends on the previous. +- Do not combine steps into one commit. Isolation is intentional. +- The "Status check" line under each commit tells you how to verify it is + already done. +- If a commit is partially done (files exist but broken), fix it within that + commit before moving on. + +--- + +## Architecture overview + +A dedicated, standalone pipeline that takes a raw short-form portrait video +and produces a burned-in-caption render, independent of the longform/shorts +camera+hook pipeline: + +``` +raw portrait video + ↓ [captions:create] extract audio → transcribe → align → diarize (if + --num-speakers > 1) → chunk into 3-word lines → + write lines.json + lines.doc.txt + Human edits lines.doc.txt (rewords lines; ids/timestamps untouched) + ↓ [captions:merge] re-parses doc, overwrites line text in lines.json + ↓ [captions:render] Remotion renders raw video pass-through + burned-in + 3-word caption lines +``` + +Output structure: + +``` +public/line-captions/ + {id}/ + source.mp4 copy of the raw input video + audio.wav extracted mono 16kHz audio + transcribe/ Transcriber output (transcript.raw.json, .vtt) + lines.json LineCaptionsDoc — meta + CaptionLine[] + lines.doc.txt human-editable doc (speaker headers + [id] text) +``` + +`lines.json` is a new artifact, distinct from `transcript.json` — it has no +segments/tokens/cuts, just fixed-window caption lines. See +`remotion/types/lineCaptions.ts`. + +### Line-break rule + +Lines never mix two speakers — chunking never carries a run of words across a +segment boundary (segments already partition by speaker after +`assign-speakers` runs). A trailing partial run (1-2 words) at the end of a +segment stands alone rather than borrowing from the next segment/speaker. + +### Editing model + +`lines.doc.txt` is edited for wording only. The `[id]` markers must not be +added/removed/reordered — `captions:merge` maps edited text back onto the +existing `lines.json` entry by id and leaves `startMs`/`endMs`/`speaker` +untouched (best-effort timing; no re-alignment). + +--- + +## Commit checklist + +### Commit 1 — `feat: add CaptionLine types and chunkIntoLines` + +Add `remotion/types/lineCaptions.ts`: +```ts +import type { Segment } from './transcript'; + +export type CaptionLine = { id: number; speaker: string; text: string; startMs: number; endMs: number }; +export type LineCaptionsMeta = { title: string; duration: number; fps: number; videoSrc?: string }; +export type LineCaptionsDoc = { meta: LineCaptionsMeta; lines: CaptionLine[] }; +``` + +Add `scripts/line-captions/chunkLines.js` exporting +`chunkIntoLines(segments, wordsPerLine = 3)`: +- Filters each segment's tokens to spoken, non-cut tokens using a local + `isSpokenToken(token)` port of the regex in `remotion/lib/tokens.ts` (kept + as a plain-JS duplicate here, matching the existing precedent of + `isSpecialToken` being duplicated across `align-transcript.js` and + `edit-transcript.js` rather than shared). +- Buckets spoken tokens into runs of exactly `wordsPerLine`, never spanning a + segment boundary. A trailing partial run stands alone. +- `startMs = firstToken.t_dtw * 1000`. `endMs` = the next run's start time + within the same segment, or for the last run in a segment: + `(lastToken.t_end ?? segment.end) * 1000`. +- Assigns sequential `id` starting at 1 across the whole transcript. +- `speaker` comes from `segment.speaker`. + +Add `scripts/line-captions/chunkLines.test.js` covering: +- An exact multiple of 3 spoken tokens → one line per 3 words. +- A non-multiple → trailing partial line of 1-2 words. +- Two adjacent segments with different speakers → no line spans both. +- `cut: true` tokens/segments are excluded entirely. +- A token without `t_end` falls back to `segment.end` for the last run. + +**Status check:** `remotion/types/lineCaptions.ts` and +`scripts/line-captions/chunkLines.js` exist; `npm run test:unit -- chunkLines` +passes. + +--- + +### Commit 2 — `feat: add captions:create script` + +Add `scripts/line-captions/create-line-captions.js`, CLI: +``` +node scripts/line-captions/create-line-captions.js --video [--num-speakers N] [--id ] +``` +- Resolves `id`: uses `--id` if given, else auto-increments `clip-{n}` by + scanning `public/line-captions/` (same scheme as `nextId` in + `scripts/shorts-wizard.js:114-120`). +- `fs.ensureDir` + copy the source video to + `public/line-captions/{id}/source.mp4`. +- Extract audio to `public/line-captions/{id}/audio.wav` using the same + ffmpeg spawn invocation as `extractAudio()` in `scripts/wizard.js:63-90` + (`-vn -ar 16000 -ac 1`). +- `const transcriber = new Transcriber({ audioPath, outputDir: path.join(clipDir, 'transcribe') }); await transcriber.init(); await transcriber.transcribe();` + (import `Transcriber` from `../transcribe/Transcriber.js`, used unmodified). +- Spawn `node scripts/align/align-transcript.js --audio --raw ` + to populate `token.t_end` (always run — unmodified script). +- If `--num-speakers` is given and `> 1`, spawn (in order, unmodified): + - `node scripts/diarize/diarize-audio.js --audio --output --num-speakers N` + - `node scripts/diarize/assign-speakers.js --diarization --raw ` +- Load `transcript.raw.json`, call `chunkIntoLines(segments)`. +- Write `public/line-captions/{id}/lines.json`: + `{ meta: { title, duration, fps, videoSrc: 'line-captions/{id}/source.mp4' }, lines }` + (`title`/`duration`/`fps` copied from `transcript.raw.json.meta`). +- Write `public/line-captions/{id}/lines.doc.txt` via a new `buildLineDoc(linesDoc)`: + group consecutive lines by `speaker` into `=== NAME ===` blocks (blank line + before/after, matching the visual grammar of `buildDoc()` in + `scripts/edit-transcript.js:683-756`), then one `[id] text` line per + `CaptionLine`. +- Print next-step instructions: open the doc, edit wording only, then run + `npm run captions:merge -- --id {id}`. + +**Status check:** `scripts/line-captions/create-line-captions.js` exists. +Running it against a short sample portrait clip produces +`public/line-captions/{id}/lines.json` and `lines.doc.txt`, and `lines.doc.txt` +groups lines under `=== SPEAKER ===` headers in readable 3-word chunks. + +--- + +### Commit 3 — `feat: add captions:merge script` + +Add `scripts/line-captions/merge-line-captions.js`, CLI: +``` +node scripts/line-captions/merge-line-captions.js --id +``` +- Reads `public/line-captions/{id}/lines.doc.txt` and `lines.json`. +- Parses the doc: tracks current speaker from `=== NAME ===` headers, matches + `/^\[(\d+)\]\s*(.*)$/` per line to get `(id, text)` pairs. +- For each parsed id found in `lines.json`, overwrites only its `text` field. + `startMs`/`endMs`/`speaker` are left untouched. +- `console.warn` (does not throw) for: a doc id with no matching `lines.json` + entry; a `lines.json` id missing from the doc entirely. +- Writes the updated `lines.json` back in place. + +**Status check:** `scripts/line-captions/merge-line-captions.js` exists. +Hand-editing one line's text in `lines.doc.txt` and running the merge command +updates that line's `text` in `lines.json` while its `startMs`/`endMs` stay +identical to before the edit. + +--- + +### Commit 4 — `feat: add LineCaptionClip Remotion composition` + +Add `remotion/components/LineCaptionOverlay.tsx`: +- Props: `{ lines: CaptionLine[]; brand: Brand }`. +- `const frame = useCurrentFrame(); const { fps } = useVideoConfig(); const ms = (frame / fps) * 1000;` +- `const active = lines.find(l => ms >= l.startMs && ms < l.endMs) ?? null;` +- Renders `active.text` using the same layout constants as + `CaptionOverlay.tsx` (`CAPTION_TOP`, `CAPTION_FONT_SIZE`, brand typography, + text-shadow) for visual consistency with the rest of the brand's captions. +- When `lines` contains more than one distinct `speaker`, tint the text color + per-speaker (stable hash of speaker name into `brand.colors` accent slots, + or a simple two-way alternation — keep it simple, this is cosmetic). + +Add `remotion/LineCaptionClip.tsx`: +- Props: `{ src: string; linesSrc: string; brandSrc?: string; brandId?: string }`. +- Loads `linesSrc` and brand JSON using the same `fetchJson` / + `normalizeStaticPath` / `delayRender`+`continueRender` pattern as + `ShortFormClip.tsx:42-46,264-293`. +- Renders `` — a + plain full-length pass-through, no `SegmentPlayer`/`CameraPlayer`, no cuts — + plus ``. +- Export `calculateLineCaptionMetadata: CalculateMetadataFunction<...>` that + fetches `linesSrc`, sets `durationInFrames = Math.ceil(meta.duration * fps)`, + `fps: 60, width: 1080, height: 1920`. + +Update `remotion/Root.tsx`: +- Add a `LINE_CAPTION_IDS` array using the same `require.context` pattern as + `SHORT_IDS` (lines 7-19), scanning `../public/line-captions` for + `/\/lines\.json$/`. +- Map it to `` + (mirroring `SHORT_IDS.map(...)` at lines 47-65). `src` is resolved inside + `LineCaptionClip` from `linesDoc.meta.videoSrc` if not passed explicitly. + +Add `remotion/components/LineCaptionOverlay.test.tsx`: smoke-renders with 2-3 +sample `CaptionLine`s at different frames, asserts the correct line's text is +shown (and that no text shows before the first line's `startMs`). + +**Status check:** `remotion/LineCaptionClip.tsx`, +`remotion/components/LineCaptionOverlay.tsx` exist; `Root.tsx` registers +`LineCaptionClip-{id}` per `public/line-captions/*/lines.json`; `npm run +test:react -- LineCaptionOverlay` passes. + +--- + +### Commit 5 — `feat: add captions:render script and npm scripts` + +Add `scripts/line-captions/render-line-captions.js`, CLI: +``` +node scripts/line-captions/render-line-captions.js --id +``` +Modeled directly on `scripts/shorts/render-short.js`: reads +`public/line-captions/{id}/lines.json` for `meta.videoSrc`, spawns +``` +npx remotion render remotion/index.ts LineCaptionClip-{id} --props='{"linesSrc":"line-captions/{id}/lines.json","brandSrc":"brand.json"}' +``` +outputting to `public/renders/{id}.mp4`. + +Add to `package.json` `scripts`: +```json +"captions:create": "node scripts/line-captions/create-line-captions.js", +"captions:merge": "node scripts/line-captions/merge-line-captions.js", +"captions:render": "node scripts/line-captions/render-line-captions.js", +``` + +**Status check:** All three `captions:*` scripts are present in +`package.json`; `npm run captions:render -- --id ` (against a clip +produced by commit 2/3) produces `public/renders/.mp4`. + +--- + +### Commit 6 — `docs: document line-caption artifacts in CLAUDE.md` + +Update `d:\Environment\deckcreate\CLAUDE.md`: +- Add a `lines.json` entry under **Data Schemas** documenting `CaptionLine` + and `LineCaptionsMeta` fields (mirror the existing `transcript.json` / + `camera-profiles.json` schema block style). +- Add rows to the **Key Source Files** table for: + `scripts/line-captions/chunkLines.js`, `create-line-captions.js`, + `merge-line-captions.js`, `render-line-captions.js`, + `remotion/types/lineCaptions.ts`, `remotion/components/LineCaptionOverlay.tsx`, + `remotion/LineCaptionClip.tsx`. + +Update `.gitignore`: add `public/line-captions/` next to the existing +`public/shorts/` line (~line 69) — same fully-generated-directory rule. + +**Status check:** `git diff CLAUDE.md .gitignore` shows the new schema entry, +file-table rows, and gitignore line. + +--- + +### Commit 7 — manual end-to-end verification (no commit, or `chore:` fixups only) + +Run the full pipeline against one real short portrait clip with two speakers: +1. `npm run captions:create -- --video --num-speakers 2` +2. Open `lines.doc.txt`, confirm 3-word lines read naturally and never mix + speakers within a line. +3. Hand-edit one line's wording. +4. `npm run captions:merge -- --id ` +5. `npx remotion studio`, select `LineCaptionClip-`, confirm captions + are burned in, timed correctly against the audio, and the edited line + shows its new wording during its original time window. +6. `tsc --noEmit` and `npm run lint` pass. + +If this surfaces bugs, fix them in a new commit rather than amending prior +steps. From 3d7c8b6c56cf73ecd85d4b1754d72de6cf515855 Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 19:22:57 +0800 Subject: [PATCH 2/9] feat: add CaptionLine types and chunkIntoLines Reconstructs whisper's BPE-level tokens into whole words before bucketing into fixed-size caption lines, so a line never splits a word or counts punctuation as its own slot. Lines never span a segment boundary, which keeps two speakers from ever sharing a line. Co-Authored-By: Claude Sonnet 5 --- .../LINE_CAPTION_SHORTS.md | 45 ++++-- remotion/types/lineCaptions.ts | 22 +++ scripts/line-captions/chunkLines.js | 67 +++++++++ scripts/line-captions/chunkLines.test.js | 135 ++++++++++++++++++ 4 files changed, 259 insertions(+), 10 deletions(-) create mode 100644 remotion/types/lineCaptions.ts create mode 100644 scripts/line-captions/chunkLines.js create mode 100644 scripts/line-captions/chunkLines.test.js diff --git a/docs/implementation-guides/LINE_CAPTION_SHORTS.md b/docs/implementation-guides/LINE_CAPTION_SHORTS.md index a3c2710..d6ab40e 100644 --- a/docs/implementation-guides/LINE_CAPTION_SHORTS.md +++ b/docs/implementation-guides/LINE_CAPTION_SHORTS.md @@ -83,24 +83,49 @@ export type LineCaptionsMeta = { title: string; duration: number; fps: number; v export type LineCaptionsDoc = { meta: LineCaptionsMeta; lines: CaptionLine[] }; ``` +Whisper tokens are BPE sub-word pieces, not whole words (`Transcriber.js:213-218` +puts raw whisper.cpp token text straight into `token.tokens[]`, no word +reconstruction) — a "word" like "jinx" can arrive as two tokens (`" j"`, +`"inx"`), and punctuation can arrive as its own token. Chunking raw tokens by +count-of-3 would split lines mid-word. `CaptionOverlay.tsx:43-103` already +solves this for its own rendering path with a `wordGroups` merge step; port a +trimmed version of the same merge (BPE-continuation attach, contraction-suffix +attach, punctuation attach — skip the reverse-iteration hook-overlap dedup, +which doesn't apply to a fresh non-overlapping transcript) into +`scripts/line-captions/chunkLines.js`, since this runs in plain Node +(`scripts/`) rather than the TS/browser `remotion/` runtime the existing +implementations live in. + Add `scripts/line-captions/chunkLines.js` exporting `chunkIntoLines(segments, wordsPerLine = 3)`: -- Filters each segment's tokens to spoken, non-cut tokens using a local - `isSpokenToken(token)` port of the regex in `remotion/lib/tokens.ts` (kept - as a plain-JS duplicate here, matching the existing precedent of - `isSpecialToken` being duplicated across `align-transcript.js` and - `edit-transcript.js` rather than shared). -- Buckets spoken tokens into runs of exactly `wordsPerLine`, never spanning a - segment boundary. A trailing partial run stands alone. -- `startMs = firstToken.t_dtw * 1000`. `endMs` = the next run's start time +- `buildWordGroups(tokens)`: drops `isSpecialToken` tokens (imported from + `../edit-transcript.js`, exported at `edit-transcript.js:2104`) and `cut` + tokens; for each remaining token, attaches it to the previous group's text + when it's a BPE continuation (`!t.text.startsWith(' ')`), a contraction + suffix (`/^'(m|s|t|re|ve|ll|d)$/i`), or punctuation-only + (`/^[^\w\s']+$/`); otherwise starts a new group with `{ text: trimmed, + t_dtw: t.t_dtw, t_end: t.t_end }`. Continuation/suffix attaches append the + trimmed token text directly (no separator); a later attach's `t_end` + overwrites the group's `t_end` so the group's end time tracks its last + token. +- `chunkIntoLines`: runs `buildWordGroups` per segment (skipping `cut` + segments), buckets the resulting whole-word groups into runs of exactly + `wordsPerLine`, never spanning a segment boundary — a trailing partial run + (1-2 words) stands alone. +- `startMs = run[0].t_dtw * 1000`. `endMs` = the next run's `t_dtw * 1000` within the same segment, or for the last run in a segment: - `(lastToken.t_end ?? segment.end) * 1000`. + `(run[run.length - 1].t_end ?? segment.end) * 1000`. +- Line text = `run.map(g => g.text).join(' ')`. - Assigns sequential `id` starting at 1 across the whole transcript. - `speaker` comes from `segment.speaker`. Add `scripts/line-captions/chunkLines.test.js` covering: -- An exact multiple of 3 spoken tokens → one line per 3 words. +- An exact multiple of 3 word-groups → one line per 3 words. - A non-multiple → trailing partial line of 1-2 words. +- A word split across two BPE tokens (no leading space on the second) is + reconstructed as one word, not counted as two. +- A punctuation-only token attaches to the preceding word instead of + starting its own group. - Two adjacent segments with different speakers → no line spans both. - `cut: true` tokens/segments are excluded entirely. - A token without `t_end` falls back to `segment.end` for the last run. diff --git a/remotion/types/lineCaptions.ts b/remotion/types/lineCaptions.ts new file mode 100644 index 0000000..4002413 --- /dev/null +++ b/remotion/types/lineCaptions.ts @@ -0,0 +1,22 @@ +export type CaptionLine = { + id: number; + speaker: string; + text: string; + /** Milliseconds from start of the source video */ + startMs: number; + endMs: number; +}; + +export type LineCaptionsMeta = { + title: string; + /** Total duration in seconds */ + duration: number; + fps: number; + /** Path to the source video relative to /public */ + videoSrc?: string; +}; + +export type LineCaptionsDoc = { + meta: LineCaptionsMeta; + lines: CaptionLine[]; +}; diff --git a/scripts/line-captions/chunkLines.js b/scripts/line-captions/chunkLines.js new file mode 100644 index 0000000..57275dd --- /dev/null +++ b/scripts/line-captions/chunkLines.js @@ -0,0 +1,67 @@ +import { isSpecialToken } from '../edit-transcript.js'; + +// Whisper contraction suffixes arrive as their own space-prefixed token (" 's", " 'm"). +function isContractionSuffix(text) { + return /^'(m|s|t|re|ve|ll|d)$/i.test(text.trim()); +} + +function isPunctuationOnly(text) { + return /^[^\w\s']+$/.test(text.trim()); +} + +// Merges whisper's BPE-level tokens into whole-word groups so a "word" split +// across multiple tokens (e.g. " j" + "inx") isn't counted as two words, and +// punctuation tokens don't get their own caption slot. +function buildWordGroups(tokens) { + const groups = []; + for (const t of tokens) { + if (t.cut || isSpecialToken(t)) continue; + const trimmed = t.text.trim(); + if (!trimmed) continue; + + const isContinuation = !t.text.startsWith(' ') || isContractionSuffix(trimmed) || isPunctuationOnly(trimmed); + if (isContinuation && groups.length > 0) { + const prev = groups[groups.length - 1]; + prev.text += trimmed; + prev.t_end = t.t_end ?? prev.t_end; + } else { + groups.push({ text: trimmed, t_dtw: t.t_dtw, t_end: t.t_end }); + } + } + return groups; +} + +/** + * Chunks a transcript's segments into fixed-size caption lines. A line never + * spans a segment boundary, so lines never mix two speakers. + */ +export function chunkIntoLines(segments, wordsPerLine = 3) { + const lines = []; + let nextId = 1; + + for (const segment of segments) { + if (segment.cut) continue; + const groups = buildWordGroups(segment.tokens); + + for (let i = 0; i < groups.length; i += wordsPerLine) { + const run = groups.slice(i, i + wordsPerLine); + const nextGroup = groups[i + wordsPerLine]; + const lastGroup = run[run.length - 1]; + + const startMs = run[0].t_dtw * 1000; + const endMs = nextGroup + ? nextGroup.t_dtw * 1000 + : (lastGroup.t_end ?? segment.end) * 1000; + + lines.push({ + id: nextId++, + speaker: segment.speaker, + text: run.map(g => g.text).join(' '), + startMs, + endMs, + }); + } + } + + return lines; +} diff --git a/scripts/line-captions/chunkLines.test.js b/scripts/line-captions/chunkLines.test.js new file mode 100644 index 0000000..2c55860 --- /dev/null +++ b/scripts/line-captions/chunkLines.test.js @@ -0,0 +1,135 @@ +import { chunkIntoLines } from './chunkLines.js'; + +function tok(text, t_dtw, t_end, cut = false) { + return { text, t_dtw, t_end, cut }; +} + +function seg(overrides) { + return { id: 1, start: 0, end: 1, speaker: 'A', cut: false, tokens: [], ...overrides }; +} + +describe('chunkIntoLines', () => { + test('chunks an exact multiple of 3 words into one line per 3 words', () => { + const segment = seg({ + end: 1, + tokens: [ + tok(' One', 0.0, 0.2), + tok(' two', 0.2, 0.4), + tok(' three', 0.4, 0.6), + tok(' four', 0.6, 0.8), + tok(' five', 0.8, 1.0), + tok(' six', 1.0, 1.2), + ], + }); + + const lines = chunkIntoLines([segment]); + + expect(lines).toHaveLength(2); + expect(lines[0]).toMatchObject({ id: 1, speaker: 'A', text: 'One two three', startMs: 0, endMs: 600 }); + expect(lines[1]).toMatchObject({ id: 2, speaker: 'A', text: 'four five six', startMs: 600 }); + }); + + test('a non-multiple of 3 leaves a trailing partial line', () => { + const segment = seg({ + end: 1, + tokens: [ + tok(' One', 0.0, 0.2), + tok(' two', 0.2, 0.4), + tok(' three', 0.4, 0.6), + tok(' four', 0.6, 0.8), + ], + }); + + const lines = chunkIntoLines([segment]); + + expect(lines).toHaveLength(2); + expect(lines[0].text).toBe('One two three'); + expect(lines[1].text).toBe('four'); + expect(lines[1].endMs).toBe(0.8 * 1000); + }); + + test('reconstructs a word split across two BPE tokens as a single word', () => { + const segment = seg({ + end: 1, + tokens: [ + tok(' This', 0.0, 0.2), + tok(' is', 0.2, 0.4), + tok(' j', 0.4, 0.5), + tok('inx', 0.5, 0.6), + ], + }); + + const lines = chunkIntoLines([segment]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe('This is jinx'); + }); + + test('attaches punctuation-only tokens to the preceding word instead of starting a new group', () => { + const segment = seg({ + end: 1, + tokens: [ + tok(' Hello', 0.0, 0.2), + tok('.', 0.2, 0.2), + tok(' World', 0.3, 0.5), + ], + }); + + const lines = chunkIntoLines([segment]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe('Hello. World'); + }); + + test('never lets a line span two speakers', () => { + const segmentA = seg({ + id: 1, + speaker: 'Natasha', + end: 0.5, + tokens: [tok(' Hey', 0.0, 0.2), tok(' there', 0.2, 0.5)], + }); + const segmentB = seg({ + id: 2, + speaker: 'Saloni', + start: 0.5, + end: 1.0, + tokens: [tok(' Hi', 0.5, 0.7), tok(' back', 0.7, 1.0)], + }); + + const lines = chunkIntoLines([segmentA, segmentB]); + + expect(lines).toHaveLength(2); + expect(lines[0]).toMatchObject({ speaker: 'Natasha', text: 'Hey there' }); + expect(lines[1]).toMatchObject({ speaker: 'Saloni', text: 'Hi back' }); + }); + + test('excludes cut tokens and cut segments entirely', () => { + const segmentA = seg({ + id: 1, + end: 0.6, + tokens: [ + tok(' kept', 0.0, 0.2), + tok(' cut-word', 0.2, 0.4, true), + tok(' also-kept', 0.4, 0.6), + ], + }); + const segmentB = seg({ id: 2, cut: true, start: 0.6, end: 1.0, tokens: [tok(' ignored', 0.6, 1.0)] }); + + const lines = chunkIntoLines([segmentA, segmentB]); + + expect(lines).toHaveLength(1); + expect(lines[0].text).toBe('kept also-kept'); + }); + + test('falls back to segment.end when the last token has no t_end', () => { + const segment = seg({ + end: 0.9, + tokens: [tok(' One', 0.0, undefined), tok(' two', 0.3, undefined)], + }); + + const lines = chunkIntoLines([segment]); + + expect(lines).toHaveLength(1); + expect(lines[0].endMs).toBe(0.9 * 1000); + }); +}); From 1192d130fa89768087ec885a8fba5979cba1131d Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 19:26:14 +0800 Subject: [PATCH 3/9] feat: add captions:create script Orchestrates the raw-portrait-video-to-caption-lines pipeline: copy source, extract audio, transcribe, force-align for exact word boundaries, diarize and assign speakers when --num-speakers > 1, then chunk into 3-word lines and write lines.json + a human-editable lines.doc.txt. Reuses the existing Transcriber class and align/diarize CLI scripts unmodified. Co-Authored-By: Claude Sonnet 5 --- scripts/line-captions/create-line-captions.js | 174 ++++++++++++++++++ .../create-line-captions.test.js | 31 ++++ 2 files changed, 205 insertions(+) create mode 100644 scripts/line-captions/create-line-captions.js create mode 100644 scripts/line-captions/create-line-captions.test.js diff --git a/scripts/line-captions/create-line-captions.js b/scripts/line-captions/create-line-captions.js new file mode 100644 index 0000000..7bbc894 --- /dev/null +++ b/scripts/line-captions/create-line-captions.js @@ -0,0 +1,174 @@ +#!/usr/bin/env node +/** + * DeckCreate — Create line-caption artifacts from a raw short-form portrait video. + * + * Usage: + * node scripts/line-captions/create-line-captions.js --video [--num-speakers N] [--id ] + */ + +import { spawn } from 'child_process'; +import fs from 'fs-extra'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import Transcriber from '../transcribe/Transcriber.js'; +import { chunkIntoLines } from './chunkLines.js'; +import { stampMetadata } from '../config/metadata.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const cwd = path.join(__dirname, '../..'); + +function parseArgs() { + const args = process.argv.slice(2); + const result = {}; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--video' && args[i + 1]) result.videoPath = args[++i]; + else if (args[i] === '--num-speakers' && args[i + 1]) result.numSpeakers = parseInt(args[++i], 10); + else if (args[i] === '--id' && args[i + 1]) result.id = args[++i]; + } + return result; +} + +async function resolveClipId(explicitId) { + const lineCaptionsDir = path.join(cwd, 'public', 'line-captions'); + await fs.ensureDir(lineCaptionsDir); + if (explicitId) return explicitId; + + const existingIds = (await fs.readdir(lineCaptionsDir)) + .filter(d => /^clip-\d+$/.test(d)) + .map(d => parseInt(d.replace('clip-', ''), 10)); + const nextId = existingIds.length > 0 ? Math.max(...existingIds) + 1 : 1; + return `clip-${nextId}`; +} + +function spawnNode(scriptPath, args) { + return new Promise((resolve, reject) => { + const proc = spawn('node', [scriptPath, ...args], { + stdio: 'inherit', + cwd, + shell: process.platform === 'win32', + }); + proc.on('close', code => (code === 0 ? resolve() : reject(new Error(`${path.basename(scriptPath)} exited with code ${code}`)))); + proc.on('error', e => reject(new Error(`Failed to spawn ${scriptPath}: ${e.message}`))); + }); +} + +function extractAudio(videoPath, outPath) { + return new Promise((resolve, reject) => { + const proc = spawn('ffmpeg', [ + '-i', videoPath, '-vn', '-ar', '16000', '-ac', '1', '-y', outPath, + ], { stdio: ['ignore', 'ignore', 'inherit'], cwd }); + proc.on('close', code => (code === 0 ? resolve() : reject(new Error(`ffmpeg exited with code ${code}`)))); + proc.on('error', e => reject(new Error(`Failed to spawn ffmpeg: ${e.message}`))); + }); +} + +function buildLineDoc(linesDoc) { + const lines = []; + let lastSpeaker = null; + + for (const line of linesDoc.lines) { + if (line.speaker !== lastSpeaker) { + if (lastSpeaker !== null) lines.push(''); + lines.push(`=== ${line.speaker} ===`); + lines.push(''); + lastSpeaker = line.speaker; + } + lines.push(`[${line.id}] ${line.text}`); + } + + return lines.join('\n') + '\n'; +} + +async function main() { + const { videoPath, numSpeakers, id: explicitId } = parseArgs(); + + if (!videoPath) { + console.error('Usage: create-line-captions.js --video [--num-speakers N] [--id ]'); + process.exit(1); + } + if (!await fs.pathExists(videoPath)) { + console.error(`❌ Video file not found: ${videoPath}`); + process.exit(1); + } + + const id = await resolveClipId(explicitId); + const clipDir = path.join(cwd, 'public', 'line-captions', id); + await fs.ensureDir(clipDir); + + console.log('\nLine captions — create'); + console.log(` Video: ${videoPath}`); + console.log(` Clip id: ${id}`); + if (numSpeakers) console.log(` Speakers: ${numSpeakers}`); + console.log(''); + + const sourcePath = path.join(clipDir, `source${path.extname(videoPath)}`); + console.log('Step 1/5: Copying source video...'); + await fs.copy(videoPath, sourcePath); + + const audioPath = path.join(clipDir, 'audio.wav'); + console.log('Step 2/5: Extracting audio...'); + await extractAudio(sourcePath, audioPath); + + const transcribeOutputDir = path.join(clipDir, 'transcribe', 'raw'); + const rawJsonPath = path.join(transcribeOutputDir, 'transcript.raw.json'); + console.log('Step 3/5: Transcribing...'); + const transcriber = new Transcriber({ audioPath, outputDir: transcribeOutputDir }); + try { + await transcriber.init(); + await transcriber.transcribe(); + } finally { + await transcriber.close(); + } + + console.log('Step 4/5: Aligning word timestamps...'); + await spawnNode(path.join(cwd, 'scripts', 'align', 'align-transcript.js'), [ + '--audio', audioPath, '--raw', rawJsonPath, + ]); + + if (numSpeakers && numSpeakers > 1) { + const diarizationJsonPath = path.join(transcribeOutputDir, 'diarization.json'); + console.log(`Diarizing ${numSpeakers} speakers...`); + await spawnNode(path.join(cwd, 'scripts', 'diarize', 'diarize-audio.js'), [ + '--audio', audioPath, '--output', diarizationJsonPath, '--num-speakers', String(numSpeakers), + ]); + await spawnNode(path.join(cwd, 'scripts', 'diarize', 'assign-speakers.js'), [ + '--diarization', diarizationJsonPath, '--raw', rawJsonPath, + ]); + } + + console.log('Step 5/5: Chunking into 3-word lines...'); + const rawTranscript = await fs.readJson(rawJsonPath); + const lines = chunkIntoLines(rawTranscript.segments); + + const linesDoc = { + meta: { + title: rawTranscript.meta.title || id, + duration: rawTranscript.meta.duration, + fps: rawTranscript.meta.fps, + videoSrc: `line-captions/${id}/${path.basename(sourcePath)}`, + }, + lines, + }; + + const linesJsonPath = path.join(clipDir, 'lines.json'); + const linesDocPath = path.join(clipDir, 'lines.doc.txt'); + await fs.writeJson(linesJsonPath, stampMetadata(linesDoc, cwd), { spaces: 2 }); + await fs.writeFile(linesDocPath, buildLineDoc(linesDoc), 'utf-8'); + + console.log(`\n✓ Wrote ${lines.length} caption lines`); + console.log(` ${path.relative(cwd, linesJsonPath)}`); + console.log(` ${path.relative(cwd, linesDocPath)}`); + console.log('\nNext steps:'); + console.log(` 1. Edit ${path.relative(cwd, linesDocPath)} — reword lines as needed (keep [id] markers as-is)`); + console.log(` 2. Run: npm run captions:merge -- --id ${id}`); + console.log(` 3. Run: npm run captions:render -- --id ${id}`); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch(err => { + console.error('❌ Error:', err.message); + process.exit(1); + }); +} + +export { buildLineDoc }; diff --git a/scripts/line-captions/create-line-captions.test.js b/scripts/line-captions/create-line-captions.test.js new file mode 100644 index 0000000..71b5a21 --- /dev/null +++ b/scripts/line-captions/create-line-captions.test.js @@ -0,0 +1,31 @@ +import { buildLineDoc } from './create-line-captions.js'; + +describe('buildLineDoc', () => { + test('groups consecutive lines under a speaker header', () => { + const doc = buildLineDoc({ + meta: { title: 't', duration: 1, fps: 60 }, + lines: [ + { id: 1, speaker: 'Natasha', text: 'this is the', startMs: 0, endMs: 600 }, + { id: 2, speaker: 'Natasha', text: 'first line here', startMs: 600, endMs: 1200 }, + ], + }); + + expect(doc).toBe( + '=== Natasha ===\n\n[1] this is the\n[2] first line here\n', + ); + }); + + test('starts a new speaker block on a speaker change', () => { + const doc = buildLineDoc({ + meta: { title: 't', duration: 1, fps: 60 }, + lines: [ + { id: 1, speaker: 'Natasha', text: 'hey there', startMs: 0, endMs: 500 }, + { id: 2, speaker: 'Saloni', text: 'hi back', startMs: 500, endMs: 1000 }, + ], + }); + + expect(doc).toBe( + '=== Natasha ===\n\n[1] hey there\n\n=== Saloni ===\n\n[2] hi back\n', + ); + }); +}); From 67a40c97722b9d33cc57fd788c5974a7472bf77f Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 19:27:35 +0800 Subject: [PATCH 4/9] feat: add captions:merge script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-parses lines.doc.txt after a human reword pass and overwrites only the text field of each matching lines.json entry by [id] — startMs/endMs/speaker are left untouched, so the caption keeps its original best-effort time window regardless of how much the wording changed. Co-Authored-By: Claude Sonnet 5 --- scripts/line-captions/merge-line-captions.js | 109 ++++++++++++++++++ .../line-captions/merge-line-captions.test.js | 67 +++++++++++ 2 files changed, 176 insertions(+) create mode 100644 scripts/line-captions/merge-line-captions.js create mode 100644 scripts/line-captions/merge-line-captions.test.js diff --git a/scripts/line-captions/merge-line-captions.js b/scripts/line-captions/merge-line-captions.js new file mode 100644 index 0000000..d24777a --- /dev/null +++ b/scripts/line-captions/merge-line-captions.js @@ -0,0 +1,109 @@ +#!/usr/bin/env node +/** + * DeckCreate — Merge a hand-edited lines.doc.txt back into lines.json. + * + * Usage: + * node scripts/line-captions/merge-line-captions.js --id + */ + +import fs from 'fs-extra'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const cwd = path.join(__dirname, '../..'); + +function parseArgs() { + const args = process.argv.slice(2); + const result = {}; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--id' && args[i + 1]) result.id = args[++i]; + } + return result; +} + +/** Parses `=== NAME ===` headers and `[id] text` lines out of a lines.doc.txt. */ +function parseLineDoc(docContent) { + const parsed = []; + let currentSpeaker = null; + + for (const rawLine of docContent.split('\n')) { + const headerMatch = rawLine.match(/^===\s*(.+?)\s*===$/); + if (headerMatch) { + currentSpeaker = headerMatch[1]; + continue; + } + const lineMatch = rawLine.match(/^\[(\d+)\]\s*(.*)$/); + if (lineMatch) { + parsed.push({ id: parseInt(lineMatch[1], 10), speaker: currentSpeaker, text: lineMatch[2].trim() }); + } + } + + return parsed; +} + +/** + * Overwrites text on the matching lines.json entry per id. startMs/endMs/speaker + * are left untouched — timing is best-effort and doesn't track rewording. + */ +function mergeLineText(linesDoc, parsedLines) { + const byId = new Map(linesDoc.lines.map(l => [l.id, l])); + const seenIds = new Set(); + + for (const parsed of parsedLines) { + const line = byId.get(parsed.id); + if (!line) { + console.warn(` ⚠ Doc line [${parsed.id}] has no matching entry in lines.json — skipped`); + continue; + } + line.text = parsed.text; + seenIds.add(parsed.id); + } + + for (const line of linesDoc.lines) { + if (!seenIds.has(line.id)) { + console.warn(` ⚠ lines.json entry [${line.id}] was not found in the doc — its text is unchanged`); + } + } + + return linesDoc; +} + +async function main() { + const { id } = parseArgs(); + if (!id) { + console.error('Usage: merge-line-captions.js --id '); + process.exit(1); + } + + const clipDir = path.join(cwd, 'public', 'line-captions', id); + const docPath = path.join(clipDir, 'lines.doc.txt'); + const linesJsonPath = path.join(clipDir, 'lines.json'); + + const [docContent, linesDoc] = await Promise.all([ + fs.readFile(docPath, 'utf-8'), + fs.readJson(linesJsonPath), + ]); + + console.log('\nLine captions — merge'); + console.log(` Doc: ${path.relative(cwd, docPath)}`); + console.log(` Lines: ${path.relative(cwd, linesJsonPath)}`); + console.log(''); + + const parsedLines = parseLineDoc(docContent); + const merged = mergeLineText(linesDoc, parsedLines); + + await fs.writeJson(linesJsonPath, merged, { spaces: 2 }); + + console.log(`\n✓ Merged ${parsedLines.length} lines`); + console.log(`\nNext step: npm run captions:render -- --id ${id}`); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch(err => { + console.error('❌ Error:', err.message); + process.exit(1); + }); +} + +export { parseLineDoc, mergeLineText }; diff --git a/scripts/line-captions/merge-line-captions.test.js b/scripts/line-captions/merge-line-captions.test.js new file mode 100644 index 0000000..36d469e --- /dev/null +++ b/scripts/line-captions/merge-line-captions.test.js @@ -0,0 +1,67 @@ +import { parseLineDoc, mergeLineText } from './merge-line-captions.js'; + +describe('parseLineDoc', () => { + test('parses [id] text lines and tracks the current speaker header', () => { + const doc = [ + '=== Natasha ===', + '', + '[1] this is the', + '[2] first line here', + '', + '=== Saloni ===', + '', + '[3] and this one', + '', + ].join('\n'); + + expect(parseLineDoc(doc)).toEqual([ + { id: 1, speaker: 'Natasha', text: 'this is the' }, + { id: 2, speaker: 'Natasha', text: 'first line here' }, + { id: 3, speaker: 'Saloni', text: 'and this one' }, + ]); + }); + + test('captures reworded text verbatim', () => { + const doc = '=== Natasha ===\n\n[1] a totally different phrase\n'; + expect(parseLineDoc(doc)).toEqual([{ id: 1, speaker: 'Natasha', text: 'a totally different phrase' }]); + }); +}); + +describe('mergeLineText', () => { + function linesDoc() { + return { + meta: { title: 't', duration: 1, fps: 60 }, + lines: [ + { id: 1, speaker: 'Natasha', text: 'original text', startMs: 0, endMs: 600 }, + { id: 2, speaker: 'Natasha', text: 'second line', startMs: 600, endMs: 1200 }, + ], + }; + } + + test('overwrites text for matching ids and preserves timing', () => { + const result = mergeLineText(linesDoc(), [ + { id: 1, speaker: 'Natasha', text: 'rewritten text' }, + { id: 2, speaker: 'Natasha', text: 'second line' }, + ]); + + expect(result.lines[0]).toMatchObject({ text: 'rewritten text', startMs: 0, endMs: 600 }); + expect(result.lines[1]).toMatchObject({ text: 'second line', startMs: 600, endMs: 1200 }); + }); + + test('warns and skips a doc id with no matching lines.json entry', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const result = mergeLineText(linesDoc(), [{ id: 99, speaker: 'Natasha', text: 'ghost line' }]); + + expect(result.lines).toHaveLength(2); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('[99]')); + warnSpy.mockRestore(); + }); + + test('warns about a lines.json id missing from the doc', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + mergeLineText(linesDoc(), [{ id: 1, speaker: 'Natasha', text: 'only this one edited' }]); + + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('[2]')); + warnSpy.mockRestore(); + }); +}); From 42f71cb55746272b7aabf338f43b9fd845da9bb1 Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 19:31:43 +0800 Subject: [PATCH 5/9] feat: add LineCaptionClip Remotion composition A plain video pass-through (no jump cuts, no camera profiles) plus LineCaptionOverlay, which shows fixed-window 3-word caption lines instead of the timing-driven grouping CaptionOverlay uses. Registers one LineCaptionClip-{id} composition per public/line-captions/*/lines.json, mirroring the existing ShortFormClip-{id} pattern in Root.tsx. Co-Authored-By: Claude Sonnet 5 --- remotion/LineCaptionClip.tsx | 87 +++++++++++++++++++ remotion/Root.tsx | 31 +++++++ .../components/LineCaptionOverlay.test.tsx | 74 ++++++++++++++++ remotion/components/LineCaptionOverlay.tsx | 58 +++++++++++++ 4 files changed, 250 insertions(+) create mode 100644 remotion/LineCaptionClip.tsx create mode 100644 remotion/components/LineCaptionOverlay.test.tsx create mode 100644 remotion/components/LineCaptionOverlay.tsx diff --git a/remotion/LineCaptionClip.tsx b/remotion/LineCaptionClip.tsx new file mode 100644 index 0000000..28d21da --- /dev/null +++ b/remotion/LineCaptionClip.tsx @@ -0,0 +1,87 @@ +import { + AbsoluteFill, + OffthreadVideo, + staticFile, + CalculateMetadataFunction, + delayRender, + continueRender, +} from 'remotion'; +import React, { useState, useEffect } from 'react'; +import { LineCaptionOverlay } from './components/LineCaptionOverlay'; +import { loadNunito } from './loadFonts'; +import type { LineCaptionsDoc } from './types/lineCaptions'; +import type { Brand } from './types/brand'; + +type LineCaptionClipProps = { + /** Path to the source video relative to /public. Defaults to lines.json's meta.videoSrc. */ + src?: string; + linesSrc: string; + brandSrc?: string; + /** Brand ID to load from brands/{brandId}/brand.json. Takes precedence over brandSrc if provided. */ + brandId?: string; +}; + +const normalizeStaticPath = (src: string) => src.replace(/^\/+/, ''); + +async function fetchJson(src: string): Promise { + const res = await fetch(staticFile(normalizeStaticPath(src))); + if (!res.ok) throw new Error(`Failed to load ${src}: ${res.status}`); + return res.json(); +} + +export const calculateLineCaptionMetadata: CalculateMetadataFunction = async ({ props }) => { + const fps = 60; + const fallback = { durationInFrames: 300, fps, width: 1080, height: 1920 }; + + try { + const linesDoc = await fetchJson(props.linesSrc); + const durationInFrames = Math.max(1, Math.ceil(linesDoc.meta.duration * fps)); + const overrideProps: LineCaptionClipProps = props.src ? props : { ...props, src: linesDoc.meta.videoSrc }; + return { durationInFrames, fps, width: 1080, height: 1920, props: overrideProps }; + } catch { + return fallback; + } +}; + +export const LineCaptionClip: React.FC = ({ + src, + linesSrc, + brandSrc = 'brand.json', + brandId, +}) => { + const resolvedBrandSrc = brandId ? `brands/${brandId}/brand.json` : brandSrc; + + const [linesDoc, setLinesDoc] = useState(null); + const [brand, setBrand] = useState(null); + + const [linesHandle] = useState(() => delayRender('Loading line captions')); + const [brandHandle] = useState(() => delayRender('Loading brand')); + const [fontHandle] = useState(() => delayRender('Loading Nunito font')); + + useEffect(() => { + fetchJson(linesSrc) + .then(data => { setLinesDoc(data); continueRender(linesHandle); }) + .catch(err => { console.error(err); continueRender(linesHandle); }); + }, [linesSrc, linesHandle]); + + useEffect(() => { + fetchJson(resolvedBrandSrc) + .then(data => { setBrand(data); continueRender(brandHandle); }) + .catch(err => { console.warn('Brand not loaded:', err.message); continueRender(brandHandle); }); + }, [resolvedBrandSrc, brandHandle]); + + useEffect(() => { + loadNunito().finally(() => continueRender(fontHandle)); + }, [fontHandle]); + + if (!linesDoc || !brand) return null; + + const resolvedSrc = staticFile(normalizeStaticPath(src ?? linesDoc.meta.videoSrc ?? '')); + + return ( + + + + + ); +}; diff --git a/remotion/Root.tsx b/remotion/Root.tsx index d54d2a5..d947a9b 100644 --- a/remotion/Root.tsx +++ b/remotion/Root.tsx @@ -2,6 +2,7 @@ import React from 'react'; import {Composition} from 'remotion'; import {MyComposition, calculateMetadata} from './Composition'; import { ShortFormClip, calculateShortMetadata } from './ShortFormClip'; +import { LineCaptionClip, calculateLineCaptionMetadata } from './LineCaptionClip'; import { OverlayGalleryComposition, GALLERY_TOTAL_FRAMES } from './components/OverlayGallery'; // require.context is a webpack API — scans public/shorts/ at bundle time @@ -18,6 +19,20 @@ const SHORT_IDS: string[] = (() => { } })(); +// require.context is a webpack API — scans public/line-captions/ at bundle time +const LINE_CAPTION_IDS: string[] = (() => { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (require as any) + .context('../public/line-captions', true, /\/lines\.json$/) + .keys() + .map((k: string) => k.split('/')[1]); + } catch { + // Directory doesn't exist - no line-caption clips configured yet + return []; + } +})(); + export const RemotionRoot: React.FC = () => { return ( <> @@ -63,6 +78,22 @@ export const RemotionRoot: React.FC = () => { calculateMetadata={calculateShortMetadata} /> ))} + {LINE_CAPTION_IDS.map((clipId) => ( + + ))} ); }; \ No newline at end of file diff --git a/remotion/components/LineCaptionOverlay.test.tsx b/remotion/components/LineCaptionOverlay.test.tsx new file mode 100644 index 0000000..ce605d5 --- /dev/null +++ b/remotion/components/LineCaptionOverlay.test.tsx @@ -0,0 +1,74 @@ +import React from 'react'; +import { render } from '@testing-library/react'; + +const mockUseCurrentFrame = jest.fn(() => 0); + +jest.mock('remotion', () => ({ + useCurrentFrame: () => mockUseCurrentFrame(), + useVideoConfig: () => ({ fps: 60 }), + AbsoluteFill: ({ children, style }: { children: React.ReactNode; style?: React.CSSProperties }) => ( +
{children}
+ ), +})); + +import { LineCaptionOverlay } from './LineCaptionOverlay'; +import type { CaptionLine } from '../types/lineCaptions'; +import type { Brand } from '../types/brand'; + +const brand = { + colors: { + text: { primary: '#fff' }, + palette: ['#f00', '#0f0', '#00f'], + }, + typography: { + fontFamily: 'Nunito', + weights: { black: 900 }, + }, +} as unknown as Brand; + +const lines: CaptionLine[] = [ + { id: 1, speaker: 'Natasha', text: 'this is the', startMs: 0, endMs: 500 }, + { id: 2, speaker: 'Natasha', text: 'first caption line', startMs: 500, endMs: 1000 }, +]; + +describe('LineCaptionOverlay', () => { + beforeEach(() => { + mockUseCurrentFrame.mockReturnValue(0); + }); + + it('renders nothing when there are no lines', () => { + mockUseCurrentFrame.mockReturnValue(0); + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it('shows the first line during its time window', () => { + mockUseCurrentFrame.mockReturnValue(0); // 0ms + const { getByText } = render(); + expect(getByText('this is the')).toBeTruthy(); + }); + + it('switches to the second line once its window starts', () => { + mockUseCurrentFrame.mockReturnValue(36); // 36/60 * 1000 = 600ms + const { getByText, queryByText } = render(); + expect(getByText('first caption line')).toBeTruthy(); + expect(queryByText('this is the')).toBeNull(); + }); + + it('renders nothing after the last line ends', () => { + mockUseCurrentFrame.mockReturnValue(120); // 2000ms + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it('tints text by speaker using the brand palette when multiple speakers are present', () => { + const multiSpeakerLines: CaptionLine[] = [ + { id: 1, speaker: 'Natasha', text: 'hey there', startMs: 0, endMs: 500 }, + { id: 2, speaker: 'Saloni', text: 'hi back', startMs: 500, endMs: 1000 }, + ]; + mockUseCurrentFrame.mockReturnValue(36); // 600ms → Saloni's line + const { getByText } = render(); + const el = getByText('hi back'); + expect(el.style.color).not.toBe(''); + }); +}); diff --git a/remotion/components/LineCaptionOverlay.tsx b/remotion/components/LineCaptionOverlay.tsx new file mode 100644 index 0000000..0fa3a49 --- /dev/null +++ b/remotion/components/LineCaptionOverlay.tsx @@ -0,0 +1,58 @@ +import React, { useMemo } from 'react'; +import { AbsoluteFill, useCurrentFrame, useVideoConfig } from 'remotion'; +import type { CaptionLine } from '../types/lineCaptions'; +import type { Brand } from '../types/brand'; + +// ── Layout constants for 1080 × 1920 (matches CaptionOverlay.tsx) ───────────── + +const CAPTION_TOP = 1300; +const CAPTION_FONT_SIZE = 60; + +type Props = { + lines: CaptionLine[]; + brand: Brand; +}; + +function colorForSpeaker(speaker: string, speakers: string[], brand: Brand): string { + if (speakers.length <= 1) return brand.colors.text.primary; + const palette = brand.colors.palette; + if (!palette || palette.length === 0) return brand.colors.text.primary; + const idx = speakers.indexOf(speaker); + return palette[idx % palette.length]; +} + +export const LineCaptionOverlay: React.FC = ({ lines, brand }) => { + const frame = useCurrentFrame(); + const { fps } = useVideoConfig(); + const ms = (frame / fps) * 1000; + + const speakers = useMemo(() => Array.from(new Set(lines.map(l => l.speaker))), [lines]); + const active = useMemo(() => lines.find(l => ms >= l.startMs && ms < l.endMs) ?? null, [lines, ms]); + + if (!active) return null; + + const { typography } = brand; + + return ( + +
+ {active.text} +
+ + ); +}; From b9ed12015b7d8ef7a87339433e74c83be2f9d526 Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 22:46:05 +0800 Subject: [PATCH 6/9] feat: add captions:render script and npm scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modeled on scripts/shorts/render-short.js — spawns remotion render against the LineCaptionClip-{id} composition and writes to public/renders/. Co-Authored-By: Claude Sonnet 5 --- package.json | 3 + scripts/line-captions/render-line-captions.js | 74 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 scripts/line-captions/render-line-captions.js diff --git a/package.json b/package.json index d3ca31a..7d33361 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,9 @@ "carousel:pdf:docker": "docker compose run --rm carousel npm run carousel:pdf", "carousel:wizard": "node scripts/carousel/carousel-wizard.js", "carousel:wizard:docker": "docker compose run --rm --service-ports carousel npm run carousel:wizard", + "captions:create": "node scripts/line-captions/create-line-captions.js", + "captions:merge": "node scripts/line-captions/merge-line-captions.js", + "captions:render": "node scripts/line-captions/render-line-captions.js", "cut:preview": "node scripts/cut-preview.js", "dev": "next dev", "diarize": "node scripts/diarize/diarize-audio.js", diff --git a/scripts/line-captions/render-line-captions.js b/scripts/line-captions/render-line-captions.js new file mode 100644 index 0000000..9f25807 --- /dev/null +++ b/scripts/line-captions/render-line-captions.js @@ -0,0 +1,74 @@ +#!/usr/bin/env node +/** + * Render a line-caption clip. + * + * Usage: + * node scripts/line-captions/render-line-captions.js --id + */ + +import fs from 'fs-extra'; +import path from 'path'; +import { spawn } from 'child_process'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const cwd = path.join(__dirname, '../..'); + +function parseArgs(argv) { + const args = {}; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--id') args.id = argv[++i]; + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + if (!args.id) { + console.error('Usage: render-line-captions.js --id '); + console.error('Example: render-line-captions.js --id clip-1'); + process.exit(1); + } + + const linesJsonPath = path.join(cwd, 'public', 'line-captions', args.id, 'lines.json'); + if (!await fs.pathExists(linesJsonPath)) { + console.error(`✗ lines.json not found: ${linesJsonPath}`); + console.error('Run captions:create (and captions:merge) first.'); + process.exit(1); + } + + const outName = `${args.id}.mp4`; + console.log(`Rendering line-caption clip: ${args.id}`); + console.log(`Output file: ${outName}`); + + const renderArgs = [ + 'remotion', 'render', + 'remotion/index.ts', + `LineCaptionClip-${args.id}`, + '--outName', outName, + '--props', JSON.stringify({ + linesSrc: `line-captions/${args.id}/lines.json`, + brandSrc: 'brand.json', + }), + ]; + + console.log(`\nRunning: npx ${renderArgs.join(' ')}`); + + await new Promise((resolve, reject) => { + const proc = spawn('npx', renderArgs, { + stdio: 'inherit', + cwd, + shell: process.platform === 'win32', + }); + proc.on('close', code => code === 0 ? resolve() : reject(new Error(`Render exited ${code}`))); + proc.on('error', e => reject(e)); + }); + + console.log(`\n✓ Rendered: public/renders/${outName}`); +} + +main().catch(err => { + console.error('✗', err.message); + process.exit(1); +}); From 5ee504bc1bbc8789ab29c26cba443e89ecb67f92 Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 22:56:33 +0800 Subject: [PATCH 7/9] docs: document lines.json schema and line-caption source files Co-Authored-By: Claude Sonnet 5 --- .gitignore | 1 + CLAUDE.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/.gitignore b/.gitignore index 44b2e44..2e96278 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ public/thumbnail/ # candidate frames, cutouts, manifest public/renders/ # final rendered .mp4 files public/output/ # carousel exports public/shorts/ +public/line-captions/ public/sync/output # ── Editable pipeline outputs (text/JSON) ───────────────────────────────────── diff --git a/CLAUDE.md b/CLAUDE.md index 2fbbf4f..c2b0c4a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,6 +105,27 @@ segments[] `CropViewport`: `cx/cy` = normalised centre (0–1), `w/h` = crop dimensions (0–1). +### lines.json + +Standalone artifact for the line-caption shorts pipeline (`public/line-captions/{id}/lines.json`) — not part of `transcript.json`; no segments/tokens/cuts. + +``` +meta + title: string + duration: number total video duration in seconds + fps: 60 + videoSrc?: string path relative to /public + +lines[] + id: number sequential across the whole clip + speaker: string + text: string exactly wordsPerLine (default 3) whole words + startMs: number milliseconds from start of source video + endMs: number +``` + +A line never spans a segment/speaker boundary. Produced by `chunkIntoLines()` (`scripts/line-captions/chunkLines.js`); human-edited via `lines.doc.txt` (`captions:create` / `captions:merge`) which only rewrites `text` — `startMs`/`endMs`/`speaker` are untouched by edits. + --- ## Rendering Model @@ -215,6 +236,13 @@ ty = (0.5 - vp.cy) × 100% | `app/components/AutoCarouselForm.tsx` | Carousel generator (810 lines) | Decompose (Phase 8) | | `app/context/AuthContext.tsx` | Auth context with hardcoded credentials | Fix (Phase 7) | | `vscode-transcript-language/src/extension.js` | VSCode transcript extension | Add Wrap-in-cut command (Phase 0.5) | +| `remotion/types/lineCaptions.ts` | `CaptionLine`, `LineCaptionsMeta`, `LineCaptionsDoc` | — | +| `remotion/components/LineCaptionOverlay.tsx` | Renders the active fixed-window `CaptionLine`; speaker-tinted via `brand.colors.palette` | — | +| `remotion/LineCaptionClip.tsx` | Raw video pass-through + `LineCaptionOverlay`; registered per clip in `Root.tsx` as `LineCaptionClip-{id}` | — | +| `scripts/line-captions/chunkLines.js` | `chunkIntoLines()` — merges BPE tokens into whole words, buckets into fixed-size (default 3) caption lines, never spanning a segment/speaker boundary | — | +| `scripts/line-captions/create-line-captions.js` | Raw portrait video → audio extract → transcribe → align → diarize (if `--num-speakers > 1`) → `lines.json` + `lines.doc.txt` | — | +| `scripts/line-captions/merge-line-captions.js` | Re-parses `lines.doc.txt`, overwrites `text` on matching `lines.json` entries by `[id]` | — | +| `scripts/line-captions/render-line-captions.js` | Renders `LineCaptionClip-{id}` to `public/renders/` | — | --- From 6c8fce7eb599d960571f8150098ac170b941c2ea Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 23:24:39 +0800 Subject: [PATCH 8/9] fix: make paths.test.ts assertions OS-agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expected strings were hardcoded with POSIX forward slashes, so every test failed on Windows where path.join produces backslashes. Build expectations with path.join instead of literal strings — the production path.ts code was already correct, only the test's assertions were platform-specific. Co-Authored-By: Claude Sonnet 5 --- scripts/config/paths.test.ts | 55 ++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/scripts/config/paths.test.ts b/scripts/config/paths.test.ts index d0f32c2..7bb1327 100644 --- a/scripts/config/paths.test.ts +++ b/scripts/config/paths.test.ts @@ -1,3 +1,4 @@ +import path from 'path'; import { projectFile, artifactDir, @@ -25,69 +26,69 @@ import { thumbnailCameraProfiles, } from './paths'; -const ROOT = '/project/root'; +const ROOT = path.join('project', 'root'); describe('project / artifact', () => { it('projectFile', () => { - expect(projectFile(ROOT)).toBe('/project/root/.ragtech/project.json'); + expect(projectFile(ROOT)).toBe(path.join(ROOT, '.ragtech', 'project.json')); }); it('artifactDir', () => { - expect(artifactDir(ROOT)).toBe('/project/root/.ragtech/artifacts'); + expect(artifactDir(ROOT)).toBe(path.join(ROOT, '.ragtech', 'artifacts')); }); }); describe('transcribe pipeline', () => { it('transcribeInput', () => { - expect(transcribeInput(ROOT)).toBe('/project/root/public/transcribe/input'); + expect(transcribeInput(ROOT)).toBe(path.join(ROOT, 'public', 'transcribe', 'input')); }); it('transcriptRaw', () => { - expect(transcriptRaw(ROOT)).toBe('/project/root/public/transcribe/output/raw/transcript.raw.json'); + expect(transcriptRaw(ROOT)).toBe(path.join(ROOT, 'public', 'transcribe', 'output', 'raw', 'transcript.raw.json')); }); it('diarizationOutput', () => { - expect(diarizationOutput(ROOT)).toBe('/project/root/public/transcribe/output/raw/diarization.json'); + expect(diarizationOutput(ROOT)).toBe(path.join(ROOT, 'public', 'transcribe', 'output', 'raw', 'diarization.json')); }); it('hookQaDir', () => { - expect(hookQaDir(ROOT)).toBe('/project/root/public/transcribe/output/hook-qa'); + expect(hookQaDir(ROOT)).toBe(path.join(ROOT, 'public', 'transcribe', 'output', 'hook-qa')); }); }); describe('edit / longform output', () => { it('transcriptOutput', () => { - expect(transcriptOutput(ROOT)).toBe('/project/root/public/edit/transcript.json'); + expect(transcriptOutput(ROOT)).toBe(path.join(ROOT, 'public', 'edit', 'transcript.json')); }); it('transcriptDoc', () => { - expect(transcriptDoc(ROOT)).toBe('/project/root/public/edit/transcript.doc.txt'); + expect(transcriptDoc(ROOT)).toBe(path.join(ROOT, 'public', 'edit', 'transcript.doc.txt')); }); }); describe('sync pipeline', () => { it('syncVideoDir', () => { - expect(syncVideoDir(ROOT)).toBe('/project/root/public/sync/video'); + expect(syncVideoDir(ROOT)).toBe(path.join(ROOT, 'public', 'sync', 'video')); }); it('syncAudioDir', () => { - expect(syncAudioDir(ROOT)).toBe('/project/root/public/sync/audio'); + expect(syncAudioDir(ROOT)).toBe(path.join(ROOT, 'public', 'sync', 'audio')); }); it('syncOutputDir', () => { - expect(syncOutputDir(ROOT)).toBe('/project/root/public/sync/output'); + expect(syncOutputDir(ROOT)).toBe(path.join(ROOT, 'public', 'sync', 'output')); }); it('syncedVideo', () => { - expect(syncedVideo(ROOT)).toBe('/project/root/public/sync/output/synced-output.mp4'); + expect(syncedVideo(ROOT)).toBe(path.join(ROOT, 'public', 'sync', 'output', 'synced-output.mp4')); }); it('syncedVideoAngle — index 1', () => { - expect(syncedVideoAngle(1, ROOT)).toBe('/project/root/public/sync/output/synced-output-1.mp4'); + expect(syncedVideoAngle(1, ROOT)).toBe(path.join(ROOT, 'public', 'sync', 'output', 'synced-output-1.mp4')); }); it('syncedVideoAngle — index 2', () => { - expect(syncedVideoAngle(2, ROOT)).toBe('/project/root/public/sync/output/synced-output-2.mp4'); + expect(syncedVideoAngle(2, ROOT)).toBe(path.join(ROOT, 'public', 'sync', 'output', 'synced-output-2.mp4')); }); it('syncedVideoAngle — index 0 throws', () => { @@ -101,55 +102,55 @@ describe('sync pipeline', () => { describe('camera', () => { it('cameraProfiles', () => { - expect(cameraProfiles(ROOT)).toBe('/project/root/public/camera/camera-profiles.json'); + expect(cameraProfiles(ROOT)).toBe(path.join(ROOT, 'public', 'camera', 'camera-profiles.json')); }); }); describe('shorts pipeline', () => { it('shortsDir', () => { - expect(shortsDir(ROOT)).toBe('/project/root/public/shorts'); + expect(shortsDir(ROOT)).toBe(path.join(ROOT, 'public', 'shorts')); }); it('shortClipDir', () => { - expect(shortClipDir('clip-01', ROOT)).toBe('/project/root/public/shorts/clip-01'); + expect(shortClipDir('clip-01', ROOT)).toBe(path.join(ROOT, 'public', 'shorts', 'clip-01')); }); it('shortTranscript', () => { - expect(shortTranscript('clip-01', ROOT)).toBe('/project/root/public/shorts/clip-01/transcript.json'); + expect(shortTranscript('clip-01', ROOT)).toBe(path.join(ROOT, 'public', 'shorts', 'clip-01', 'transcript.json')); }); it('shortDoc', () => { - expect(shortDoc('clip-01', ROOT)).toBe('/project/root/public/shorts/clip-01/transcript.doc.txt'); + expect(shortDoc('clip-01', ROOT)).toBe(path.join(ROOT, 'public', 'shorts', 'clip-01', 'transcript.doc.txt')); }); it('shortsCameraProfiles', () => { - expect(shortsCameraProfiles(ROOT)).toBe('/project/root/public/shorts/camera-profiles.json'); + expect(shortsCameraProfiles(ROOT)).toBe(path.join(ROOT, 'public', 'shorts', 'camera-profiles.json')); }); it('shortsTranscriptRaw', () => { expect(shortsTranscriptRaw(ROOT)).toBe( - '/project/root/public/shorts/transcribe/output/raw/transcript.raw.json', + path.join(ROOT, 'public', 'shorts', 'transcribe', 'output', 'raw', 'transcript.raw.json'), ); }); }); describe('carousel pipeline', () => { it('carouselClipDir', () => { - expect(carouselClipDir('ep-42', ROOT)).toBe('/project/root/public/carousel/ep-42'); + expect(carouselClipDir('ep-42', ROOT)).toBe(path.join(ROOT, 'public', 'carousel', 'ep-42')); }); it('carouselTranscript', () => { - expect(carouselTranscript('ep-42', ROOT)).toBe('/project/root/public/carousel/ep-42/transcript.json'); + expect(carouselTranscript('ep-42', ROOT)).toBe(path.join(ROOT, 'public', 'carousel', 'ep-42', 'transcript.json')); }); it('carouselDoc', () => { - expect(carouselDoc('ep-42', ROOT)).toBe('/project/root/public/carousel/ep-42/transcript.doc.txt'); + expect(carouselDoc('ep-42', ROOT)).toBe(path.join(ROOT, 'public', 'carousel', 'ep-42', 'transcript.doc.txt')); }); }); describe('thumbnail pipeline', () => { it('thumbnailCameraProfiles', () => { - expect(thumbnailCameraProfiles(ROOT)).toBe('/project/root/public/thumbnail/camera-profiles.json'); + expect(thumbnailCameraProfiles(ROOT)).toBe(path.join(ROOT, 'public', 'thumbnail', 'camera-profiles.json')); }); }); @@ -157,7 +158,7 @@ describe('cwd defaults', () => { it('uses process.cwd() as root when cwd is omitted', () => { const result = transcriptOutput(); expect(result.startsWith(process.cwd())).toBe(true); - expect(result.endsWith('/public/edit/transcript.json')).toBe(true); + expect(result.endsWith(path.join('public', 'edit', 'transcript.json'))).toBe(true); }); }); From b15d3e38805939851fa62bceb91badbeaf3a4a1c Mon Sep 17 00:00:00 2001 From: Natasha Ann Lum Date: Mon, 6 Jul 2026 23:33:42 +0800 Subject: [PATCH 9/9] fix: exclude .venv from pre-push secret-token scan A vendored, gitignored Python virtualenv (.venv/Lib/site-packages/PIL/ ImageFont.py) contains a base64 font-glyph blob that coincidentally matches the sk-[A-Za-z0-9]{48} pattern, false-positiving as a leaked key and blocking every push on machines with a local .venv present. Exclude it the same way node_modules and .next already are. Co-Authored-By: Claude Sonnet 5 --- .husky/pre-push | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.husky/pre-push b/.husky/pre-push index 00076a7..8876945 100755 --- a/.husky/pre-push +++ b/.husky/pre-push @@ -57,7 +57,7 @@ if grep -rEn \ 'ghp_[A-Za-z0-9]{36}|AKIA[0-9A-Z]{16}|sk-ant-[A-Za-z0-9_\-]{90,}|sk-[A-Za-z0-9]{48}' \ --include='*.ts' --include='*.tsx' --include='*.js' \ --include='*.json' --include='*.py' \ - --exclude-dir=node_modules --exclude-dir=.next \ + --exclude-dir=node_modules --exclude-dir=.next --exclude-dir=.venv \ .; then echo "✗ Secret token detected — remove from source before pushing." exit 1