From c8c66ce8f41a6ae9bd8e4b9fd932aa94cf155260 Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Tue, 28 Jul 2026 20:04:56 +0200 Subject: [PATCH 1/4] feat(subtitles): take WebVTT cue settings from packet side data #233 shipped styling for every text format and reported WebVTT placement as upstream-blocked. That was the wrong layer. libavcodec's WebVTT decoder really does drop line/position/align (webvttdec.c says so as a @todo in its header), so nothing about them is in the ASS event line it synthesises. The demuxer keeps them: libavformat/webvttdec.c attaches the verbatim settings string per packet as AV_PKT_DATA_WEBVTT_SETTINGS, and matroskadec.c propagates the same side data for WebVTT in Matroska. So both decoders now read that side data and map it onto SubtitleTextPlacement, and the #112 packet store carries the string through a rebuild (side data does not survive in the payload, so a stored packet would otherwise lose the placement a freshly demuxed one has). An ASS \an or \pos still wins: that came from the payload itself. Two deliberate limits, both because the placement models an anchor and a point rather than a box: `size` has nowhere to go, and a `position` without a `line` keeps only the alignment column, since a point needs both axes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01At1agC2qU2Y2MX2BKLdT4N --- .../AetherEngine/AetherEngine+Subtitles.swift | 4 + .../Decoder/EmbeddedSubtitleDecoder.swift | 5 + .../Decoder/SubtitleDecoder.swift | 5 + .../Subtitles/SubtitlePacketStore.swift | 32 +++- .../Subtitles/WebVTTCueSettings.swift | 147 ++++++++++++++++++ .../Issue233WebVTTCueSettingsTests.swift | 146 +++++++++++++++++ 6 files changed, 332 insertions(+), 7 deletions(-) create mode 100644 Sources/AetherEngine/Subtitles/WebVTTCueSettings.swift create mode 100644 Tests/AetherEngineTests/Issue233WebVTTCueSettingsTests.swift diff --git a/Sources/AetherEngine/AetherEngine+Subtitles.swift b/Sources/AetherEngine/AetherEngine+Subtitles.swift index ee2ff5aa..effa9fa0 100644 --- a/Sources/AetherEngine/AetherEngine+Subtitles.swift +++ b/Sources/AetherEngine/AetherEngine+Subtitles.swift @@ -579,6 +579,10 @@ extension AetherEngine { pkt.pointee.dts = pkt.pointee.pts pkt.pointee.duration = Int64((entry.durationSeconds * 1000).rounded()) pkt.pointee.flags = entry.flags + // #233: WebVTT placement lives in side data, not in the payload, so it has to be put back. + if let settings = entry.webvttSettings { + WebVTTCueSettings.attach(settings: settings, to: pkt) + } return decoder.decode(packet: pkt, streamTimeBase: AVRational(num: 1, den: 1000)) } diff --git a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift index 8a560ed0..2818d119 100644 --- a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift +++ b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift @@ -238,6 +238,11 @@ final class EmbeddedSubtitleDecoder { } avsubtitle_free(&sub) + // #233: WebVTT cue settings never reach the ASS event line (the decoder drops them), but + // the demuxer keeps them on the packet, and the #112 store carries them through a rebuild. + // An ASS `\an` / `\pos` still wins, since that came from the payload itself. + placement = placement ?? WebVTTCueSettings.placement(onPacket: packet) + let merged = textLines .joined(separator: "\n") .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Sources/AetherEngine/Decoder/SubtitleDecoder.swift b/Sources/AetherEngine/Decoder/SubtitleDecoder.swift index 50747331..4f262a91 100644 --- a/Sources/AetherEngine/Decoder/SubtitleDecoder.swift +++ b/Sources/AetherEngine/Decoder/SubtitleDecoder.swift @@ -284,6 +284,11 @@ enum SubtitleDecoder { } avsubtitle_free(&sub) + // #233: WebVTT cue settings never reach the ASS event line (the decoder drops + // them), but the demuxer keeps them on the packet. An ASS `\an` / `\pos` still + // wins, since that came from the payload itself. + placement = placement ?? WebVTTCueSettings.placement(onPacket: pkt) + let merged = lines .joined(separator: "\n") .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Sources/AetherEngine/Subtitles/SubtitlePacketStore.swift b/Sources/AetherEngine/Subtitles/SubtitlePacketStore.swift index b148ad9c..1f2cc864 100644 --- a/Sources/AetherEngine/Subtitles/SubtitlePacketStore.swift +++ b/Sources/AetherEngine/Subtitles/SubtitlePacketStore.swift @@ -13,6 +13,19 @@ struct StoredSubtitlePacket: Sendable { /// decode packet (AV_PKT_FLAG_KEY matters for bitmap acquisition points). let flags: Int32 let payload: Data + /// #233: `AV_PKT_DATA_WEBVTT_SETTINGS` as attached by the demuxer. WebVTT cue settings live + /// only in packet side data (the decoder never puts them in the ASS line), so a rebuilt packet + /// loses the cue's placement unless the string rides along with the payload. + let webvttSettings: String? + + init(ptsSeconds: Double, durationSeconds: Double, flags: Int32, payload: Data, + webvttSettings: String? = nil) { + self.ptsSeconds = ptsSeconds + self.durationSeconds = durationSeconds + self.flags = flags + self.payload = payload + self.webvttSettings = webvttSettings + } } final class SubtitlePacketStore: @unchecked Sendable { @@ -103,21 +116,23 @@ final class SubtitlePacketStore: @unchecked Sendable { } func append(streamIndex: Int32, ptsSeconds: Double, durationSeconds: Double, - flags: Int32 = 0, payload: Data) { + flags: Int32 = 0, payload: Data, webvttSettings: String? = nil) { lock.lock(); defer { lock.unlock() } appendLocked(streamIndex: streamIndex, ptsSeconds: ptsSeconds, - durationSeconds: durationSeconds, flags: flags, payload: payload) + durationSeconds: durationSeconds, flags: flags, payload: payload, + webvttSettings: webvttSettings) } private func appendLocked(streamIndex: Int32, ptsSeconds: Double, durationSeconds: Double, - flags: Int32, payload: Data) { + flags: Int32, payload: Data, webvttSettings: String? = nil) { let before = bytesByStream[streamIndex] ?? 0 var entries = entriesByStream[streamIndex] ?? [] var bytes = before let entry = StoredSubtitlePacket(ptsSeconds: ptsSeconds, durationSeconds: durationSeconds, flags: flags, - payload: payload) + payload: payload, + webvttSettings: webvttSettings) // #235: several packets legitimately share a PTS. ASS/SSA authors overlapping lines on // identical Start/End, and a karaoke or layered-style track puts a whole burst of distinct // Dialogue events on one timestamp. Only a byte-identical payload is the pump and the @@ -222,21 +237,24 @@ final class SubtitlePacketStore: @unchecked Sendable { flags: packet.pointee.flags, payload: Data(bytes: data, count: Int(packet.pointee.size)), assembleSplitDisplaySets: assembleSplitDisplaySets, - writer: writer) + writer: writer, + webvttSettings: WebVTTCueSettings.settings(onPacket: packet)) } /// Testable core of `harvest`. ptsSeconds nil = packet carried no PTS (AV_NOPTS_VALUE): /// dropped on the per-packet path, folded into the pending set on the assembly path. func harvestChunk(streamIndex: Int32, ptsSeconds: Double?, durationSeconds: Double, flags: Int32, payload: Data, assembleSplitDisplaySets: Bool, - writer: Writer = .pump) { + writer: Writer = .pump, webvttSettings: String? = nil) { lock.lock(); defer { lock.unlock() } guard assembleSplitDisplaySets else { guard let ptsSeconds else { return } appendLocked(streamIndex: streamIndex, ptsSeconds: ptsSeconds, - durationSeconds: durationSeconds, flags: flags, payload: payload) + durationSeconds: durationSeconds, flags: flags, payload: payload, + webvttSettings: webvttSettings) return } + // The assembly path below is PGS display sets; those carry no WebVTT settings. // Mirror the decoder's SUP-wrapper rule: strip a leading "PG" 10-byte header so // concatenated chunks form one clean [type][len BE][body] segment run. var chunk = payload diff --git a/Sources/AetherEngine/Subtitles/WebVTTCueSettings.swift b/Sources/AetherEngine/Subtitles/WebVTTCueSettings.swift new file mode 100644 index 00000000..8fe528c9 --- /dev/null +++ b/Sources/AetherEngine/Subtitles/WebVTTCueSettings.swift @@ -0,0 +1,147 @@ +import CoreGraphics +import Foundation +import Libavcodec + +/// WebVTT cue settings (`line`, `position`, `align`, `size`, `vertical`) mapped onto +/// `SubtitleTextPlacement` (#233). +/// +/// libavcodec's WebVTT decoder does not convert these into the ASS event line it synthesises +/// (`libavcodec/webvttdec.c` says so as a `@todo` in its file header), so the markup path carries +/// no trace of them. The demuxer keeps them: `libavformat/webvttdec.c` attaches the verbatim +/// settings string to each packet as `AV_PKT_DATA_WEBVTT_SETTINGS`, and `matroskadec.c` propagates +/// the same side data for WebVTT in Matroska. Both subtitle decoders read that side data and hand +/// the string here. +/// +/// Two deliberate limits, both because `SubtitleTextPlacement` models an anchor and a point rather +/// than a box: +/// - `size` (box width) has nowhere to go and is ignored. +/// - a point needs both axes, so a `position` without a `line` keeps only the alignment column. +/// The column is where a host's horizontal margin logic already lives, so little is lost. +enum WebVTTCueSettings { + + /// Placement a cue-settings string asks for, or nil when it asks for nothing the engine can + /// express. Values follow the WebVTT grammar, plus the older `left` / `right` / `middle` + /// spellings that real tools still emit. + static func placement(fromSettings settings: String) -> SubtitleTextPlacement? { + var column: Int? // 0 left, 1 centre, 2 right + var row: Int? // 0 bottom, 1 middle, 2 top + var lineFraction: Double? + var positionFraction: Double? + var sawPlacementSetting = false + + for field in settings.split(whereSeparator: { $0 == " " || $0 == "\t" }) { + let parts = field.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + guard parts.count == 2 else { continue } + let value = String(parts[1]) + + switch parts[0].lowercased() { + case "vertical": + // A writing mode, not a placement. Rendering sideways text at a horizontal anchor + // is worse than handing the cue back to the host untouched. + if value == "rl" || value == "lr" { return nil } + + case "align": + guard let c = Self.column(forAlign: value) else { continue } + column = c + sawPlacementSetting = true + + case "line": + let fields = value.split(separator: ",", omittingEmptySubsequences: false) + guard let first = fields.first else { continue } + if first.hasSuffix("%") { + guard let f = Self.fraction(String(first)) else { continue } + lineFraction = f + // The spec's default line alignment is `start`, which anchors the box TOP at + // the offset: at line:90% a two-line cue then hangs off the frame and the + // renderer has to walk it back. Anchoring to the nearer edge is the stable + // reading of the same intent. An explicit alignment still wins, below. + row = f >= 0.5 ? 0 : 2 + } else if let n = Int(first) { + // Line boxes need the rendered line height, which the engine does not have. + // The half of the frame the number names is unambiguous, so that is all it + // yields; no anchor point, so the host keeps its own margin. + row = n < 0 ? 0 : 2 + } else { + continue + } + if fields.count > 1, let explicit = Self.row(forLineAlign: String(fields[1])) { + row = explicit + } + sawPlacementSetting = true + + case "position": + let head = value.split(separator: ",", omittingEmptySubsequences: false).first ?? "" + guard let f = Self.fraction(String(head)) else { continue } + positionFraction = f + sawPlacementSetting = true + + default: + continue // size, region and anything else carry no placement the engine models. + } + } + + guard sawPlacementSetting else { return nil } + + let c = column ?? 1 + let r = row ?? 0 + var position: CGPoint? + if let y = lineFraction { + // Only a vertical percentage makes a point meaningful; x then follows the column so + // the anchor and the alignment agree. + let x = positionFraction ?? [0.0, 0.5, 1.0][c] + position = CGPoint(x: x, y: y) + } + return SubtitleTextPlacement(alignment: r * 3 + c + 1, position: position) + } + + /// The settings string the demuxer attached to this packet, if any. Verbatim, exactly as it + /// followed the timestamp line in the source. + static func settings(onPacket packet: UnsafePointer) -> String? { + var size = 0 + guard let raw = av_packet_get_side_data(packet, AV_PKT_DATA_WEBVTT_SETTINGS, &size), + size > 0 else { return nil } + return String(bytes: UnsafeRawBufferPointer(start: raw, count: size), encoding: .utf8) + } + + /// Placement for a packet in one step: nil unless the packet carries settings that resolve. + static func placement(onPacket packet: UnsafePointer) -> SubtitleTextPlacement? { + settings(onPacket: packet).flatMap { placement(fromSettings: $0) } + } + + /// Re-attach a settings string to a rebuilt packet (#112 store path), so a cue decoded from + /// the packet store resolves its placement the same way a freshly demuxed one does. + @discardableResult + static func attach(settings: String, to packet: UnsafeMutablePointer) -> Bool { + var bytes = Array(settings.utf8) + guard !bytes.isEmpty, + let buf = av_packet_new_side_data(packet, AV_PKT_DATA_WEBVTT_SETTINGS, bytes.count) + else { return false } + memcpy(buf, &bytes, bytes.count) + return true + } + + private static func column(forAlign value: String) -> Int? { + switch value.lowercased() { + case "start", "left": return 0 + case "center", "centre", "middle": return 1 + case "end", "right": return 2 + default: return nil + } + } + + private static func row(forLineAlign value: String) -> Int? { + switch value.lowercased() { + case "start": return 2 + case "center", "centre", "middle": return 1 + case "end": return 0 + default: return nil + } + } + + /// `NN%` as a [0, 1] fraction, clamped. Out of range is a malformed file rather than a + /// different intent, so it clamps instead of dropping the setting. + private static func fraction(_ value: String) -> Double? { + guard value.hasSuffix("%"), let percent = Double(value.dropLast()) else { return nil } + return min(1.0, max(0.0, percent / 100.0)) + } +} diff --git a/Tests/AetherEngineTests/Issue233WebVTTCueSettingsTests.swift b/Tests/AetherEngineTests/Issue233WebVTTCueSettingsTests.swift new file mode 100644 index 00000000..ba243e23 --- /dev/null +++ b/Tests/AetherEngineTests/Issue233WebVTTCueSettingsTests.swift @@ -0,0 +1,146 @@ +import Testing +import Foundation +@testable import AetherEngine + +/// #233 follow-up: WebVTT cue settings never reached the host, and the first pass concluded they +/// could not. +/// +/// That conclusion looked at the wrong layer. libavcodec's WebVTT *decoder* really does drop +/// `line` / `position` / `align` (`libavcodec/webvttdec.c` says so as a `@todo` in its file +/// header), so nothing about the placement is in the ASS event line the decoder synthesises. The +/// *demuxer* keeps them: `libavformat/webvttdec.c` attaches the verbatim settings string to every +/// packet as `AV_PKT_DATA_WEBVTT_SETTINGS`, and `matroskadec.c` propagates the same side data for +/// WebVTT in Matroska. So the placement travels one layer above where the parser was looking. +struct Issue233WebVTTCueSettingsTests { + + private func placement(_ settings: String) -> SubtitleTextPlacement? { + WebVTTCueSettings.placement(fromSettings: settings) + } + + // MARK: - Horizontal + + @Test("align picks the ASS numpad column and leaves the row at the bottom default") + func alignColumns() { + #expect(placement("align:start")?.alignment == 1) + #expect(placement("align:center")?.alignment == 2) + #expect(placement("align:end")?.alignment == 3) + // Old draft spellings still emitted by real tools. + #expect(placement("align:left")?.alignment == 1) + #expect(placement("align:right")?.alignment == 3) + #expect(placement("align:middle")?.alignment == 2) + // A column alone is an anchor, not a geometric point: the host keeps its own margin. + #expect(placement("align:center")?.position == nil) + } + + // MARK: - Vertical + + @Test("a line percentage becomes a position anchored to the nearer edge") + func linePercentage() { + let top = placement("line:10%") + #expect(top?.alignment == 8) + #expect(top?.position?.x == 0.5) + #expect(top?.position?.y == 0.1) + + // 90% with the spec's default line alignment would anchor the box TOP at 90% and hang a + // two-line cue off the frame. Anchoring to the nearer edge renders what the file means. + let bottom = placement("line:90%") + #expect(bottom?.alignment == 2) + #expect(bottom?.position?.y == 0.9) + } + + @Test("an explicit line alignment wins over the nearer-edge default") + func lineAlignSuffix() { + #expect(placement("line:10%,start")?.alignment == 8) + #expect(placement("line:10%,center")?.alignment == 5) + #expect(placement("line:10%,end")?.alignment == 2) + #expect(placement("line:90%,start")?.alignment == 8) + } + + @Test("a line NUMBER keeps its half of the frame and no position") + func lineNumber() { + // Line boxes need the rendered line height, which the engine does not have. The half of + // the frame the number asks for is unambiguous, so that is all it yields. + #expect(placement("line:0")?.alignment == 8) + #expect(placement("line:0")?.position == nil) + #expect(placement("line:-1")?.alignment == 2) + #expect(placement("line:-1")?.position == nil) + #expect(placement("line:3")?.alignment == 8) + } + + // MARK: - Combined + + @Test("position and align resolve the horizontal anchor of a placed cue") + func positionWithLine() { + let p = placement("line:90% position:20% align:start") + #expect(p?.alignment == 1) + #expect(p?.position?.x == 0.2) + #expect(p?.position?.y == 0.9) + } + + @Test("a horizontal offset without a line keeps the column only") + func positionWithoutLine() { + // A point needs both axes; the engine models no half-position. The column still carries + // most of the intent and leaves the host's vertical margin intact. + let p = placement("position:20% align:start") + #expect(p?.alignment == 1) + #expect(p?.position == nil) + } + + @Test("settings are separated by spaces or tabs") + func separators() { + #expect(placement("align:start\tline:0")?.alignment == 7) + #expect(placement(" align:end line:0 ")?.alignment == 9) + } + + // MARK: - Nothing to say + + @Test("settings the engine cannot express produce no placement") + func unsupported() { + #expect(placement("") == nil) + #expect(placement("size:50%") == nil) + #expect(placement("region:fred") == nil) + // Vertical writing mode is not a placement the compositor can honour, and placing sideways + // text as if it were horizontal is worse than leaving it to the host. + #expect(placement("vertical:rl") == nil) + #expect(placement("vertical:lr align:start line:0") == nil) + } + + @Test("malformed values are ignored rather than guessed") + func malformed() { + #expect(placement("line:abc") == nil) + #expect(placement("position:%") == nil) + #expect(placement("align:") == nil) + #expect(placement("line:200%")?.position?.y == 1.0) // clamped, not dropped + #expect(placement("line:-30%")?.position?.y == 0.0) + } + + // MARK: - End to end through the real demuxer + + @Test("cue settings survive a sidecar WebVTT decode") + func sidecarDecodeCarriesPlacement() async throws { + let vtt = """ + WEBVTT + + 00:00:01.000 --> 00:00:03.000 line:10% align:start + Top left caption + + 00:00:04.000 --> 00:00:06.000 + Plain caption + """ + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("ae-issue233-\(ProcessInfo.processInfo.globallyUniqueString).vtt") + try vtt.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + let result = try await SubtitleDecoder.decodeFile(url: url) + #expect(result.cues.count == 2) + + let placed = try #require(result.cues.first) + #expect(placed.placement?.alignment == 7) + #expect(placed.placement?.position?.x == 0.0) + #expect(placed.placement?.position?.y == 0.1) + + // A cue that asks for nothing still asks for nothing. + #expect(result.cues.last?.placement == nil) + } +} From fe904d47cb0c4f7311ea0e203c9fe62b4b9f22f1 Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Tue, 28 Jul 2026 20:05:11 +0200 Subject: [PATCH 2/4] fix(subtitles): keep the teletext grid row when the page carries no \an Reported on #233: teletext captions lost their vertical placement. Only on pages libzvbi does not classify as subtitle pages. gen_sub_ass derives the vertical anchor from the grid row itself and emits it as {\anN}: vertical_align = (2 - (av_clip(i + 1, 0, 23) / 8)); av_bprintf(&buf, "{\\an%d}", alignment + vertical_align * 3); but that block sits behind is_subtitle_page, which comes from the row-0 header flags (NEWSFLASH clear AND SUBTITLE set AND SUPPRESS_HEADER set). When a broadcaster does not set all three, or the header has not been seen yet, the else path writes the whole page instead, one " \N" per grid row with the empty ones included. The row ordinal is then the only carrier of the position, and edgeTrimmed was throwing it away. teletextBody now falls back to ffmpeg's own formula on that ordinal rather than inventing a scale, and never overrides an \an that arrived: {\an2} is explicitly bottom and produces exactly the same empty output that "no information" would. Coarse on purpose, three bands is what the source encodes; per-row offsets make consecutive cues of different heights sit at different heights, which reads as a blink. Two consistency fixes in the same area, found while measuring this: - The styled path edge-trimmed with a literal space/tab/newline test while the plain path used .whitespacesAndNewlines, so a Unicode space (U+00A0 among them) survived on a styled cue and not on an unstyled one. Teletext cannot produce one (IS_TXT_SPACE maps U+00A0 to a space, which decode_string writes as \h), but SRT, WebVTT and ASS payloads can. - The #107 blank-line fold worked per run plus one run boundary, which misses the case the source produces most easily: the padding of an empty row carries the spacing attribute that changes colour, so the blank row lands in a whitespace-only run of its own and breaks the chain. It now folds the flattened sequence and re-splits along the original run boundaries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01At1agC2qU2Y2MX2BKLdT4N --- .../Subtitles/SubtitleRectText.swift | 118 ++++++++++++--- .../Issue233TeletextRowPlacementTests.swift | 138 ++++++++++++++++++ 2 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 Tests/AetherEngineTests/Issue233TeletextRowPlacementTests.swift diff --git a/Sources/AetherEngine/Subtitles/SubtitleRectText.swift b/Sources/AetherEngine/Subtitles/SubtitleRectText.swift index f41ca0b3..8fe70a9a 100644 --- a/Sources/AetherEngine/Subtitles/SubtitleRectText.swift +++ b/Sources/AetherEngine/Subtitles/SubtitleRectText.swift @@ -114,9 +114,14 @@ enum SubtitleRectText { /// -> newline and `\h` -> space. Tags that merely look like these (`\be`, `\bord`, `\iclip`, /// `\shad`, `\fscx`, `\fsp`) are left alone. Adjacent runs of equal styling are collapsed. /// nil when nothing displayable remains. + /// + /// `firstTextRow` is the 0-based index of the newline-delimited row the first displayable + /// character sits on, measured BEFORE the edge trim removes it. libzvbi carries teletext + /// row positioning in exactly that ordinal on pages it does not flag as subtitle pages + /// (see `teletextBody`); every other format ignores it. static func styledRuns(fromASSEventLine line: String, playRes: CGSize = SubtitleRectText.defaultASSPlayRes) - -> (runs: [SubtitleTextRun], placement: SubtitleTextPlacement?)? { + -> (runs: [SubtitleTextRun], placement: SubtitleTextPlacement?, firstTextRow: Int)? { var body = line if body.hasPrefix("Dialogue: ") { body.removeFirst("Dialogue: ".count) } let parts = body.split(separator: ",", maxSplits: 8, omittingEmptySubsequences: false) @@ -167,9 +172,24 @@ enum SubtitleRectText { var runs = pieces.map { $0.style.run($0.text) } let placement = (alignment == nil && position == nil) ? nil : SubtitleTextPlacement(alignment: alignment, position: position) + let firstTextRow = Self.firstTextRow(in: runs) guard let trimmed = edgeTrimmed(runs) else { return nil } runs = trimmed - return (runs, placement) + return (runs, placement, firstTextRow) + } + + /// Index of the row the first displayable character sits on, counting the newline-delimited + /// rows of the untrimmed run sequence from 0. + private static func firstTextRow(in runs: [SubtitleTextRun]) -> Int { + var row = 0 + for run in runs { + for ch in run.text { + if ch.isNewline { row += 1; continue } + if ch.isWhitespace { continue } + return row + } + } + return row } /// Apply one override block's tags. Inline attributes mutate `style`; `\an` and `\pos` are @@ -236,15 +256,18 @@ enum SubtitleRectText { // teletext ass can prefix a row-positioning newline that would otherwise render as a blank // line ONLY on coloured cues (#107). Interior blank lines are folded separately below; // single line breaks and colours are preserved. + // Predicate matches the plain path's `.whitespacesAndNewlines` (Unicode Zs plus tab plus + // the newline characters). A literal-space/tab/newline test let U+00A0 survive on a styled + // cue that an unstyled one trimmed, so the two paths disagreed on the same payload. while let first = cleaned.first { - let d = String(first.text.drop(while: { $0 == " " || $0 == "\t" || $0 == "\n" })) + let d = String(first.text.drop(while: \.isWhitespace)) if d.isEmpty { cleaned.removeFirst(); continue } cleaned[0] = first.withText(d) break } while let last = cleaned.last { var s = last.text - while let c = s.last, c == " " || c == "\t" || c == "\n" { s.removeLast() } + while let c = s.last, c.isWhitespace { s.removeLast() } if s.isEmpty { cleaned.removeLast(); continue } cleaned[cleaned.count - 1] = last.withText(s) break @@ -256,25 +279,50 @@ enum SubtitleRectText { /// Collapse interior blank lines across a run sequence (#107). libzvbi joins teletext rows with /// `\N`, so a caption whose lines sit on non-adjacent rows (an empty row between them, used only /// for vertical placement) arrives as `line1\n\nline2` and would render a blank line the - /// broadcaster never intended. Consecutive newlines (optionally separated by horizontal - /// whitespace) fold to one; single line breaks and colours are preserved. The empty row lands - /// inside a single run in practice, but a colour change at a row boundary could split it, so the - /// boundary between adjacent runs is folded too. + /// broadcaster never intended. Consecutive newlines separated by nothing but horizontal + /// whitespace fold to one; single line breaks, colours and the indentation of a real row are + /// preserved. + /// + /// Folds the FLATTENED sequence and re-splits it along the original run boundaries. A per-run + /// regex plus an adjacent-pair check missed the case the source produces most easily: the + /// padding of an empty row carries the spacing attribute that changes colour, so the blank row + /// can land in a whitespace-only run of its own and break the chain (`line` / whitespace run / + /// `line`). private static func collapseInteriorBlankLines(_ runs: [SubtitleTextRun]) -> [SubtitleTextRun] { - var folded = runs.map { run in - run.withText(run.text.replacingOccurrences( - of: #"\n(?:[ \t]*\n)+"#, with: "\n", options: .regularExpression)) + var chars: [Character] = [] + var owner: [Int] = [] + for (index, run) in runs.enumerated() { + for ch in run.text { + chars.append(ch) + owner.append(index) + } } + + var keep = [Bool](repeating: true, count: chars.count) var i = 0 - while i < folded.count - 1 { - if folded[i].text.hasSuffix("\n") { - var next = folded[i + 1].text - while next.hasPrefix("\n") { next.removeFirst() } - folded[i + 1] = folded[i + 1].withText(next) + while i < chars.count { + guard chars[i].isNewline else { i += 1; continue } + // Scan past horizontal whitespace and further newlines; everything up to and including + // the last newline found is one blank-row gap and collapses onto the first newline. + var probe = i + 1 + var lastNewline = i + while probe < chars.count { + let c = chars[probe] + if c.isNewline { lastNewline = probe } + else if !c.isWhitespace { break } + probe += 1 } - i += 1 + if lastNewline > i { + for k in (i + 1)...lastNewline { keep[k] = false } + } + i = lastNewline + 1 + } + + var texts = [String](repeating: "", count: runs.count) + for (index, ch) in chars.enumerated() where keep[index] { + texts[owner[index]].append(ch) } - return folded.filter { !$0.text.isEmpty } + return zip(runs, texts).map { $0.withText($1) }.filter { !$0.text.isEmpty } } /// Body + placement for any text rect's ASS event line (#233): `.richText` when a run asks for @@ -288,13 +336,41 @@ enum SubtitleRectText { } /// Teletext variant (#107): identical, plus the interior blank-line fold libzvbi's row joining - /// requires. That fold is deliberately teletext-only, since a blank line in an ASS or SRT cue - /// can be intentional. + /// requires, plus the grid-row placement fallback below. Both are deliberately teletext-only, + /// since a blank line in an ASS or SRT cue can be intentional and a leading one is never a row + /// ordinal. static func teletextBody(fromASSEventLine line: String, playRes: CGSize = SubtitleRectText.defaultASSPlayRes) -> (body: SubtitleCue.Body, placement: SubtitleTextPlacement?)? { guard let parsed = styledRuns(fromASSEventLine: line, playRes: playRes) else { return nil } - return body(for: collapseInteriorBlankLines(parsed.runs)).map { ($0, parsed.placement) } + let placement = parsed.placement ?? gridPlacement(firstTextRow: parsed.firstTextRow) + return body(for: collapseInteriorBlankLines(parsed.runs)).map { ($0, placement) } + } + + /// Vertical anchor for a teletext page that carried no `\an` (#233). + /// + /// `gen_sub_ass` derives the anchor from the grid row and emits it as `{\anN}`, but only for + /// pages `subtitle_map` flags as subtitle pages (row-0 header: NEWSFLASH clear AND SUBTITLE set + /// AND SUPPRESS_HEADER set). Off that path it writes the whole page instead, one `" \N"` per + /// grid row with the empty ones included, and the row ordinal becomes the only carrier of the + /// position. This is that same derivation applied to the ordinal, so a page keeps the placement + /// its broadcaster chose whether or not the flags made it through: + /// + /// vertical_align = 2 - av_clip(i + 1, 0, 23) / 8 + /// an = alignment + vertical_align * 3 + /// + /// with `i` the grid row (emitted row + 1, since `txt_chop_top` defaults to 1) and the + /// horizontal alignment left at ffmpeg's own default of 2, centre: the column analysis it does + /// for subtitle pages needs the per-row trim that this path never ran. Coarse on purpose. Three + /// bands is what the source encodes; mapping each row to its own offset makes consecutive cues + /// of different heights sit at different heights, which reads as a blink. + /// + /// Never overrides an `\an` that arrived, including `{\an2}`: bottom is a real answer and it + /// looks exactly like no answer. + static func gridPlacement(firstTextRow row: Int) -> SubtitleTextPlacement { + let gridRow = row + 1 + let verticalAlign = 2 - min(max(gridRow + 1, 0), 23) / 8 + return SubtitleTextPlacement(alignment: 2 + verticalAlign * 3, position: nil) } private static func body(for runs: [SubtitleTextRun]) -> SubtitleCue.Body? { diff --git a/Tests/AetherEngineTests/Issue233TeletextRowPlacementTests.swift b/Tests/AetherEngineTests/Issue233TeletextRowPlacementTests.swift new file mode 100644 index 00000000..1e900d51 --- /dev/null +++ b/Tests/AetherEngineTests/Issue233TeletextRowPlacementTests.swift @@ -0,0 +1,138 @@ +import Testing +import Foundation +@testable import AetherEngine + +/// #233 follow-up, reported by tresby: teletext captions lost their vertical placement. +/// +/// Only on pages libzvbi does not classify as subtitle pages. `gen_sub_ass` +/// (`libavcodec/libzvbi-teletextdec.c`) derives the vertical anchor from the grid row itself and +/// emits it as `{\anN}`: +/// +/// vertical_align = (2 - (av_clip(i + 1, 0, 23) / 8)); +/// av_bprintf(&buf, "{\\an%d}", alignment + vertical_align * 3); +/// +/// but that whole block sits behind `is_subtitle_page`, which comes from the row-0 header flags +/// (NEWSFLASH clear AND SUBTITLE set AND SUPPRESS_HEADER set). When a broadcaster does not set all +/// three, or the header has not been seen yet, the else path runs instead: no `\an`, no per-row +/// trim, and every row including the empty ones goes out as `" \N"`. The row ordinal is then the +/// only carrier of the position, and `edgeTrimmed` was throwing it away. +/// +/// So the fallback reproduces ffmpeg's own formula on the emitted rows rather than inventing a +/// scale, and it must never fire when `\an` is present: `{\an2}` is explicitly bottom and produces +/// exactly the same empty derived output that "no information" would. +struct Issue233TeletextRowPlacementTests { + + /// Event line whose body starts on emitted row `row` (0-based), the shape a non-subtitle page + /// produces: one `" \N"` per grid row, blanks included. + private func page(firstTextRow row: Int, prefix: String = "") -> String { + let blanks = String(repeating: " \\N", count: row) + return "0,0,Teletext,,0,0,0,,\(prefix)\(blanks)CAPTION TEXT" + } + + private func alignment(_ line: String) -> Int? { + SubtitleRectText.teletextBody(fromASSEventLine: line)?.placement?.alignment + } + + // MARK: - Derivation + + @Test("the first text row picks the same third ffmpeg would have picked") + func rowThirds() { + // Emitted row r sits on grid row r + 1 (txt_chop_top defaults to 1), and ffmpeg tests + // i + 1, so the boundaries are r <= 5 top, r <= 13 middle, r >= 14 bottom. + #expect(alignment(page(firstTextRow: 0)) == 8) + #expect(alignment(page(firstTextRow: 5)) == 8) + #expect(alignment(page(firstTextRow: 6)) == 5) + #expect(alignment(page(firstTextRow: 13)) == 5) + #expect(alignment(page(firstTextRow: 14)) == 2) + #expect(alignment(page(firstTextRow: 21)) == 2) + } + + @Test("rows padded with hard spaces are still blank rows") + func hardSpacePadding() { + // libzvbi writes every space as \h, so a "blank" row is not an empty string. + let padded = "0,0,Teletext,,0,0,0,," + + String(repeating: "\\h\\h\\h \\N", count: 15) + "CAPTION" + #expect(alignment(padded) == 2) + } + + @Test("the derived placement carries no anchor point") + func noPosition() { + // A third of the frame is all the row ordinal says. Inventing a \pos-style anchor from it + // would pin the block and take the host's margin away. + let p = SubtitleRectText.teletextBody(fromASSEventLine: page(firstTextRow: 16))?.placement + #expect(p?.position == nil) + } + + // MARK: - What must not change + + @Test("an explicit an wins, including when it means bottom") + func explicitAlignmentWins() { + // The trap: {\an2} is bottom, and bottom is also what "derived nothing" looks like. + #expect(alignment(page(firstTextRow: 2, prefix: #"{\an2}"#)) == 2) + #expect(alignment(page(firstTextRow: 18, prefix: #"{\an8}"#)) == 8) + #expect(alignment(page(firstTextRow: 0, prefix: #"{\an1}"#)) == 1) + } + + @Test("the derivation is teletext only") + func teletextOnly() { + // A leading newline in an ASS or SRT cue is an intentional blank line, not a grid row. + let line = page(firstTextRow: 16) + #expect(SubtitleRectText.styledBody(fromASSEventLine: line)?.placement == nil) + } + + @Test("the body is unchanged by the derivation") + func bodyUnchanged() { + let parsed = SubtitleRectText.teletextBody(fromASSEventLine: page(firstTextRow: 16)) + #expect(parsed?.body.flattenedText == "CAPTION TEXT") + } + + // MARK: - Blank-line folding across runs (#107 hardening) + + @Test("a blank row split into its own run by a colour change still folds") + func blankRowSplitByColourChange() { + // The padding of an otherwise empty row carries the spacing attribute that changes colour, + // so the blank row can land in a run of its own. Folding per run, and across one run + // boundary, both miss that: the chain is line -> whitespace-only run -> line. + let line = #"0,0,Teletext,,0,0,0,,a\N{\c&H0000FF&}\h\h\N{\c&H00FF00&}b"# + let body = SubtitleRectText.teletextBody(fromASSEventLine: line)?.body + #expect(body?.flattenedText == "a\nb") + } + + @Test("interior blank rows still fold, single line breaks still survive") + func foldRegression() { + let folded = #"0,0,Teletext,,0,0,0,,line one\N \Nline two"# + #expect(SubtitleRectText.teletextBody(fromASSEventLine: folded)?.body.flattenedText + == "line one\nline two") + let kept = #"0,0,Teletext,,0,0,0,,line one\Nline two"# + #expect(SubtitleRectText.teletextBody(fromASSEventLine: kept)?.body.flattenedText + == "line one\nline two") + } + + // MARK: - Whitespace predicates + + @Test("the styled path trims the same characters the plain path does") + func unicodeWhitespaceEdgeTrim() { + // The plain path trims with .whitespacesAndNewlines (Unicode Zs); the styled path tested + // for literal space / tab / newline, so a NO-BREAK SPACE survived on a styled cue and not + // on an unstyled one. Teletext cannot produce one (IS_TXT_SPACE maps U+00A0 to a space, + // which decode_string writes as \h), but SRT, WebVTT and ASS payloads can. + let nbsp = "\u{00A0}" + let line = "0,0,Default,,0,0,0,,{\\c&HFFFFFF&}\(nbsp)\(nbsp)styled\(nbsp)" + let runs = SubtitleRectText.styledRuns(fromASSEventLine: line)?.runs ?? [] + #expect(runs.map(\.text).joined() == "styled") + + let plain = SubtitleRectText.plainText(fromASSEventLine: + "0,0,Default,,0,0,0,,\(nbsp)\(nbsp)plain\(nbsp)") + #expect(plain == "plain") + } +} + +private extension SubtitleCue.Body { + var flattenedText: String { + switch self { + case .text(let t): return t + case .richText(let runs): return runs.map(\.text).joined() + case .image: return "" + } + } +} From c3b935e89664f99be2bf7a4564a19a2b1e8e8615 Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Tue, 28 Jul 2026 20:07:48 +0200 Subject: [PATCH 3/4] docs: text subtitle placement, WebVTT settings and the teletext grid row Brings the subtitle section up to what 5.26.0 actually shipped (the text-codec bullet still described the pre-#233 behaviour, where override blocks were stripped) and adds the two paths placement now arrives on. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01At1agC2qU2Y2MX2BKLdT4N --- CHANGELOG.md | 10 ++++++++++ docs/formats.md | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a799ca..665f42c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,16 @@ the public-API contract. ## [Unreleased] +### Added + +- **WebVTT cue settings reach the host after all: `line`, `position` and `align` arrive as `SubtitleCue.placement`.** 5.26.0 reported them as upstream-blocked, which was a conclusion about the wrong layer. libavcodec's WebVTT decoder really does drop them, and says so as a `@todo` in `webvttdec.c`'s file header, so nothing about the placement is in the ASS event line it synthesises. The demuxer keeps them: `libavformat/webvttdec.c` attaches the verbatim settings string to every packet as `AV_PKT_DATA_WEBVTT_SETTINGS`, and `matroskadec.c` propagates the same side data for WebVTT in Matroska. Both subtitle decoders now read it, and the packet store carries the string through a rebuild, since side data does not live in the payload and a stored packet would otherwise lose the placement a freshly demuxed one has. A percentage `line` becomes an anchor point plus the alignment row, anchored to the frame edge it is nearer (the spec's default line alignment would pin the box top at `line:90%` and hang a two-line cue off the frame); a `line` number keeps only the half of the frame it names, because line boxes need a rendered line height the engine does not have. An ASS `\an` or `\pos` still wins, since that came from the payload itself. `size` and `vertical` have no equivalent in the placement model and are ignored, and a `position` without a `line` keeps only the alignment column, because an anchor point needs both axes. + +### Fixed + +- **Teletext captions keep their vertical placement on pages libzvbi does not flag as subtitle pages.** `gen_sub_ass` derives the vertical anchor from the grid row itself and emits it as `{\anN}`, which the engine has read since 5.26.0, but that whole block sits behind `is_subtitle_page`. That flag comes from the row-0 page header (NEWSFLASH clear, SUBTITLE set, SUPPRESS_HEADER set), and when a broadcaster does not set all three, or the header has not been seen yet, the else path writes the whole page instead: one `" \N"` per grid row with the empty ones included, no `\an`, no per-row trim. The ordinal of the first non-blank row is then the only carrier of the position, and the edge trim was removing it before anything could read it. That ordinal now feeds libzvbi's own third-of-the-page formula rather than an invented scale, so a caption the broadcaster moved to the top of frame to clear a lower-third graphic stays there. Deliberately coarse: three bands is what the source encodes, and mapping each row to its own offset makes consecutive cues of different heights sit at different heights, which reads as a blink. It never overrides an `\an` that arrived, including `{\an2}`, which is explicitly bottom and produces exactly the empty derived output that "no information" would. Reported by tresby (#233). +- **A styled cue trims the same whitespace an unstyled one does.** The plain path trimmed with `.whitespacesAndNewlines`, the styled path tested for a literal space, tab or newline, so a Unicode space (U+00A0 among them) survived on a styled cue and not on an unstyled one carrying the same text. Teletext cannot produce one, since libzvbi maps U+00A0 to a space and writes it as `\h`, but SRT, WebVTT and ASS payloads can. +- **A blank teletext row that a colour change split into its own run folds like any other.** The #107 interior blank-line fold worked per run plus one run boundary, which misses the shape the source produces most easily: the padding of an otherwise empty row carries the spacing attribute that changes colour, so the blank row lands in a whitespace-only run of its own and breaks the chain between the two text rows. The fold now runs on the flattened sequence and re-splits along the original run boundaries, so run structure cannot hide a blank row from it. + ## [5.26.0] - 2026-07-28 ([release notes](https://github.com/superuser404notfound/AetherEngine/releases/tag/5.26.0)) diff --git a/docs/formats.md b/docs/formats.md index 814cdea3..552bf9a4 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -84,7 +84,7 @@ Matroska CodecPrivate doesn't usually carry the pre-parsed `dec3` / `dac3` box c Subtitle cues come from one read: EVERY embedded subtitle stream (text and bitmap) stays in the session demuxer's keep-set and is harvested by a tap on the host's existing source read (each packet is observed then dropped, never muxed). Harvested packets are retained compressed in a per-session `SubtitlePacketStore` (300 s trailing window, byte-capped per stream) and a playhead-paced drainer decodes the selected stream near the playhead into the overlay. PGS streams in MPEG-TS (Blu-ray) arrive as one display set split across several PES packets (PCS|WDS|PDS|ODS|END, some without a PTS of their own); the store reassembles those chunks into one self-contained entry at the PCS presentation PTS before retention, so the drainer always decodes complete display sets (Matroska already carries one complete set per packet and is stored as-is), so enabling any embedded track is instant: the selection backfills synchronously from the store, with no positioning seek and no recovery machinery, even on remote disc images; the tap re-attaches with the producer across seeks and restarts. The tap's forward coverage ends at the producer's read position (its forward park, #102), which on direct play sits only a few seconds past the playhead; on VOD sessions a background subtitle-only side reader (the forward prefetcher, #151) therefore extends the store to the drainer's full 60 s lead window (raised to 270 s while a bitmap track's OCR rendition is armed, so the store covers the OCR worker's 240 s window), so `subtitleCues` holds cues ahead of the playhead for a host-applied ADVANCE sync offset, text and bitmap alike (live sessions skip it; content past the edge does not exist). It is best-effort: if it cannot open or wedges, the tap-fed behavior above is unchanged. Its positioning seek is anchored on the subtitle stream's own timestamp axis (#234): left to libavformat's default reference the anchor follows whichever stream is not fully discarded, so the pacing stream that gives the park its control point (#230) would move it onto video keyframes, and on Matroska that lands in a later cluster and drops a landing cue that starts further back than the destination. Sidecars are fetched once. Each packet decodes through `avcodec_decode_subtitle2` (except in-band CEA-608, which has an in-house line-21 decoder, see below), and the result lands in a single `[SubtitleCue]` published list: -- **Text codecs** (SubRip / ASS / SSA / WebVTT / mov_text) → `SubtitleCue.body = .text(String)`. ASS dialogue headers and override blocks (`{\an8}`, `{\b1}`, ...) are stripped; `\N` becomes a real newline so the host can render with regular text layout. DVB teletext subtitles (live broadcast) decode through libzvbi (`libzvbi_teletextdec`, configured to emit ASS so the per-character colour broadcasters use to distinguish speakers survives rather than being flattened) with page-state semantics (#107). A page carrying colour publishes as `SubtitleCue.body = .richText([SubtitleTextRun])` (each run an optional RGB `SubtitleColor`, nil meaning the host default); a page with no colour stays plain `.text`, and `cue.text` flattens either form for consumers that do not render colour. The decoded page defaults to libzvbi's auto-detected subtitle page, overridable per channel via `LoadOptions.teletextPage` (for example AU page 801, which libzvbi does not always flag as a subtitle page). libzvbi emits page content open-ended ("valid until replaced") and page erases as empty events, so every teletext event trims earlier open cues at its start; roll-up captions build and replace cleanly, an erase clears the line, and a 120 s cap bounds a ghost line if transmission stops without either. Validated end to end against Australian FTA broadcasts (1080i25 H.264, captions on page 801), where the interlaced video deinterlaces through the software path's bwdif filter. +- **Text codecs** (SubRip / ASS / SSA / WebVTT / mov_text) → `SubtitleCue.body = .text(String)`, or `.richText([SubtitleTextRun])` when the cue asks for styling (#233). libavcodec converts every text format into an ASS event line before the engine sees it, so one override parser serves all of them: inline `\b` / `\i` / `\u` / `\s` / `\c` / `\1c` / `\fn` / `\fs` / `\r` become attributes on the runs (`isBold`, `isItalic`, `isUnderlined`, `isStruckThrough`, `color`, `fontName`, `fontSize`), and a cue that asks for nothing stays plain `.text` with the exact string it produced before. `\N` becomes a real newline either way, and `cue.text` flattens both forms. Cue-level `\an` and `\pos` lift out into `SubtitleCue.placement` (ASS numpad alignment plus an optional `[0..1]`-normalised anchor, y from the top) instead of splitting a run. WebVTT cue settings reach the same field from packet side data (`AV_PKT_DATA_WEBVTT_SETTINGS`): the WebVTT decoder drops `line` / `position` / `align`, but the demuxer keeps them, so `line:10% align:start` arrives as an alignment and an anchor. `size` and `vertical` have no equivalent in the placement model and are ignored, and a `position` without a `line` keeps only the alignment column, since an anchor point needs both axes. DVB teletext subtitles (live broadcast) decode through libzvbi (`libzvbi_teletextdec`, configured to emit ASS so the per-character colour broadcasters use to distinguish speakers survives rather than being flattened) with page-state semantics (#107). A page carrying colour publishes as `SubtitleCue.body = .richText([SubtitleTextRun])` (each run an optional RGB `SubtitleColor`, nil meaning the host default); a page with no colour stays plain `.text`, and `cue.text` flattens either form for consumers that do not render colour. Teletext placement arrives two ways, because libzvbi writes it two ways: on a page flagged as a subtitle page it derives the vertical anchor from the grid row and emits `{\anN}`, which lands in `SubtitleCue.placement` like any other alignment; on a page it does not flag (broadcaster leaves NEWSFLASH / SUBTITLE / SUPPRESS_HEADER unset, or the row-0 header has not been seen yet) it writes the whole page instead, one row per line, and the ordinal of the first non-blank row is the only carrier of the position. The engine applies libzvbi's own third-of-the-page formula to that ordinal, so a caption the broadcaster moved to the top of frame stays there either way, and never overrides an `\an` that did arrive. The decoded page defaults to libzvbi's auto-detected subtitle page, overridable per channel via `LoadOptions.teletextPage` (for example AU page 801, which libzvbi does not always flag as a subtitle page). libzvbi emits page content open-ended ("valid until replaced") and page erases as empty events, so every teletext event trims earlier open cues at its start; roll-up captions build and replace cleanly, an erase clears the line, and a 120 s cap bounds a ghost line if transmission stops without either. Validated end to end against Australian FTA broadcasts (1080i25 H.264, captions on page 801), where the interlaced video deinterlaces through the software path's bwdif filter. - **Bitmap codecs** (PGS / HDMV PGS / DVB / DVD) → `.image(SubtitleImage)`. The indexed pixel plane is walked through its palette, premultiplied against alpha, and wrapped as a `CGImage`. Position is normalised in `[0..1]` against the composition canvas, whose coded pixel size rides along as `SubtitleImage.canvasSize` (#112): a cropped-video rip can author a canvas taller than the coded video, so hosts map the canvas width-aligned and center-anchored onto the on-screen video rect to land cues where the disc authored them; `.zero` means treat canvas == video. A display set carrying multiple composition objects (a forced sign plus dialogue is a common real-disc shape) fans into one cue per object, all sharing the set's start PTS, and every object is retained and rendered (#146); each object's `AV_SUBTITLE_FLAG_FORCED` rides along as `SubtitleImage.isForced`, surfaced per cue via `SubtitleCue.isForced` (track-level forcedness stays on `TrackInfo.isForced`). - **External files** (a separate `.srt` / `.ass` / `.vtt` URL) → register as first-class tracks (see below) or one-shot via `selectSidecarSubtitle(url:httpHeaders:)`, which opens its own short-lived `AVFormatContext`, decodes the whole file once, atomically swaps the result into `subtitleCues`. The fetch forwards the session's `LoadOptions.httpHeaders` by default (WebDAV auth and friends); pass the call's own `httpHeaders` to override per fetch. - **In-band CEA-608 closed captions** (`eia_608` / QuickTime `c608`, a demuxable caption track) → `.text(String)`. FFmpegBuild ships no `ccaption` decoder, so these never reach `avcodec_decode_subtitle2`. Instead a read-only tap on the segment producer's existing source connection reads the caption track's `cc_data` (its packets are kept in the demuxer's keep-set, observed, then dropped, never muxed, so the loopback-HLS output stays byte-identical), an in-house line-21 decoder (validated against FFmpeg's `ccaption_dec.c`) turns it into cues, and they publish on the same `subtitleCues` overlay path as every other codec. First cut: field-1 / channel CC1; CEA-708 (DTVCC) and field 2 are follow-ons. Captions carried only inside the video bitstream (ATSC A/53 `cc_data`, the US broadcast/cable case) are extracted too (#131): on the native remux path the segment producer scans H.264/HEVC video packets for `user_data_registered_itu_t_t35` SEI (GA94), reorders the decode-order groups to presentation order by the packet DTS watermark, and feeds the same line-21 decoder; on the software-decode path (MPEG-2 and friends) the triplets come from `AV_FRAME_DATA_A53_CC` decoded-frame side data. Since no caption AVStream exists, a synthetic `eia_608` track (id 99608) surfaces lazily on the first real (non-padding) caption pair, so uncaptioned channels never show a dead menu entry. Host-overlay only (no PiP / AirPlay), like the bitmap codecs. (#77) From 507f80da6680b01933b7156e0ab959234da890f4 Mon Sep 17 00:00:00 2001 From: Vincent Herbst Date: Tue, 28 Jul 2026 20:27:52 +0200 Subject: [PATCH 4/4] chore(deps): pin FFmpegBuild 2.3.0 for the webvtt demuxer Standalone .vtt files could not be opened at all: only the webvtt decoder was in the build, so avformat_open_input rejected the file with AVERROR_INVALIDDATA and an external WebVTT subtitle never loaded. 2.3.0 enables the demuxer, which is also what attaches the cue settings this branch reads. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01At1agC2qU2Y2MX2BKLdT4N --- Package.resolved | 6 +++--- Package.swift | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Package.resolved b/Package.resolved index f14480a7..cf544f41 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "15e2a2ab60b2c56e8de5ab631c52a26c4886a5492c8dda0e5d300e6453f08e00", + "originHash" : "3fd26f753e7b139739998c22e66193e5e6f0e77db1a2fe5bdb44dea41ad3f3c5", "pins" : [ { "identity" : "ffmpegbuild", "kind" : "remoteSourceControl", "location" : "https://github.com/superuser404notfound/FFmpegBuild", "state" : { - "revision" : "e832f22c89909297b7abe3a94c05b2816414b054", - "version" : "2.2.0" + "revision" : "ac70e2cc0cec574bad8029d901a9881b73ef161c", + "version" : "2.3.0" } }, { diff --git a/Package.swift b/Package.swift index ad7c8ce7..71c6ae82 100644 --- a/Package.swift +++ b/Package.swift @@ -29,7 +29,7 @@ let package = Package( // No network stack, we use custom AVIO + URLSession for HTTP streams. // Resolved over Git rather than a local path so consumers (and // Xcode Cloud) can build without a sibling FFmpegBuild checkout. - .package(url: "https://github.com/superuser404notfound/FFmpegBuild", from: "2.2.0"), // 2.2.0: matroska TTS warn-only per RFC 9559 (#145 rework); 2.1.3: sup demuxer (raw PGS sidecars); 2.1.2: matroska TrackTimestampScale clamp (#145, dropped in 2.2.0); 2.1.1: pgssubdec Epoch-Continue retain (#142); 2.1.0: yadif_videotoolbox + hwupload (Metal GPU deinterlace); 2.0.0: dynamic frameworks (LGPL), zvbi GPL excision + .package(url: "https://github.com/superuser404notfound/FFmpegBuild", from: "2.3.0"), // 2.3.0: webvtt demuxer (standalone .vtt sidecars, plus the cue settings as packet side data); 2.2.0: matroska TTS warn-only per RFC 9559 (#145 rework); 2.1.3: sup demuxer (raw PGS sidecars); 2.1.2: matroska TrackTimestampScale clamp (#145, dropped in 2.2.0); 2.1.1: pgssubdec Epoch-Continue retain (#142); 2.1.0: yadif_videotoolbox + hwupload (Metal GPU deinterlace); 2.0.0: dynamic frameworks (LGPL), zvbi GPL excision // Pure-Swift SMB2 client (MIT) that speaks the protocol over // NWConnection. Replaces AMSMB2/libsmb2, which EPERMs on tvOS/iOS. // Pinned to the 0.3.x minor: SMBClient is pre-1.0 with an actively