From 184414f42ee24f15cbdf2cdaf0db348094e74e10 Mon Sep 17 00:00:00 2001 From: Christian Benincasa Date: Thu, 14 May 2026 20:00:24 -0400 Subject: [PATCH] feat: integrate new stream selector with the ffmpeg pipeline --- server/src/ffmpeg/FFmpegModule.ts | 4 +- server/src/ffmpeg/FfmpegStreamFactory.ts | 29 +- .../ffmpeg/StreamSelectionEvaluator.test.ts | 375 ++++++++++++++++++ server/src/ffmpeg/StreamSelectionEvaluator.ts | 57 ++- server/src/ffmpeg/StreamSelector.ts | 4 + .../StreamSelectionProfileResolver.ts | 47 ++- types/src/schemas/streamSelectionSchema.ts | 2 + 7 files changed, 479 insertions(+), 39 deletions(-) diff --git a/server/src/ffmpeg/FFmpegModule.ts b/server/src/ffmpeg/FFmpegModule.ts index 18b9541ab..f8271849f 100644 --- a/server/src/ffmpeg/FFmpegModule.ts +++ b/server/src/ffmpeg/FFmpegModule.ts @@ -5,6 +5,7 @@ import { ContainerModule } from 'inversify'; import type { ChannelOrm } from '../db/schema/Channel.ts'; import { bindAssistedFactory } from '../util/assistedInject.ts'; import { FfmpegInfo } from './ffmpegInfo.ts'; +import { StreamSelector } from './StreamSelector.ts'; export type FFmpegAssistedFactory = ( transcodeConfig: TranscodeConfigOrm, @@ -18,7 +19,8 @@ const FFmpegModule = new ContainerModule(({ bind }) => { FfmpegStreamFactory, ); - bind(FfmpegInfo).toSelf(); + bind(FfmpegInfo).toSelf().inSingletonScope(); + bind(StreamSelector).toSelf().inSingletonScope(); }); export { FFmpegModule }; diff --git a/server/src/ffmpeg/FfmpegStreamFactory.ts b/server/src/ffmpeg/FfmpegStreamFactory.ts index 663f04b46..7ce6e6cbf 100644 --- a/server/src/ffmpeg/FfmpegStreamFactory.ts +++ b/server/src/ffmpeg/FfmpegStreamFactory.ts @@ -30,7 +30,6 @@ import { isUndefined } from 'lodash-es'; import type { DeepReadonly } from 'ts-essentials'; import { match, P } from 'ts-pattern'; import type { ContentBackedStreamLineupItem } from '../db/derived_types/StreamLineup.ts'; -import type { IChannelDB } from '../db/interfaces/IChannelDB.ts'; import { FeatureFlagService } from '../services/FeatureFlagService.ts'; import { isImageBasedSubtitle } from '../stream/util.ts'; import { KEYS } from '../types/inject.ts'; @@ -42,7 +41,6 @@ import { FfmpegPlaybackParamsCalculator } from './FfmpegPlaybackParamsCalculator import { FfmpegProcess } from './FfmpegProcess.ts'; import { FfmpegTranscodeSession } from './FfmpegTrancodeSession.ts'; import { StreamSelector } from './StreamSelector.ts'; -import { SubtitleStreamPicker } from './SubtitleStreamPicker.ts'; import { AudioStream, EmbeddedSubtitleStream, @@ -110,7 +108,6 @@ export class FfmpegStreamFactory { @injected(KEYS.SettingsDB) private settingsDB: ISettingsDB, @injected(KEYS.PipelineBuilderFactory) private pipelineBuilderFactory: PipelineBuilderFactory, - @injected(KEYS.ChannelDB) private channelDB: IChannelDB, @injected(FeatureFlagService) private featureFlagService: FeatureFlagService, @injected(StreamSelector) private streamSelector: StreamSelector, @@ -624,21 +621,21 @@ export class FfmpegStreamFactory { let subtitleSource: Nullable = null; let subtitleRendition: SubtitleRenditionInfo | undefined; + // In passthrough mode, only sidecar (Convert) is available since we're + // not re-encoding video for burn-in. Need audioDetails because + // selectAudioAndSubtitleStreams requires them. if ( - isDefined(streamDetails.subtitleDetails) && - this.channel.subtitlesEnabled + isDefined(streamDetails.audioDetails) && + isDefined(streamDetails.subtitleDetails) ) { - const subtitlePreferences = - await this.channelDB.getChannelSubtitlePreferences(this.channel.uuid); - - const pickedSubtitleStream = await SubtitleStreamPicker.pickSubtitles( - subtitlePreferences, - lineupItem, - streamDetails.subtitleDetails, - // In passthrough mode, always prefer text-based subs for sidecar - // since burn-in is not available. - { preferTextBased: true }, - ); + const { subtitleStream: pickedSubtitleStream } = + await this.streamSelector.selectAudioAndSubtitleStreams({ + channel: this.channel, + lineupItem, + audioStreams: streamDetails.audioDetails, + subtitleStreams: streamDetails.subtitleDetails, + hints: { preferTextBased: true }, + }); if (pickedSubtitleStream) { this.logger.trace('Using subtitle stream: %O', pickedSubtitleStream); diff --git a/server/src/ffmpeg/StreamSelectionEvaluator.test.ts b/server/src/ffmpeg/StreamSelectionEvaluator.test.ts index 1ac511f7b..4f23a968e 100644 --- a/server/src/ffmpeg/StreamSelectionEvaluator.test.ts +++ b/server/src/ffmpeg/StreamSelectionEvaluator.test.ts @@ -1327,4 +1327,379 @@ describe('evaluateStreamSelectionProfile', () => { expect(result.subtitleStream).toBeNull(); }); }); + + describe('preferTextBased', () => { + it('without preferTextBased, by_language picks first matching stream regardless of codec type', async () => { + // Image-based sub appears first in stream order, text-based second. + // Without preferTextBased, the image-based sub should be picked + // because it comes first and allowImageBased is true. + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + languageCodeISO6392: 'eng', + codec: 'hdmv_pgs_subtitle', + type: 'embedded', + }), + makeSubtitleStream({ + index: 3, + languageCodeISO6392: 'eng', + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'by_language', + languages: ['eng'], + filterType: 'any', + allowImageBased: true, + allowExternal: true, + // preferTextBased not set (defaults to false) + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + expect(result.subtitleStream).not.toBeNull(); + // Image-based sub is first and should be selected + expect(result.subtitleStream!.index).toBe(2); + expect(result.subtitleStream!.codec).toBe('hdmv_pgs_subtitle'); + }); + + it('without preferTextBased, embedded text sub goes through extraction', async () => { + // The mock adds path: '/fake/path.vtt' via getSubtitleDetailsWithExtractedPath. + // Without preferTextBased, embedded text subs must go through that extraction + // step, so the returned stream should have the mock's path property. + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + languageCodeISO6392: 'eng', + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'by_language', + languages: ['eng'], + filterType: 'any', + allowImageBased: true, + allowExternal: true, + // preferTextBased not set (defaults to false) + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + expect(result.subtitleStream).not.toBeNull(); + // Extraction mock adds path — confirms the extraction path was taken + expect((result.subtitleStream as Record)['path']).toBe( + '/fake/path.vtt', + ); + }); + + it('without preferTextBased, default embedded text sub goes through extraction', async () => { + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + default: true, + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { type: 'default' }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + expect(result.subtitleStream).not.toBeNull(); + expect((result.subtitleStream as Record)['path']).toBe( + '/fake/path.vtt', + ); + }); + + it('by_language with action preferTextBased selects text-based sub over image-based', async () => { + // Image-based sub appears first in stream order, text-based second. + // With preferTextBased, text-based should be selected. + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + languageCodeISO6392: 'eng', + codec: 'hdmv_pgs_subtitle', + type: 'embedded', + }), + makeSubtitleStream({ + index: 3, + languageCodeISO6392: 'eng', + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'by_language', + languages: ['eng'], + filterType: 'any', + allowImageBased: true, + allowExternal: true, + preferTextBased: true, + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + expect(result.subtitleStream).not.toBeNull(); + expect(result.subtitleStream!.index).toBe(3); + expect(result.subtitleStream!.codec).toBe('srt'); + }); + + it('by_language with runtime hint preferTextBased selects text-based sub over image-based', async () => { + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + languageCodeISO6392: 'eng', + codec: 'hdmv_pgs_subtitle', + type: 'embedded', + }), + makeSubtitleStream({ + index: 3, + languageCodeISO6392: 'eng', + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'by_language', + languages: ['eng'], + filterType: 'any', + allowImageBased: true, + allowExternal: true, + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + { preferTextBased: true }, + ); + + expect(result.subtitleStream).not.toBeNull(); + expect(result.subtitleStream!.index).toBe(3); + expect(result.subtitleStream!.codec).toBe('srt'); + }); + + it('by_language with preferTextBased returns embedded text sub without extraction', async () => { + // When preferTextBased is active, embedded text-based subs should be + // returned directly (no call to getSubtitleDetailsWithExtractedPath), + // meaning they won't have the fake path added by the mock. + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + languageCodeISO6392: 'eng', + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'by_language', + languages: ['eng'], + filterType: 'any', + allowImageBased: true, + allowExternal: true, + preferTextBased: true, + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + expect(result.subtitleStream).not.toBeNull(); + // The mock adds path: '/fake/path.vtt' — since preferTextBased bypasses + // extraction, the original stream is returned without that property. + expect(result.subtitleStream!.index).toBe(2); + expect( + (result.subtitleStream as Record)['path'], + ).toBeUndefined(); + }); + + it('default with preferTextBased returns embedded text sub without extraction', async () => { + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + default: true, + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'default', + preferTextBased: true, + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + expect(result.subtitleStream).not.toBeNull(); + expect(result.subtitleStream!.index).toBe(2); + expect( + (result.subtitleStream as Record)['path'], + ).toBeUndefined(); + }); + + it('runtime hint overrides action preferTextBased=false', async () => { + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + languageCodeISO6392: 'eng', + codec: 'hdmv_pgs_subtitle', + type: 'embedded', + }), + makeSubtitleStream({ + index: 3, + languageCodeISO6392: 'eng', + codec: 'srt', + type: 'embedded', + }), + ]; + // Action has preferTextBased: false (default) + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'by_language', + languages: ['eng'], + filterType: 'any', + allowImageBased: true, + allowExternal: true, + preferTextBased: false, + }, + }), + ]); + const celService = makeCelService(true); + + // Runtime hint overrides to true + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + { preferTextBased: true }, + ); + + expect(result.subtitleStream).not.toBeNull(); + expect(result.subtitleStream!.index).toBe(3); + expect(result.subtitleStream!.codec).toBe('srt'); + }); + + it('default with preferTextBased sorts text-based before image-based for default selection', async () => { + const subs: SubtitleStreamDetails[] = [ + makeSubtitleStream({ + index: 2, + default: true, + codec: 'hdmv_pgs_subtitle', + type: 'embedded', + }), + makeSubtitleStream({ + index: 3, + default: true, + codec: 'srt', + type: 'embedded', + }), + ]; + const profile = makeProfile([ + makeRule({ + subtitleAction: { + type: 'default', + preferTextBased: true, + }, + }), + ]); + const celService = makeCelService(true); + + const result = await evaluateStreamSelectionProfile( + profile, + audioStreams, + subs, + celService, + celContext, + lineupItem, + ); + + // Text-based default should be found first due to sorting + expect(result.subtitleStream).not.toBeNull(); + expect(result.subtitleStream!.index).toBe(3); + expect(result.subtitleStream!.codec).toBe('srt'); + }); + }); }); diff --git a/server/src/ffmpeg/StreamSelectionEvaluator.ts b/server/src/ffmpeg/StreamSelectionEvaluator.ts index a6ffa30f7..f6996cbac 100644 --- a/server/src/ffmpeg/StreamSelectionEvaluator.ts +++ b/server/src/ffmpeg/StreamSelectionEvaluator.ts @@ -84,6 +84,10 @@ export function buildCelContext( }; } +export type StreamSelectionHints = { + preferTextBased?: boolean; +}; + export async function evaluateStreamSelectionProfile( profile: StreamSelectionProfile, audioStreams: NonEmptyArray, @@ -91,6 +95,7 @@ export async function evaluateStreamSelectionProfile( celService: CelEvaluationService, celContext: StreamSelectionCelContext, lineupItem: ContentBackedStreamLineupItem, + hints?: StreamSelectionHints, ): Promise { for (const rule of profile.rules) { const conditionResult = celService.evaluate(rule.condition, celContext); @@ -105,6 +110,7 @@ export async function evaluateStreamSelectionProfile( rule.subtitleAction, subtitleStreams, lineupItem, + hints, ); return { audioStream, subtitleStream }; } @@ -178,6 +184,7 @@ async function resolveSubtitleAction( action: SubtitleAction, subtitleStreams: SubtitleStreamDetails[] | undefined, lineupItem: ContentBackedStreamLineupItem, + hints?: StreamSelectionHints, ): Promise { switch (action.type) { case 'disable': @@ -187,10 +194,34 @@ async function resolveSubtitleAction( if (!subtitleStreams || subtitleStreams.length === 0) { return null; } - const defaultStream = subtitleStreams.find((s) => s.default); + + const effectivePreferText = + hints?.preferTextBased || action.preferTextBased; + + const candidates = [...subtitleStreams]; + if (effectivePreferText) { + candidates.sort((a, b) => { + const aImage = isImageBasedSubtitle(a.codec) ? 1 : 0; + const bImage = isImageBasedSubtitle(b.codec) ? 1 : 0; + return aImage - bImage; + }); + } + + const defaultStream = candidates.find((s) => s.default); if (!defaultStream) { return null; } + + // When preferTextBased is active and the default is an embedded text-based + // sub, return it directly — the caller will extract via the pipeline. + if ( + effectivePreferText && + defaultStream.type === 'embedded' && + !isImageBasedSubtitle(defaultStream.codec) + ) { + return defaultStream; + } + const extracted = await SubtitleStreamPicker.getSubtitleDetailsWithExtractedPath( lineupItem, @@ -221,9 +252,21 @@ async function resolveSubtitleAction( return null; } + const effectivePreferText = + hints?.preferTextBased || action.preferTextBased; + + const candidates = [...subtitleStreams]; + if (effectivePreferText) { + candidates.sort((a, b) => { + const aImage = isImageBasedSubtitle(a.codec) ? 1 : 0; + const bImage = isImageBasedSubtitle(b.codec) ? 1 : 0; + return aImage - bImage; + }); + } + for (const lang of action.languages) { const langLower = lang.toLowerCase(); - for (const stream of subtitleStreams) { + for (const stream of candidates) { // Language match if ( stream.languageCodeISO6392?.toLowerCase() !== langLower && @@ -251,6 +294,16 @@ async function resolveSubtitleAction( continue; } + // When preferTextBased is active and we have an embedded text-based + // sub, return it directly — the caller will extract via the pipeline. + if ( + effectivePreferText && + !isImageBasedSubtitle(stream.codec) && + stream.type === 'embedded' + ) { + return stream; + } + // For embedded text-based subs, verify extraction if ( !isImageBasedSubtitle(stream.codec) && diff --git a/server/src/ffmpeg/StreamSelector.ts b/server/src/ffmpeg/StreamSelector.ts index 56009d268..3d9939c00 100644 --- a/server/src/ffmpeg/StreamSelector.ts +++ b/server/src/ffmpeg/StreamSelector.ts @@ -13,12 +13,14 @@ import { buildCelContext, evaluateStreamSelectionProfile, } from './StreamSelectionEvaluator.ts'; +import type { StreamSelectionHints } from './StreamSelectionEvaluator.ts'; type StreamSelectRequest = { channel: ChannelOrm; lineupItem: ContentBackedStreamLineupItem; audioStreams: NonEmptyArray; subtitleStreams: Array; + hints?: StreamSelectionHints; }; @injectable() @@ -34,6 +36,7 @@ export class StreamSelector { lineupItem, audioStreams, subtitleStreams, + hints, }: StreamSelectRequest) { const selectionCtx = { channelId: channel.uuid, @@ -61,6 +64,7 @@ export class StreamSelector { this.celService, celContext, lineupItem, + hints, ); } } diff --git a/server/src/services/StreamSelectionProfileResolver.ts b/server/src/services/StreamSelectionProfileResolver.ts index a33df23dc..345a4b21e 100644 --- a/server/src/services/StreamSelectionProfileResolver.ts +++ b/server/src/services/StreamSelectionProfileResolver.ts @@ -168,29 +168,36 @@ export class StreamSelectionProfileResolver { } : { type: 'default' as const }; - // Build subtitle action from channel subtitle preferences - const subtitlePrefs = - await this.channelDB.getChannelSubtitlePreferences(channelId); + // Check if the channel has subtitles enabled + const channel = await this.channelDB.getChannel(channelId); let subtitleAction: StreamSelectionRule['subtitleAction']; - // Filter out preferences with filterType 'none' — they mean - // "don't match subtitles for this language", matching the - // behavior of SubtitleStreamPicker.pickSubtitles. - const activeSubtitlePrefs = subtitlePrefs.filter( - (p) => p.filterType !== 'none', - ); - if (activeSubtitlePrefs.length > 0) { - const sorted = orderBy(activeSubtitlePrefs, 'priority', 'asc'); - const topPref = sorted[0]!; - subtitleAction = { - type: 'by_language' as const, - languages: sorted.map((p) => p.languageCode), - filterType: topPref.filterType ?? 'any', - allowImageBased: Boolean(topPref.allowImageBased ?? true), - allowExternal: Boolean(topPref.allowExternal ?? true), - }; + if (!channel?.subtitlesEnabled) { + subtitleAction = { type: 'disable' as const }; } else { - subtitleAction = { type: 'default' as const }; + // Build subtitle action from channel subtitle preferences + const subtitlePrefs = + await this.channelDB.getChannelSubtitlePreferences(channelId); + + // Filter out preferences with filterType 'none' — they mean + // "don't match subtitles for this language", matching the + // legacy SubtitleStreamPicker behavior. + const activeSubtitlePrefs = subtitlePrefs.filter( + (p) => p.filterType !== 'none', + ); + if (activeSubtitlePrefs.length > 0) { + const sorted = orderBy(activeSubtitlePrefs, 'priority', 'asc'); + const topPref = sorted[0]!; + subtitleAction = { + type: 'by_language' as const, + languages: sorted.map((p) => p.languageCode), + filterType: topPref.filterType ?? 'any', + allowImageBased: Boolean(topPref.allowImageBased ?? true), + allowExternal: Boolean(topPref.allowExternal ?? true), + }; + } else { + subtitleAction = { type: 'default' as const }; + } } rules.push({ diff --git a/types/src/schemas/streamSelectionSchema.ts b/types/src/schemas/streamSelectionSchema.ts index 90faa7a41..c550355bb 100644 --- a/types/src/schemas/streamSelectionSchema.ts +++ b/types/src/schemas/streamSelectionSchema.ts @@ -36,10 +36,12 @@ export const SubtitleActionByLanguageSchema = z.object({ filterType: SubtitleFilterSchema.default('any'), allowImageBased: z.boolean().default(true), allowExternal: z.boolean().default(true), + preferTextBased: z.boolean().default(false), }); export const SubtitleActionDefaultSchema = z.object({ type: z.literal('default'), + preferTextBased: z.boolean().default(false), }); export const SubtitleActionSchema = z.discriminatedUnion('type', [