From 99bcc3ad7b9f06f6afc14e22a4f6d91dac4ba7df Mon Sep 17 00:00:00 2001 From: lmdev <138343225+lmdevv@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:17:07 -0400 Subject: [PATCH] api & web/captions: unify transcript downloads --- api/src/processing/helpers/caption-formats.js | 132 +++++++++++++++++- api/src/processing/helpers/captions.js | 116 +++++++++++---- .../processing/helpers/tests/captions.test.js | 72 +++++++++- api/src/processing/match-action.js | 1 + api/src/processing/services/loom.js | 8 +- api/src/processing/services/rutube.js | 25 ++-- api/src/processing/services/tiktok.js | 25 ++-- api/src/processing/services/twitter.js | 73 +++++----- api/src/processing/services/vimeo.js | 17 ++- api/src/processing/services/vk.js | 15 +- api/src/processing/services/youtube.js | 114 ++++++--------- api/src/stream/captions.js | 76 +++++----- api/src/stream/manage.js | 28 ++-- docs/api.md | 15 +- web/i18n/en/button.json | 3 +- web/i18n/en/error/api.json | 4 +- web/i18n/en/settings.json | 5 +- web/src/components/dialog/DialogButton.svelte | 4 +- .../components/dialog/DialogContainer.svelte | 2 +- web/src/components/dialog/DialogHolder.svelte | 2 +- .../components/dialog/NoScriptDialog.svelte | 2 +- web/src/lib/api/saving-handler.ts | 33 ++++- web/src/lib/download.ts | 3 +- web/src/lib/settings/validate.ts | 3 +- web/src/lib/state/dialogs.ts | 8 +- web/src/lib/types/settings.ts | 1 + web/src/lib/types/settings/v2.ts | 2 +- web/src/lib/types/settings/v7.ts | 5 +- web/src/routes/settings/metadata/+page.svelte | 8 +- 29 files changed, 548 insertions(+), 254 deletions(-) diff --git a/api/src/processing/helpers/caption-formats.js b/api/src/processing/helpers/caption-formats.js index a753c5b0..f471bf63 100644 --- a/api/src/processing/helpers/caption-formats.js +++ b/api/src/processing/helpers/caption-formats.js @@ -85,17 +85,134 @@ export const parseSrt = input => { return cues; }; +const timestampSeconds = timestamp => timestamp.split(":").reduce( + (seconds, part) => seconds * 60 + Number(part), 0 +); + +const vttTimestamp = seconds => { + const milliseconds = Math.round(seconds * 1000); + if (!Number.isFinite(milliseconds) || milliseconds < 0) { + throw new Error("invalid caption timestamp"); + } + const hours = Math.floor(milliseconds / 3600000); + const minutes = Math.floor(milliseconds / 60000) % 60; + const remaining = milliseconds % 60000; + return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:` + + `${(remaining / 1000).toFixed(3).padStart(6, "0")}`; +}; + +// HLS repeats boundary cues and may use a different local clock per segment. +// Keep the first segment's timeline while applying subsequent timestamp maps. +// https://www.rfc-editor.org/rfc/rfc8216#section-3.5 +export const mergeCaptionSegments = segments => { + const blocks = new Set(); + const metadata = new Set(); + const clockWrap = 2 ** 33; + let firstOffset, previousMpeg; + let adjustment; + let wraps = 0; + + for (const segment of segments) { + const normalized = segment.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + const hasHeader = /^WEBVTT(?:\s|$)/.test(normalized); + if (hasHeader) { + const header = normalized.split(/\n{2,}/)[0]; + const map = header.match(/^X-TIMESTAMP-MAP=(.+)$/m)?.[1]; + const local = map?.match(/LOCAL:([\d:.]+)/)?.[1]; + const mpeg = map?.match(/MPEGTS:(\d+)/)?.[1]; + if (map && (!local || !mpeg)) throw new Error("invalid caption timestamp map"); + const timestamp = Number(mpeg); + if (map && previousMpeg !== undefined + && timestamp - previousMpeg < -clockWrap / 2) wraps++; + if (map) previousMpeg = timestamp; + const offset = map + ? (timestamp + wraps * clockWrap) / 90000 - timestampSeconds(local) + : 0; + firstOffset ??= offset; + adjustment = offset - firstOffset; + } else if (adjustment === undefined) { + throw new Error("invalid VTT segment"); + } + + const segmentBlocks = normalized.split(/\n{2,}/).slice(hasHeader ? 1 : 0); + for (const block of segmentBlocks) { + if (/^(STYLE|REGION)(?:\s|$)/.test(block)) { + metadata.add(block.trimEnd()); + continue; + } + if (/^NOTE(?:\s|$)/.test(block)) continue; + const lines = block.trimEnd().split("\n"); + const timingIndex = lines.findIndex(line => line.includes(" --> ")); + if (timingIndex < 0) continue; + const timing = lines[timingIndex].match(/^(\S+)\s+-->\s+(\S+)(.*)$/); + if (!timing) throw new Error("invalid caption cue"); + const start = vttTimestamp(timestampSeconds(timing[1]) + adjustment); + const end = vttTimestamp(timestampSeconds(timing[2]) + adjustment); + // Identifiers can change across segments; timing and payload identify a cue. + blocks.add([ + `${start} --> ${end}${timing[3]}`, + ...lines.slice(timingIndex + 1), + ].join("\n")); + } + } + return `WEBVTT\n\n${[...metadata, ...blocks].join("\n\n")}\n`; +}; + +const collapseRollingCues = cues => { + const result = []; + let run = [], rows = [], rolling = false; + const flush = () => { + if (rolling) { + result.push({ ...run[0], end: run.at(-1).end, text: rows.join("\n") }); + } else { + for (const cue of run) result.push(cue); + } + run = []; + rows = []; + rolling = false; + }; + + for (const cue of cues) { + const nextRows = cue.text.split("\n"); + const previous = run.at(-1); + let shared = 0; + if (previous && timestampSeconds(cue.start) >= timestampSeconds(previous.start) + && timestampSeconds(cue.start) <= timestampSeconds(previous.end)) { + const previousRows = previous.text.split("\n"); + for (let count = Math.min(previousRows.length, nextRows.length); count > 0; count--) { + if (previousRows.slice(-count).every((row, index) => row === nextRows[index])) { + shared = count; + break; + } + } + // A changing multiline window distinguishes rolling text from repeated speech. + if (shared && previous.text !== cue.text + && (previousRows.length > 1 || nextRows.length > 1)) rolling = true; + } + if (!shared) flush(); + for (const row of nextRows.slice(shared)) rows.push(row); + run.push(cue); + } + flush(); + return result; +}; + const transcriptLines = cues => { const lines = []; - for (const cue of cues) { + let previous; + for (const cue of collapseRollingCues(cues)) { const text = cue.text.replace(/\s*\n\s*/g, " ").replace(/\s+/g, " ").trim(); - if (!text || lines.at(-1) === text) continue; + if (!text) continue; - if (lines.length && text.startsWith(`${lines.at(-1)} `)) { + const overlaps = previous + && timestampSeconds(cue.start) >= timestampSeconds(previous.start) + && timestampSeconds(cue.start) < timestampSeconds(previous.end); + if (overlaps && (text === lines.at(-1) || text.startsWith(`${lines.at(-1)} `))) { lines[lines.length - 1] = text; } else { lines.push(text); } + previous = cue; } return lines; }; @@ -108,17 +225,18 @@ const srtTimestamp = timestamp => { const escapeMarkdown = value => value.replace(/[\\`*_{}\[\]<>#+.!|~-]/g, "\\$&"); export const convertCaptions = (input, format, metadata = {}, sourceFormat = "vtt") => { + if (!["vtt", "srt"].includes(sourceFormat)) throw new Error("unsupported caption source format"); + const cues = sourceFormat === "srt" ? parseSrt(input) : parseVtt(input); + if (!cues.length) throw new Error("caption file has no cues"); + if (sourceFormat === "vtt" && format === "vtt") { return input.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); } - const cues = sourceFormat === "srt" ? parseSrt(input) : parseVtt(input); - if (!cues.length) throw new Error("caption file has no cues"); - if (format === "vtt") { return "WEBVTT\n\n" + cues.map(cue => [ `${cue.start} --> ${cue.end}`, - cue.text, + cue.text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"), ].join("\n")).join("\n\n") + "\n"; } diff --git a/api/src/processing/helpers/captions.js b/api/src/processing/helpers/captions.js index 370429cc..306a2adc 100644 --- a/api/src/processing/helpers/captions.js +++ b/api/src/processing/helpers/captions.js @@ -1,12 +1,50 @@ -const normalizedLanguage = language => language?.toLowerCase().replace("_", "-"); +import { convertLanguageCode } from "../../misc/language-codes.js"; -export const matchesCaptionLanguage = (language, requested) => { - if (!requested || !language) return true; +// Services use both ISO 639-1 and ISO 639-2, sometimes with a script or region. +export const normalizeCaptionLanguage = language => { + if (!language) return; + const [base, ...parts] = language.replaceAll("_", "-").split("-"); + const normalized = [ + base.length === 3 ? convertLanguageCode(base) || base : base, + ...parts, + ].join("-"); + try { + return new Intl.Locale(normalized).toString(); + } catch { + return normalized.toLowerCase(); + } +}; - const normalized = normalizedLanguage(language); - const normalizedRequested = normalizedLanguage(requested); - return normalized === normalizedRequested - || normalized?.split("-")[0] === normalizedRequested?.split("-")[0]; +const languageRank = (language, requested) => { + if (!requested) return 0; + if (!language) return Infinity; + if (language === requested) return 0; + + try { + const track = new Intl.Locale(language); + const target = new Intl.Locale(requested); + if (track.language !== target.language) return Infinity; + // Bare languages accept any script. Explicit scripts or regions must + // not fall back to a different writing system, e.g. zh-Hant -> zh-Hans. + if ((target.script || target.region) + && track.maximize().script !== target.maximize().script) return Infinity; + return 1; + } catch { + return Infinity; + } +}; + +export const captionSelectionError = tracks => { + const available = tracks.filter(track => track.url); + if (!available.length) return { error: "captions.unavailable" }; + return { + error: "captions.language_unavailable", + context: { + languages: [...new Set(available.map(track => + normalizeCaptionLanguage(track.language) + ).filter(Boolean))], + }, + }; }; export const createCaptionResponse = ({ @@ -45,24 +83,50 @@ export const createCaptionResponse = ({ }; }; -export const selectCaptionTrack = (tracks, subtitleLang, allowAutomatic = false) => { - const allowed = tracks.filter(track => allowAutomatic || track.kind !== "asr"); - if (!allowed.length) return; - - const requested = normalizedLanguage(subtitleLang); - if (!requested) { - return allowed.find(track => track.kind !== "asr") || allowed[0]; +const captionResource = (resource, playlistURL) => { + const output = { + url: new URL(resource.uri, playlistURL).toString(), + }; + if (resource.byterange) { + const { offset, length } = resource.byterange; + output.headers = { range: `bytes=${offset}-${offset + length - 1}` }; } - - const matching = allowed.filter(track => { - const language = normalizedLanguage(track.language_code); - const baseLanguage = language?.split("-")[0]; - const requestedBase = requested.split("-")[0]; - return language === requested - || language?.startsWith(`${requested}-`) - || requested.startsWith(`${language}-`) - || (allowAutomatic && baseLanguage === requestedBase); - }); - - return matching.find(track => track.kind !== "asr") || matching[0]; + return output; +}; + +export const captionSegmentResources = (segments, playlistURL) => { + const resources = []; + let previousMap; + + for (const segment of segments) { + if (segment.map) { + const map = captionResource(segment.map, playlistURL); + const mapKey = `${map.url}:${map.headers?.range || ""}`; + if (mapKey !== previousMap) { + resources.push(map); + previousMap = mapKey; + } + } else { + previousMap = undefined; + } + resources.push(captionResource(segment, playlistURL)); + } + return resources; +}; + +// Tracks share { url, language, automatic, sourceFormat }; adapters may add fields. +// Preserve source order when language rank and manual/automatic status are equal. +export const selectCaptionTrack = (tracks, language, allowAutomatic = true) => { + const requested = normalizeCaptionLanguage(language); + let selected, bestRank = Infinity; + for (const track of tracks) { + if (!track.url || (!allowAutomatic && track.automatic)) continue; + const rank = languageRank(normalizeCaptionLanguage(track.language), requested) * 2 + + Number(!!track.automatic); + if (rank < bestRank) { + selected = track; + bestRank = rank; + } + } + return selected; }; diff --git a/api/src/processing/helpers/tests/captions.test.js b/api/src/processing/helpers/tests/captions.test.js index 1ff32e88..d5ed9ff6 100644 --- a/api/src/processing/helpers/tests/captions.test.js +++ b/api/src/processing/helpers/tests/captions.test.js @@ -1,8 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { selectCaptionTrack } from "../captions.js"; -import { convertCaptions, parseSrt, parseVtt } from "../caption-formats.js"; +import { captionSegmentResources, selectCaptionTrack } from "../captions.js"; +import { convertCaptions, mergeCaptionSegments, parseSrt, parseVtt } from "../caption-formats.js"; import { apiSchema } from "../../schema.js"; const sample = `\uFEFFWEBVTT\r @@ -52,7 +52,7 @@ test("SRT input converts to transcripts and VTT", () => { test("plain text collapses growing automatic-caption cues", () => { assert.equal( - convertCaptions(sample, "txt"), + convertCaptions(sample.replace("00:02.500 -->", "00:01.500 -->"), "txt"), "Hello & welcome back\n\nFinal line\n" ); }); @@ -83,14 +83,14 @@ test("VTT output is passed through with normalized newlines", () => { test("caption selection prefers manual tracks and matches base languages", () => { const tracks = [ - { language_code: "en-US", kind: "asr" }, - { language_code: "en-GB" }, - { language_code: "fr" }, + { url: "auto", language: "en-US", automatic: true }, + { url: "manual", language: "en-GB" }, + { url: "french", language: "fr" }, ]; assert.equal(selectCaptionTrack(tracks, "en", true), tracks[1]); assert.equal(selectCaptionTrack(tracks, "en-AU", true), tracks[1]); - assert.equal(selectCaptionTrack(tracks, "en-US", false), undefined); + assert.equal(selectCaptionTrack(tracks, "en-US", false), tracks[1]); assert.equal(selectCaptionTrack(tracks, undefined, true), tracks[1]); assert.equal(selectCaptionTrack(tracks, "de", true), undefined); }); @@ -116,3 +116,61 @@ test("caption API requests default to plain text and reject unknown formats", () captionFormat: "json", }).success, false); }); + +test("adjoining rolling windows collapse but separate repeated speech survives", () => { + const input = "WEBVTT\n\n" + + "00:00.000 --> 00:00.010\nA new game\n\n" + + "00:00.010 --> 00:02.000\nA new game\n\n" + + "00:02.000 --> 00:04.000\nA new game\nwith a large world\n\n" + + "00:04.000 --> 00:06.000\nwith a large world\n\n" + + "00:10.000 --> 00:11.000\nNo.\n\n" + + "00:11.000 --> 00:12.000\nNo.\n"; + assert.equal(convertCaptions(input, "txt"), "A new game with a large world\n\nNo.\n\nNo.\n"); + assert.equal(convertCaptions(input, "md"), "# Transcript\n\nA new game with a large world\n\nNo\\.\n\nNo\\.\n"); + assert.equal(parseVtt(convertCaptions(input, "vtt")).length, 6); + assert.equal(parseSrt(convertCaptions(input, "srt")).length, 6); +}); + +test("HLS merging preserves cue settings and markup and removes repeated boundary cues", () => { + const cue = "00:00:09.000 --> 00:00:12.000 align:start\nHello & welcome"; + const merged = mergeCaptionSegments([ + `WEBVTT\n\nSTYLE\n::cue { color: white; }\n\nfirst-id\n${cue}\n`, + `WEBVTT\n\nnew-id\n${cue}\n\n00:00:12.000 --> 00:00:14.000\nSecond segment\n`, + "WEBVTT\n", + ]); + assert.equal(parseVtt(merged).length, 2); + assert.ok(merged.includes(cue)); + assert.ok(merged.includes("STYLE\n::cue { color: white; }")); + assert.equal(convertCaptions(merged, "txt"), "Hello & welcome\n\nSecond segment\n"); +}); + +test("HLS merging accepts media fragments after an initialization section", () => { + const merged = mergeCaptionSegments([ + "WEBVTT\nX-TIMESTAMP-MAP=LOCAL:00:00:00.000,MPEGTS:900000\n", + "00:00:00.000 --> 00:00:02.000\nFirst fragment\n", + "00:00:02.000 --> 00:00:04.000\nSecond fragment\n", + ]); + assert.equal(convertCaptions(merged, "txt"), "First fragment\n\nSecond fragment\n"); +}); + +test("HLS resources include changed initialization sections and byte ranges", () => { + const map = { uri: "captions.vtt", byterange: { offset: 0, length: 100 } }; + const resources = captionSegmentResources([ + { uri: "captions.vtt", byterange: { offset: 100, length: 50 }, map }, + { uri: "captions.vtt", byterange: { offset: 150, length: 60 }, map }, + { + uri: "captions.vtt", + byterange: { offset: 310, length: 70 }, + map: { uri: "captions.vtt", byterange: { offset: 210, length: 100 } }, + }, + ], "https://example.com/path/playlist.m3u8"); + + assert.deepEqual(resources.map(resource => resource.headers.range), [ + "bytes=0-99", + "bytes=100-149", + "bytes=150-209", + "bytes=210-309", + "bytes=310-379", + ]); + assert.ok(resources.every(resource => resource.url === "https://example.com/path/captions.vtt")); +}); diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 9331a009..6120cfa5 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -35,6 +35,7 @@ export default function({ originalRequest: r.originalRequest, subtitles: r.subtitles, captionFormat: r.captionFormat, + captionSourceFormat: r.captionSourceFormat, captionMetadata: !disableMetadata ? r.captionMetadata : {}, cover: !disableMetadata ? r.cover : false, cropCover: !disableMetadata ? r.cropCover : false, diff --git a/api/src/processing/services/loom.js b/api/src/processing/services/loom.js index 47272a17..a949ad12 100644 --- a/api/src/processing/services/loom.js +++ b/api/src/processing/services/loom.js @@ -1,7 +1,8 @@ import { genericUserAgent } from "../../config.js"; import { createCaptionResponse, - matchesCaptionLanguage, + selectCaptionTrack, + captionSelectionError, } from "../helpers/captions.js"; const craftHeaders = id => ({ @@ -102,8 +103,9 @@ export default async function({ }) { if (isCaptionOnly) { const transcript = await getTranscript(id); - if (!transcript || !matchesCaptionLanguage(transcript.language, captionLanguage)) { - return { error: "fetch.empty" }; + const tracks = transcript ? [transcript] : []; + if (!selectCaptionTrack(tracks, captionLanguage)) { + return captionSelectionError(tracks); } return createCaptionResponse({ diff --git a/api/src/processing/services/rutube.js b/api/src/processing/services/rutube.js index 96139ab4..5b69a1a4 100644 --- a/api/src/processing/services/rutube.js +++ b/api/src/processing/services/rutube.js @@ -2,7 +2,8 @@ import HLS from "hls-parser"; import { env } from "../../config.js"; import { createCaptionResponse, - matchesCaptionLanguage, + selectCaptionTrack, + captionSelectionError, } from "../helpers/captions.js"; async function requestJSON(url) { @@ -16,7 +17,7 @@ const delta = (a, b) => Math.abs(a - b); export default async function(obj) { if (obj.yappyId) { - if (obj.isCaptionOnly) return { error: "fetch.empty" }; + if (obj.isCaptionOnly) return captionSelectionError([]); const yappy = await requestJSON( `https://rutube.ru/pangolin/api/web/yappy/v4/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=1` @@ -47,17 +48,21 @@ export default async function(obj) { if (play.live_streams?.hls) return { error: "content.video.live" }; if (obj.isCaptionOnly) { - const caption = play.captions?.find(caption => - ["webvtt", "srt"].includes(caption.format) - && matchesCaptionLanguage(caption.code, obj.captionLanguage) - ); - if (!caption) return { error: "fetch.empty" }; + const tracks = (play.captions || []) + .filter(caption => ["webvtt", "srt"].includes(caption.format)) + .map(caption => ({ + url: caption.file, + language: caption.code, + sourceFormat: caption.format === "webvtt" ? "vtt" : "srt", + })); + const caption = selectCaptionTrack(tracks, obj.captionLanguage); + if (!caption) return captionSelectionError(tracks); return createCaptionResponse({ - url: caption.file, + url: caption.url, format: obj.captionFormat, - sourceFormat: caption.format === "webvtt" ? "vtt" : caption.format, - language: caption.code, + sourceFormat: caption.sourceFormat, + language: caption.language, service: "rutube", id: obj.id, title: play.title?.trim(), diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 01a21bb4..67541b80 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -7,7 +7,9 @@ import { createStream } from "../../stream/manage.js"; import { convertLanguageCode } from "../../misc/language-codes.js"; import { createCaptionResponse, - matchesCaptionLanguage, + selectCaptionTrack, + captionSelectionError, + normalizeCaptionLanguage, } from "../helpers/captions.js"; const shortDomain = "https://vt.tiktok.com/"; @@ -77,20 +79,19 @@ export default async function(obj) { } if (obj.isCaptionOnly) { - const requestedLanguage = convertLanguageCode(obj.captionLanguage) - || obj.captionLanguage; - const subtitle = detail.video?.subtitleInfos?.find(subtitle => - subtitle.Format === "webvtt" - && matchesCaptionLanguage(subtitle.LanguageCodeName, requestedLanguage) - ); - if (!subtitle) return { error: "fetch.empty" }; + const tracks = (detail.video?.subtitleInfos || []) + .filter(subtitle => subtitle.Format === "webvtt") + .map(subtitle => ({ + url: subtitle.Url, + language: normalizeCaptionLanguage(subtitle.LanguageCodeName), + })); + const subtitle = selectCaptionTrack(tracks, obj.captionLanguage); + if (!subtitle) return captionSelectionError(tracks); - const language = convertLanguageCode(subtitle.LanguageCodeName) - || subtitle.LanguageCodeName; return createCaptionResponse({ - url: subtitle.Url, + url: subtitle.url, format: obj.captionFormat, - language, + language: subtitle.language, service: "tiktok", id: postId, title: detail.desc?.trim(), diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index bdb0373a..883d842e 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -3,8 +3,10 @@ import { genericUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; import { + captionSegmentResources, createCaptionResponse, - matchesCaptionLanguage, + selectCaptionTrack, + captionSelectionError, } from "../helpers/captions.js"; const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail'; @@ -274,54 +276,54 @@ export default async function({ url, filename, }); - const extractSubtitles = async (hlsUrl, requestedLanguage) => { + const getSubtitleTracks = async hlsUrl => { const mainHls = await fetch(hlsUrl).then(r => r.text()).catch(() => {}); - if (!mainHls) return; + if (!mainHls) return []; + return (HLS.parse(mainHls)?.variants?.[0]?.subtitles || []).map(subtitle => ({ + url: new URL(subtitle.uri, hlsUrl).toString(), + language: subtitle.language, + })); + }; - const subtitle = HLS.parse(mainHls)?.variants[0]?.subtitles?.find( - subtitle => matchesCaptionLanguage(subtitle.language, requestedLanguage) - ); + const resolveSubtitle = async (subtitle, allSegments = false) => { if (!subtitle) return; - - const subtitleUrl = new URL(subtitle.uri, hlsUrl).toString(); - const subtitleHls = await fetch(subtitleUrl).then(r => r.text()); + const subtitleHls = await fetch(subtitle.url).then(r => r.text()).catch(() => {}); if (!subtitleHls) return; - const finalSubtitlePath = HLS.parse(subtitleHls)?.segments?.[0].uri; - if (!finalSubtitlePath) return; - - const finalSubtitleUrl = new URL(finalSubtitlePath, subtitleUrl).toString(); - + const segments = HLS.parse(subtitleHls)?.segments; + if (!segments?.length) return; + const urls = allSegments + ? captionSegmentResources(segments, subtitle.url) + : new URL(segments[0].uri, subtitle.url).toString(); return { - url: finalSubtitleUrl, + url: urls, language: subtitle.language, }; - } + }; if (isCaptionOnly) { + const tracks = []; for (const mediaItem of media) { if (mediaItem.type !== "video") continue; - const hlsVariant = mediaItem.video_info?.variants?.find( variant => variant.content_type === "application/x-mpegURL" ); - if (!hlsVariant) continue; - - const subtitles = await extractSubtitles(hlsVariant.url, captionLanguage); - if (!subtitles) continue; - - return createCaptionResponse({ - url: subtitles.url, - format: captionFormat, - language: subtitles.language, - service: "twitter", - id, - source: `https://x.com/i/status/${id}`, - headers: { "user-agent": genericUserAgent }, - }); + if (hlsVariant) tracks.push(...await getSubtitleTracks(hlsVariant.url)); } + const selected = selectCaptionTrack(tracks, captionLanguage); + if (!selected) return captionSelectionError(tracks); + const subtitles = await resolveSubtitle(selected, true); + if (!subtitles) return { error: "captions.unavailable" }; - return { error: "fetch.empty" }; + return createCaptionResponse({ + url: subtitles.url, + format: captionFormat, + language: subtitles.language, + service: "twitter", + id, + source: `https://x.com/i/status/${id}`, + headers: { "user-agent": genericUserAgent }, + }); } switch (media?.length) { @@ -348,10 +350,9 @@ export default async function({ v => v.content_type === "application/x-mpegURL" ); if (hlsVariant) { - const { url, language } = await extractSubtitles( - hlsVariant.url, - subtitleLang - ) || {}; + const { url, language } = await resolveSubtitle(selectCaptionTrack( + await getSubtitleTracks(hlsVariant.url), subtitleLang + )) || {}; subtitles = url; if (language) fileMetadata = { sublanguage: language }; } diff --git a/api/src/processing/services/vimeo.js b/api/src/processing/services/vimeo.js index a3b54495..493b74cf 100644 --- a/api/src/processing/services/vimeo.js +++ b/api/src/processing/services/vimeo.js @@ -4,7 +4,8 @@ import { merge } from '../../misc/utils.js'; import { getCookie } from "../cookie/manager.js"; import { createCaptionResponse, - matchesCaptionLanguage, + selectCaptionTrack, + captionSelectionError, } from "../helpers/captions.js"; const resolutionMatch = { @@ -200,16 +201,18 @@ const getHLS = async (configURL, obj) => { export default async function(obj) { if (obj.isCaptionOnly) { const config = await getPlayerConfig(obj.id, obj.password); - const track = config?.request?.text_tracks?.find(track => - track.url - && matchesCaptionLanguage(track.lang, obj.captionLanguage) - ); - if (!track) return { error: "fetch.empty" }; + if (!config) return { error: "fetch.fail" }; + const tracks = (config.request?.text_tracks || []).map(track => ({ + url: track.url, + language: track.lang, + })); + const track = selectCaptionTrack(tracks, obj.captionLanguage); + if (!track) return captionSelectionError(tracks); return createCaptionResponse({ url: new URL(track.url, "https://player.vimeo.com/").toString(), format: obj.captionFormat, - language: track.lang, + language: track.language, service: "vimeo", id: obj.id, title: config.video?.title, diff --git a/api/src/processing/services/vk.js b/api/src/processing/services/vk.js index 381498ad..2bb90808 100644 --- a/api/src/processing/services/vk.js +++ b/api/src/processing/services/vk.js @@ -1,7 +1,8 @@ import { env } from "../../config.js"; import { createCaptionResponse, - matchesCaptionLanguage, + selectCaptionTrack, + captionSelectionError, } from "../helpers/captions.js"; const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"]; @@ -117,17 +118,17 @@ export default async function ({ } if (isCaptionOnly) { - const subtitle = video.subtitles?.find(subtitle => - subtitle.title?.endsWith(".vtt") - && matchesCaptionLanguage(subtitle.lang, captionLanguage) - ); - if (!subtitle) return { error: "fetch.empty" }; + const tracks = (video.subtitles || []) + .filter(subtitle => subtitle.title?.endsWith(".vtt")) + .map(subtitle => ({ url: subtitle.url, language: subtitle.lang })); + const subtitle = selectCaptionTrack(tracks, captionLanguage); + if (!subtitle) return captionSelectionError(tracks); const id = `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`; return createCaptionResponse({ url: subtitle.url, format: captionFormat, - language: subtitle.lang, + language: subtitle.language, service: "vk", id, title: video.title?.trim(), diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 2f7a8a8a..7db3566f 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -7,7 +7,12 @@ import { Innertube, Platform, Session } from "youtubei.js"; import { env } from "../../config.js"; import { getCookie } from "../cookie/manager.js"; import { getYouTubeSession } from "../helpers/youtube-session.js"; -import { selectCaptionTrack } from "../helpers/captions.js"; +import { + captionSegmentResources, + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; // https://github.com/LuanRT/YouTube.js/pull/1052 Platform.shim.eval = async (data) => { @@ -136,21 +141,14 @@ const getHlsVariants = async (hlsManifest, dispatcher) => { return variants; } -const normalizedLanguage = language => language?.toLowerCase().replace("_", "-"); +const captionTracks = info => (info.captions?.caption_tracks || []).map(track => ({ + url: track.base_url, + language: track.language_code, + automatic: track.kind === "asr", +})); -const getSubtitles = async ( - info, - dispatcher, - subtitleLang, - allowAutomatic = false -) => { - const preferredCap = selectCaptionTrack( - info.captions.caption_tracks, - subtitleLang, - allowAutomatic - ); - - const captionsUrl = preferredCap?.base_url; +const getSubtitles = async (info, dispatcher, preferredCap, allSegments = false) => { + const captionsUrl = preferredCap?.url; if (!captionsUrl) return; if (!captionsUrl.includes("exp=xpe")) { @@ -159,7 +157,7 @@ const getSubtitles = async ( return { url: url.toString(), - language: preferredCap.language_code, + language: preferredCap.language, } } @@ -168,7 +166,7 @@ const getSubtitles = async ( // so instead we just use subtitles from HLS const hlsVariants = await getHlsVariants( - info.streaming_data.hls_manifest_url, + info.streaming_data?.hls_manifest_url, dispatcher ); if (hlsVariants?.error) return; @@ -177,23 +175,28 @@ const getSubtitles = async ( const hlsSubtitles = hlsVariants[0]?.subtitles; if (!hlsSubtitles?.length) return; - const preferredHls = hlsSubtitles.find(subtitle => - normalizedLanguage(subtitle.language) - ?.startsWith(normalizedLanguage(preferredCap.language_code)) + const preferredHls = selectCaptionTrack( + hlsSubtitles.map(subtitle => ({ ...subtitle, url: subtitle.uri })), + preferredCap.language ); if (!preferredHls) return; + const playlistURL = new URL(preferredHls.uri, info.streaming_data.hls_manifest_url).toString(); const fetchedHlsSubs = - await fetch(preferredHls.uri, { dispatcher }) + await fetch(playlistURL, { dispatcher }) .then(r => r.status === 200 ? r.text() : undefined) .catch(() => {}); + if (!fetchedHlsSubs) return; const parsedSubs = HLS.parse(fetchedHlsSubs); - if (!parsedSubs) return; + if (!parsedSubs?.segments?.length) return; + const urls = allSegments + ? captionSegmentResources(parsedSubs.segments, playlistURL) + : new URL(parsedSubs.segments[0].uri, playlistURL).toString(); return { - url: parsedSubs.segments[0]?.uri, + url: urls, language: preferredHls.language, } } @@ -349,54 +352,23 @@ export default async function (o) { } if (o.isCaptionOnly) { - const captionTracks = info.captions?.caption_tracks || []; - if (!captionTracks.length) { - return { error: "youtube.captions_unavailable" }; - } + const tracks = captionTracks(info); + const selectedTrack = selectCaptionTrack(tracks, o.captionLanguage || o.subtitleLang); + if (!selectedTrack) return captionSelectionError(tracks); - const selectedTrack = selectCaptionTrack( - captionTracks, - o.captionLanguage || o.subtitleLang, - true - ); - if (!selectedTrack) { - return { - error: "youtube.caption_language_unavailable", - context: { - languages: [...new Set(captionTracks.map(track => track.language_code))], - } - }; - } + const captions = await getSubtitles(info, o.dispatcher, selectedTrack, true); + if (!captions?.url) return { error: "captions.unavailable" }; - const captions = await getSubtitles( - info, - o.dispatcher, - selectedTrack.language_code, - true - ); - if (!captions?.url) { - return { error: "youtube.captions_unavailable" }; - } - - return { - type: "captions", - isCaptionOnly: true, - urls: captions.url, - captionFormat: o.captionFormat, - captionLanguage: captions.language, - filenameAttributes: { - service: "youtube", - id: o.id, - title: basicInfo.title.trim(), - author: basicInfo.author.replace("- Topic", "").trim(), - }, - captionMetadata: { - title: basicInfo.title.trim(), - author: basicInfo.author.replace("- Topic", "").trim(), - language: captions.language, - source: `https://www.youtube.com/watch?v=${o.id}`, - }, - }; + return createCaptionResponse({ + url: captions.url, + format: o.captionFormat, + language: captions.language, + service: "youtube", + id: o.id, + title: basicInfo.title.trim(), + author: basicInfo.author.replace("- Topic", "").trim(), + source: `https://www.youtube.com/watch?v=${o.id}`, + }); } const normalizeQuality = res => { @@ -566,7 +538,9 @@ export default async function (o) { } if (o.subtitleLang && !o.isAudioOnly && info.captions?.caption_tracks?.length) { - const videoSubtitles = await getSubtitles(info, o.dispatcher, o.subtitleLang); + const videoSubtitles = await getSubtitles( + info, o.dispatcher, selectCaptionTrack(captionTracks(info), o.subtitleLang, false) + ); if (videoSubtitles) { subtitles = videoSubtitles; } diff --git a/api/src/stream/captions.js b/api/src/stream/captions.js index a137985c..10412d46 100644 --- a/api/src/stream/captions.js +++ b/api/src/stream/captions.js @@ -3,7 +3,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { closeRequest, closeResponse } from "./shared.js"; import { destroyInternalStream } from "./manage.js"; -import { convertCaptions } from "../processing/helpers/caption-formats.js"; +import { convertCaptions, mergeCaptionSegments } from "../processing/helpers/caption-formats.js"; const defaultAgent = new Agent(); const maxCaptionSize = 8 * 1024 * 1024; @@ -17,43 +17,50 @@ const contentTypes = { export default async function captions(streamInfo, res) { const abortController = new AbortController(); - const shutdown = () => ( - closeRequest(abortController), - closeResponse(res), - destroyInternalStream(streamInfo.urls) - ); + const urls = [streamInfo.urls].flat(); + const cleanup = () => { + closeRequest(abortController); + urls.forEach(destroyInternalStream); + }; + res.once("close", cleanup); + res.once("error", cleanup); try { - const response = await request(streamInfo.urls, { - signal: abortController.signal, - maxRedirections: 4, - dispatcher: defaultAgent, - headers: streamInfo.headers, - }); - - if (response.statusCode < 200 || response.statusCode > 299) { - return shutdown(); - } - - const declaredSize = Number(response.headers["content-length"]); - if (declaredSize > maxCaptionSize) { - res.status(413).end(); - return shutdown(); - } - - const chunks = []; + if (res.destroyed) return; + const segments = []; let size = 0; - for await (const chunk of response.body) { - size += chunk.length; - if (size > maxCaptionSize) { - res.status(413).end(); - return shutdown(); + for (const url of urls) { + const response = await request(url, { + signal: abortController.signal, + maxRedirections: 4, + dispatcher: defaultAgent, + headers: streamInfo.headers, + }); + + if (response.statusCode < 200 || response.statusCode > 299) { + return closeResponse(res); } - chunks.push(chunk); + + const declaredSize = Number(response.headers["content-length"]); + if (size + declaredSize > maxCaptionSize) { + res.status(413).end(); + return; + } + + const chunks = []; + for await (const chunk of response.body) { + size += chunk.length; + if (size > maxCaptionSize) { + res.status(413).end(); + return; + } + chunks.push(chunk); + } + segments.push(Buffer.concat(chunks).toString("utf8")); } const body = convertCaptions( - Buffer.concat(chunks).toString("utf8"), + segments.length === 1 ? segments[0] : mergeCaptionSegments(segments), streamInfo.captionFormat, streamInfo.captionMetadata, streamInfo.captionSourceFormat @@ -64,8 +71,11 @@ export default async function captions(streamInfo, res) { res.setHeader("Content-Type", contentTypes[streamInfo.captionFormat]); res.setHeader("Content-Length", Buffer.byteLength(body)); res.status(200).send(body); - destroyInternalStream(streamInfo.urls); } catch { - shutdown(); + if (!res.destroyed) closeResponse(res); + } finally { + cleanup(); + res.off("close", cleanup); + res.off("error", cleanup); } } diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index b38cb439..4e1bd74f 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -47,6 +47,7 @@ export function createStream(obj) { subtitles: obj.subtitles, captionFormat: obj.captionFormat, + captionSourceFormat: obj.captionSourceFormat, captionMetadata: obj.captionMetadata, }; @@ -262,21 +263,32 @@ const transplantTunnel = async function (dispatcher) { function wrapStream(streamInfo) { const url = streamInfo.urls; + const createWrappedStream = resource => { + if (typeof resource === "string") { + return createInternalStream(resource, streamInfo, streamInfo.type === "captions"); + } + if (streamInfo.type !== "captions" || typeof resource?.url !== "string") { + throw "invalid url"; + } + return createInternalStream( + resource.url, + { + ...streamInfo, + headers: { ...streamInfo.headers, ...resource.headers }, + }, + true + ); + }; + if (streamInfo.originalRequest) { streamInfo.transplant = transplantTunnel.bind(streamInfo); } if (typeof url === 'string') { - streamInfo.urls = createInternalStream( - url, - streamInfo, - streamInfo.type === "captions" - ); + streamInfo.urls = createWrappedStream(url); } else if (Array.isArray(url)) { for (const idx in streamInfo.urls) { - streamInfo.urls[idx] = createInternalStream( - streamInfo.urls[idx], streamInfo - ); + streamInfo.urls[idx] = createWrappedStream(streamInfo.urls[idx]); } } else throw 'invalid urls'; diff --git a/docs/api.md b/docs/api.md index 067bc45a..3900a4eb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -102,10 +102,17 @@ downloading video or audio. supported services are listed in the [api readme](/api/README.md#supported-services). `captionLanguage` selects the caption language, falling back to `subtitleLang` when -omitted. if both are omitted, cobalt prefers the first manually provided track and -falls back to an automatic caption track. when a language is provided, cobalt matches -the full language code first and then its base language. manually provided captions -take priority over automatic captions. +omitted. if both are omitted, cobalt chooses an available track. when a service +identifies automatic captions, manually provided tracks take priority. + +when a language is provided, an exact match takes priority over compatible regional +variants, with manual captions preferred within each tier. explicitly requested +writing systems are preserved, so `zh-Hant` will not fall back to `zh-Hans`. + +if captions exist but none match, the api returns +`error.api.captions.language_unavailable`. omit both language fields to retry with an +available track. if no downloadable tracks exist, the error is +`error.api.captions.unavailable`. `txt` and `md` produce readable transcripts without timestamps. `vtt` preserves the original webvtt cues, while `srt` converts those cues to the subrip format. diff --git a/web/i18n/en/button.json b/web/i18n/en/button.json index 71d4365f..5be8adea 100644 --- a/web/i18n/en/button.json +++ b/web/i18n/en/button.json @@ -26,5 +26,6 @@ "clear_cache": "clear cache", "remove": "remove", "retry": "retry", - "delete": "delete" + "delete": "delete", + "download.transcript_any_language": "download anyway" } diff --git a/web/i18n/en/error/api.json b/web/i18n/en/error/api.json index c2e38960..5329500a 100644 --- a/web/i18n/en/error/api.json +++ b/web/i18n/en/error/api.json @@ -62,5 +62,7 @@ "youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!", "youtube.no_session_tokens": "couldn't get required session tokens for youtube. this may be caused by a restriction on youtube's side. try again in a few seconds, but if this issue sticks, please report it!", "youtube.captions_unavailable": "this video doesn't have any captions i can download.", - "youtube.caption_language_unavailable": "this video doesn't have captions in the selected language. available languages: {{ languages }}." + "youtube.caption_language_unavailable": "this video doesn't have captions in the selected language. available languages: {{ languages }}.", + "captions.unavailable": "this video doesn't have any captions i can download.", + "captions.language_unavailable": "this video doesn't have captions in your preferred language. you can download an available transcript instead." } diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index 5a263dac..6f3be3fe 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -77,7 +77,7 @@ "captions.method": "transcript saving method", "captions.method.description": "how to save transcripts from the save button. \"ask\" shows both options every time.", "captions.language": "preferred transcript language", - "captions.language.description": "cobalt will pick the transcript in this language when the video has multiple caption tracks. otherwise it falls back to the video's default track.", + "captions.language.description": "pick a preferred transcript language, or let cobalt choose an available track automatically. if your preferred language isn't available, cobalt will ask before downloading another language.", "audio.youtube.better_audio": "youtube audio quality", "audio.youtube.better_audio.title": "prefer better quality", @@ -168,5 +168,6 @@ "tabs": "navigation", "tabs.hide_remux": "hide the remux tab", - "tabs.hide_remux.description": "if you don't use the remux tool, you can hide it from the navigation bar." + "tabs.hide_remux.description": "if you don't use the remux tool, you can hide it from the navigation bar.", + "captions.language.automatic": "automatic" } diff --git a/web/src/components/dialog/DialogButton.svelte b/web/src/components/dialog/DialogButton.svelte index 69aaf722..f1aa3203 100644 --- a/web/src/components/dialog/DialogButton.svelte +++ b/web/src/components/dialog/DialogButton.svelte @@ -55,7 +55,9 @@ .popup-button { width: 100%; - height: 40px; + min-height: 40px; + height: auto; + padding-block: 8px; transition: 0.2s opacity; } diff --git a/web/src/components/dialog/DialogContainer.svelte b/web/src/components/dialog/DialogContainer.svelte index dc1bfb76..bb1a8fd2 100644 --- a/web/src/components/dialog/DialogContainer.svelte +++ b/web/src/components/dialog/DialogContainer.svelte @@ -22,7 +22,7 @@ // check if dialog parent is still present if (dialogParent) { dialogParent.close(); - killDialog(); + killDialog(id); } }, 150); } diff --git a/web/src/components/dialog/DialogHolder.svelte b/web/src/components/dialog/DialogHolder.svelte index b519340d..363e8018 100644 --- a/web/src/components/dialog/DialogHolder.svelte +++ b/web/src/components/dialog/DialogHolder.svelte @@ -15,7 +15,7 @@ -->
- {#each $dialogs as dialog} + {#each $dialogs as dialog (dialog.id)} {#if dialog.type === "small"} {:else if dialog.type === "picker"} diff --git a/web/src/components/dialog/NoScriptDialog.svelte b/web/src/components/dialog/NoScriptDialog.svelte index c9ba9c2e..45ef1595 100644 --- a/web/src/components/dialog/NoScriptDialog.svelte +++ b/web/src/components/dialog/NoScriptDialog.svelte @@ -4,7 +4,7 @@ {#if !browser} -