From d63deb94acad5eb2b233bddc57bc60efc43d3989 Mon Sep 17 00:00:00 2001 From: lmdev <138343225+lmdevv@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:39:05 -0400 Subject: [PATCH] api/youtube: add standalone caption downloads --- .github/test.sh | 1 + api/package.json | 1 + api/src/processing/helpers/caption-formats.js | 117 ++++++++++++++++++ api/src/processing/helpers/captions.js | 23 ++++ .../processing/helpers/tests/captions.test.js | 97 +++++++++++++++ api/src/processing/match-action.js | 38 +++++- api/src/processing/match.js | 16 ++- api/src/processing/request.js | 9 ++ api/src/processing/schema.js | 12 +- api/src/processing/services/youtube.js | 76 ++++++++++-- api/src/stream/captions.js | 69 +++++++++++ api/src/stream/manage.js | 9 +- api/src/stream/stream.js | 4 + api/src/util/tests/youtube.json | 16 ++- 14 files changed, 473 insertions(+), 15 deletions(-) create mode 100644 api/src/processing/helpers/caption-formats.js create mode 100644 api/src/processing/helpers/captions.js create mode 100644 api/src/processing/helpers/tests/captions.test.js create mode 100644 api/src/stream/captions.js diff --git a/.github/test.sh b/.github/test.sh index 90f7f19c..6f788f03 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -12,6 +12,7 @@ waitport() { } test_api() { + pnpm run --prefix api test:unit waitport 3000 curl -m 3 http://localhost:3000/ API_RESPONSE=$(curl -m 10 http://localhost:3000/ \ diff --git a/api/package.json b/api/package.json index 3166ba79..84de764b 100644 --- a/api/package.json +++ b/api/package.json @@ -11,6 +11,7 @@ "scripts": { "start": "node src/cobalt", "test": "node src/util/test", + "test:unit": "node --test src/processing/helpers/tests/*.test.js", "token:jwt": "node src/util/generate-jwt-secret" }, "repository": { diff --git a/api/src/processing/helpers/caption-formats.js b/api/src/processing/helpers/caption-formats.js new file mode 100644 index 00000000..ee80787e --- /dev/null +++ b/api/src/processing/helpers/caption-formats.js @@ -0,0 +1,117 @@ +const namedEntities = { + amp: "&", + apos: "'", + gt: ">", + lt: "<", + nbsp: " ", + quot: '"', +}; + +const decodeEntities = text => text.replace( + /&(#(?:x[0-9a-f]+|\d+)|[a-z]+);/gi, + (entity, value) => { + if (value[0] !== "#") { + return namedEntities[value.toLowerCase()] ?? entity; + } + + const hex = value[1]?.toLowerCase() === "x"; + const codePoint = Number.parseInt(value.slice(hex ? 2 : 1), hex ? 16 : 10); + try { + return String.fromCodePoint(codePoint); + } catch { + return entity; + } + } +); + +const cleanCueText = text => decodeEntities( + text + .replace(/<\d{2}:\d{2}(?::\d{2})?\.\d{3}>/g, "") + .replace(//gi, "\n") + .replace(/<[^>]+>/g, "") +).split("\n") + .map(line => line.trim()) + .filter(Boolean) + .join("\n"); + +export const parseVtt = input => { + const normalized = input.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + const blocks = normalized.split(/\n{2,}/); + const cues = []; + + for (const block of blocks) { + const lines = block.split("\n").map(line => line.trimEnd()); + if (!lines.length || /^(WEBVTT|NOTE|STYLE|REGION)(?:\s|$)/.test(lines[0])) { + continue; + } + + const timingIndex = lines.findIndex(line => line.includes(" --> ")); + if (timingIndex < 0) continue; + + const timing = lines[timingIndex].match( + /^(\d{2}:)?\d{2}:\d{2}\.\d{3}\s+-->\s+(\d{2}:)?\d{2}:\d{2}\.\d{3}/ + )?.[0]; + if (!timing) continue; + + const [ start, end ] = timing.split(/\s+-->\s+/); + const text = cleanCueText(lines.slice(timingIndex + 1).join("\n")); + if (text) cues.push({ start, end, text }); + } + + return cues; +}; + +const transcriptLines = cues => { + const lines = []; + for (const cue of cues) { + const text = cue.text.replace(/\s*\n\s*/g, " ").replace(/\s+/g, " ").trim(); + if (!text || lines.at(-1) === text) continue; + + if (lines.length && text.startsWith(`${lines.at(-1)} `)) { + lines[lines.length - 1] = text; + } else { + lines.push(text); + } + } + return lines; +}; + +const srtTimestamp = timestamp => { + const withHours = timestamp.split(":").length === 2 ? `00:${timestamp}` : timestamp; + return withHours.replace(".", ","); +}; + +const escapeMarkdown = value => value.replace(/[\\`*_{}\[\]<>#+.!|~-]/g, "\\$&"); + +export const convertCaptions = (vtt, format, metadata = {}) => { + if (format === "vtt") { + return vtt.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + } + + const cues = parseVtt(vtt); + if (!cues.length) throw new Error("caption file has no cues"); + + if (format === "srt") { + return cues.map((cue, index) => [ + index + 1, + `${srtTimestamp(cue.start)} --> ${srtTimestamp(cue.end)}`, + cue.text, + ].join("\n")).join("\n\n") + "\n"; + } + + const transcript = transcriptLines(cues); + if (format === "txt") return transcript.join("\n\n") + "\n"; + + if (format === "md") { + const heading = `# ${escapeMarkdown(metadata.title || "Transcript")}`; + const details = [ + metadata.author && `author: ${escapeMarkdown(metadata.author)}`, + metadata.language && `language: ${escapeMarkdown(metadata.language)}`, + metadata.source && `source: <${metadata.source}>`, + ].filter(Boolean); + + return [ heading, ...details, ...transcript.map(escapeMarkdown) ].join("\n\n") + "\n"; + } + + throw new Error("unsupported caption format"); +}; diff --git a/api/src/processing/helpers/captions.js b/api/src/processing/helpers/captions.js new file mode 100644 index 00000000..3e3307cd --- /dev/null +++ b/api/src/processing/helpers/captions.js @@ -0,0 +1,23 @@ +const normalizedLanguage = language => language?.toLowerCase().replace("_", "-"); + +export const selectCaptionTrack = (tracks, subtitleLang, allowAutomatic = false) => { + const allowed = tracks.filter(track => allowAutomatic || track.kind !== "asr"); + if (!allowed.length) return; + + const requested = normalizedLanguage(subtitleLang); + if (!requested) { + return allowed.find(track => track.kind !== "asr") || allowed[0]; + } + + const matching = allowed.filter(track => { + const language = normalizedLanguage(track.language_code); + const baseLanguage = language?.split("-")[0]; + const requestedBase = requested.split("-")[0]; + return language === requested + || language?.startsWith(`${requested}-`) + || requested.startsWith(`${language}-`) + || (allowAutomatic && baseLanguage === requestedBase); + }); + + return matching.find(track => track.kind !== "asr") || matching[0]; +}; diff --git a/api/src/processing/helpers/tests/captions.test.js b/api/src/processing/helpers/tests/captions.test.js new file mode 100644 index 00000000..65c62bda --- /dev/null +++ b/api/src/processing/helpers/tests/captions.test.js @@ -0,0 +1,97 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { selectCaptionTrack } from "../captions.js"; +import { convertCaptions, parseVtt } from "../caption-formats.js"; +import { apiSchema } from "../../schema.js"; + +const sample = `\uFEFFWEBVTT\r +\r +NOTE generated fixture\r +ignored\r +\r +first cue\r +00:00.000 --> 00:02.500 align:start position:0%\r +Hello & welcome\r +\r +00:02.500 --> 00:04.000\r +Hello & welcome back\r +\r +00:04.000 --> 00:06.250\r +Final line\r +`; + +test("parseVtt extracts and cleans cues", () => { + assert.deepEqual(parseVtt(sample), [ + { start: "00:00.000", end: "00:02.500", text: "Hello & welcome" }, + { start: "00:02.500", end: "00:04.000", text: "Hello & welcome back" }, + { start: "00:04.000", end: "00:06.250", text: "Final line" }, + ]); +}); + +test("plain text collapses growing automatic-caption cues", () => { + assert.equal( + convertCaptions(sample, "txt"), + "Hello & welcome back\n\nFinal line\n" + ); +}); + +test("SRT output numbers cues and converts timestamps", () => { + assert.match(convertCaptions(sample, "srt"), /^1\n00:00:00,000 --> 00:00:02,500/); + assert.match(convertCaptions(sample, "srt"), /3\n00:00:04,000 --> 00:00:06,250\nFinal line\n$/); +}); + +test("Markdown output includes escaped metadata and transcript", () => { + const markdown = convertCaptions(sample, "md", { + title: "A *small* test", + author: "cobalt", + language: "en", + source: "https://www.youtube.com/watch?v=abc", + }); + + assert.match(markdown, /^# A \\\*small\\\* test/); + assert.match(markdown, /language: en/); + assert.match(markdown, /source: /); +}); + +test("VTT output is passed through with normalized newlines", () => { + const output = convertCaptions(sample, "vtt"); + assert.ok(output.startsWith("WEBVTT\n")); + assert.ok(!output.includes("\r")); +}); + +test("caption selection prefers manual tracks and matches base languages", () => { + const tracks = [ + { language_code: "en-US", kind: "asr" }, + { language_code: "en-GB" }, + { language_code: "fr" }, + ]; + + assert.equal(selectCaptionTrack(tracks, "en", true), tracks[1]); + assert.equal(selectCaptionTrack(tracks, "en-AU", true), tracks[1]); + assert.equal(selectCaptionTrack(tracks, "en-US", false), undefined); + assert.equal(selectCaptionTrack(tracks, undefined, true), tracks[1]); + assert.equal(selectCaptionTrack(tracks, "de", true), undefined); +}); + +test("empty and unknown caption formats fail", () => { + assert.throws(() => convertCaptions("WEBVTT\n", "txt")); + assert.throws(() => convertCaptions(sample, "json")); +}); + +test("caption API requests default to plain text and reject unknown formats", () => { + const parsed = apiSchema.safeParse({ + url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + downloadMode: "captions", + }); + + assert.equal(parsed.success, true); + assert.equal(parsed.data.captionFormat, "txt"); + assert.equal(parsed.data.downloadMode, "captions"); + + assert.equal(apiSchema.safeParse({ + url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + downloadMode: "captions", + captionFormat: "json", + }).success, false); +}); diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index ff7175c9..9331a009 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -34,12 +34,15 @@ export default function({ requestIP, originalRequest: r.originalRequest, subtitles: r.subtitles, + captionFormat: r.captionFormat, + captionMetadata: !disableMetadata ? r.captionMetadata : {}, cover: !disableMetadata ? r.cover : false, cropCover: !disableMetadata ? r.cropCover : false, }, params = {}; - if (r.isPhoto) action = "photo"; + if (r.isCaptionOnly) action = "captions"; + else if (r.isPhoto) action = "photo"; else if (r.picker) action = "picker" else if (r.isGif && convertGif) action = "gif"; else if (isAudioOnly) action = "audio"; @@ -70,6 +73,37 @@ export default function({ params = { type: "proxy" }; break; + case "captions": { + const extension = r.captionFormat; + const language = r.captionLanguage; + const baseName = createFilename( + { + ...r.filenameAttributes, + extension, + qualityLabel: language, + }, + filenameStyle, + false, + false + ); + const textName = createFilename( + { + ...r.filenameAttributes, + extension: "txt", + qualityLabel: language, + }, + filenameStyle, + false, + false + ); + params = { + type: "captions", + filename: baseName, + captionTextFilename: textName, + }; + break; + } + case "gif": params = { type: "gif" }; break; @@ -252,7 +286,7 @@ export default function({ // TODO: add support for HLS // (very painful) - if (!params.isHLS && responseType !== "picker") { + if (action !== "captions" && !params.isHLS && responseType !== "picker") { const isPreferredWithExtra = localProcessing === "preferred" && extraProcessingTypes.has(params.type); diff --git a/api/src/processing/match.js b/api/src/processing/match.js index e97f7600..9966cbb8 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -49,7 +49,8 @@ export default async function({ host, patternMatch, params, authType }) { try { let r, isAudioOnly = params.downloadMode === "audio", - isAudioMuted = params.downloadMode === "mute"; + isAudioMuted = params.downloadMode === "mute", + isCaptionOnly = params.downloadMode === "captions"; if (!testers[host]) { return createResponse("error", { @@ -65,6 +66,12 @@ export default async function({ host, patternMatch, params, authType }) { }); } + if (isCaptionOnly && host !== "youtube") { + return createResponse("error", { + code: "error.api.service.captions_not_supported" + }); + } + // youtubeHLS will be fully removed in the future let youtubeHLS = params.youtubeHLS; const hlsEnv = env.enableDeprecatedYoutubeHls; @@ -121,9 +128,12 @@ export default async function({ host, patternMatch, params, authType }) { dubLang: params.youtubeDubLang, youtubeHLS, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage, + captionFormat: params.captionFormat, } - if (url.hostname === "music.youtube.com" || isAudioOnly) { + if (!isCaptionOnly && (url.hostname === "music.youtube.com" || isAudioOnly)) { fetchInfo.quality = "1080"; fetchInfo.codec = "vp9"; fetchInfo.isAudioOnly = true; @@ -284,7 +294,7 @@ export default async function({ host, patternMatch, params, authType }) { } if (r.error) { - let context; + let context = r.context; switch(r.error) { case "content.too_long": context = { diff --git a/api/src/processing/request.js b/api/src/processing/request.js index b9ebca05..cec8783d 100644 --- a/api/src/processing/request.js +++ b/api/src/processing/request.js @@ -48,6 +48,15 @@ export function createResponse(responseType, responseData) { url: createStream(responseData), filename: responseData?.filename } + if (responseData?.type === "captions") { + response.text = responseData.captionFormat === "txt" + ? response.url + : createStream({ + ...responseData, + captionFormat: "txt", + filename: responseData.captionTextFilename, + }); + } break; case "local-processing": diff --git a/api/src/processing/schema.js b/api/src/processing/schema.js index 7570a8b0..e9c8f7d6 100644 --- a/api/src/processing/schema.js +++ b/api/src/processing/schema.js @@ -15,9 +15,13 @@ export const apiSchema = z.object({ ).default("mp3"), downloadMode: z.enum( - ["auto", "audio", "mute"] + ["auto", "audio", "mute", "captions"] ).default("auto"), + captionFormat: z.enum( + ["txt", "vtt", "srt", "md"] + ).default("txt"), + filenameStyle: z.enum( ["classic", "pretty", "basic", "nerdy"] ).default("basic"), @@ -50,6 +54,12 @@ export const apiSchema = z.object({ .regex(/^[0-9a-zA-Z\-]+$/) .optional(), + captionLanguage: z.string() + .min(2) + .max(8) + .regex(/^[0-9a-zA-Z\-]+$/) + .optional(), + disableMetadata: z.boolean().default(false), allowH265: z.boolean().default(false), diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 55385971..2f7a8a8a 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -7,6 +7,7 @@ import { Innertube, Platform, Session } from "youtubei.js"; import { env } from "../../config.js"; import { getCookie } from "../cookie/manager.js"; import { getYouTubeSession } from "../helpers/youtube-session.js"; +import { selectCaptionTrack } from "../helpers/captions.js"; // https://github.com/LuanRT/YouTube.js/pull/1052 Platform.shim.eval = async (data) => { @@ -135,9 +136,18 @@ const getHlsVariants = async (hlsManifest, dispatcher) => { return variants; } -const getSubtitles = async (info, dispatcher, subtitleLang) => { - const preferredCap = info.captions.caption_tracks.find(caption => - caption.kind !== 'asr' && caption.language_code.startsWith(subtitleLang) +const normalizedLanguage = language => language?.toLowerCase().replace("_", "-"); + +const getSubtitles = async ( + info, + dispatcher, + subtitleLang, + allowAutomatic = false +) => { + const preferredCap = selectCaptionTrack( + info.captions.caption_tracks, + subtitleLang, + allowAutomatic ); const captionsUrl = preferredCap?.base_url; @@ -167,8 +177,9 @@ const getSubtitles = async (info, dispatcher, subtitleLang) => { const hlsSubtitles = hlsVariants[0]?.subtitles; if (!hlsSubtitles?.length) return; - const preferredHls = hlsSubtitles.find( - subtitle => subtitle.language.startsWith(subtitleLang) + const preferredHls = hlsSubtitles.find(subtitle => + normalizedLanguage(subtitle.language) + ?.startsWith(normalizedLanguage(preferredCap.language_code)) ); if (!preferredHls) return; @@ -217,7 +228,7 @@ export default async function (o) { ); // we can get subtitles reliably only from the iOS client - if (o.subtitleLang) { + if (o.subtitleLang || o.isCaptionOnly) { innertubeClient = "IOS"; useSession = false; } @@ -324,7 +335,7 @@ export default async function (o) { return { error: "content.video.live" }; } - if (basicInfo.duration > env.durationLimit) { + if (!o.isCaptionOnly && basicInfo.duration > env.durationLimit) { return { error: "content.too_long" }; } @@ -337,6 +348,57 @@ export default async function (o) { } } + if (o.isCaptionOnly) { + const captionTracks = info.captions?.caption_tracks || []; + if (!captionTracks.length) { + return { error: "youtube.captions_unavailable" }; + } + + const selectedTrack = selectCaptionTrack( + captionTracks, + o.captionLanguage || o.subtitleLang, + true + ); + if (!selectedTrack) { + return { + error: "youtube.caption_language_unavailable", + context: { + languages: [...new Set(captionTracks.map(track => track.language_code))], + } + }; + } + + const captions = await getSubtitles( + info, + o.dispatcher, + selectedTrack.language_code, + true + ); + if (!captions?.url) { + return { error: "youtube.captions_unavailable" }; + } + + return { + type: "captions", + isCaptionOnly: true, + urls: captions.url, + captionFormat: o.captionFormat, + captionLanguage: captions.language, + filenameAttributes: { + service: "youtube", + id: o.id, + title: basicInfo.title.trim(), + author: basicInfo.author.replace("- Topic", "").trim(), + }, + captionMetadata: { + title: basicInfo.title.trim(), + author: basicInfo.author.replace("- Topic", "").trim(), + language: captions.language, + source: `https://www.youtube.com/watch?v=${o.id}`, + }, + }; + } + const normalizeQuality = res => { const shortestSide = Math.min(res.height, res.width); return videoQualities.find(qual => qual >= shortestSide); diff --git a/api/src/stream/captions.js b/api/src/stream/captions.js new file mode 100644 index 00000000..dbe94ad4 --- /dev/null +++ b/api/src/stream/captions.js @@ -0,0 +1,69 @@ +import { Agent, request } from "undici"; +import { create as contentDisposition } from "content-disposition-header"; + +import { closeRequest, closeResponse } from "./shared.js"; +import { destroyInternalStream } from "./manage.js"; +import { convertCaptions } from "../processing/helpers/caption-formats.js"; + +const defaultAgent = new Agent(); +const maxCaptionSize = 8 * 1024 * 1024; + +const contentTypes = { + md: "text/markdown; charset=utf-8", + srt: "application/x-subrip; charset=utf-8", + txt: "text/plain; charset=utf-8", + vtt: "text/vtt; charset=utf-8", +}; + +export default async function captions(streamInfo, res) { + const abortController = new AbortController(); + const shutdown = () => ( + closeRequest(abortController), + closeResponse(res), + destroyInternalStream(streamInfo.urls) + ); + + try { + const response = await request(streamInfo.urls, { + signal: abortController.signal, + maxRedirections: 4, + dispatcher: defaultAgent, + }); + + if (response.statusCode < 200 || response.statusCode > 299) { + return shutdown(); + } + + const declaredSize = Number(response.headers["content-length"]); + if (declaredSize > maxCaptionSize) { + res.status(413).end(); + return shutdown(); + } + + const chunks = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + if (size > maxCaptionSize) { + res.status(413).end(); + return shutdown(); + } + chunks.push(chunk); + } + + const body = convertCaptions( + Buffer.concat(chunks).toString("utf8"), + streamInfo.captionFormat, + streamInfo.captionMetadata + ); + + res.setHeader("Cross-Origin-Resource-Policy", "cross-origin"); + res.setHeader("Content-Disposition", contentDisposition(streamInfo.filename)); + res.setHeader("Content-Type", contentTypes[streamInfo.captionFormat]); + res.setHeader("Content-Length", Buffer.byteLength(body)); + res.status(200).send(body); + destroyInternalStream(streamInfo.urls); + } catch { + shutdown(); + } +} diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index 93e9e652..b38cb439 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -45,6 +45,9 @@ export function createStream(obj) { // url to a subtitle file subtitles: obj.subtitles, + + captionFormat: obj.captionFormat, + captionMetadata: obj.captionMetadata, }; // FIXME: this is now a Promise, but it is not awaited @@ -264,7 +267,11 @@ function wrapStream(streamInfo) { } if (typeof url === 'string') { - streamInfo.urls = createInternalStream(url, streamInfo); + streamInfo.urls = createInternalStream( + url, + streamInfo, + streamInfo.type === "captions" + ); } else if (Array.isArray(url)) { for (const idx in streamInfo.urls) { streamInfo.urls[idx] = createInternalStream( diff --git a/api/src/stream/stream.js b/api/src/stream/stream.js index 1290c029..5e620571 100644 --- a/api/src/stream/stream.js +++ b/api/src/stream/stream.js @@ -1,5 +1,6 @@ import proxy from "./proxy.js"; import ffmpeg from "./ffmpeg.js"; +import captions from "./captions.js"; import { closeResponse } from "./shared.js"; import { internalStream } from "./internal.js"; @@ -10,6 +11,9 @@ export default async function(res, streamInfo) { case "proxy": return await proxy(streamInfo, res); + case "captions": + return await captions(streamInfo, res); + case "internal": return await internalStream(streamInfo.data, res); diff --git a/api/src/util/tests/youtube.json b/api/src/util/tests/youtube.json index cb4964be..4a83fb18 100644 --- a/api/src/util/tests/youtube.json +++ b/api/src/util/tests/youtube.json @@ -240,5 +240,19 @@ "code": 200, "status": "tunnel" } + }, + { + "name": "standalone English transcript", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "subtitleLang": "en" + }, + "expected": { + "code": 200, + "status": "tunnel" + } } -] \ No newline at end of file +]