diff --git a/.github/test.sh b/.github/test.sh index 90f7f19c..6f788f03 100755 --- a/.github/test.sh +++ b/.github/test.sh @@ -12,6 +12,7 @@ waitport() { } test_api() { + pnpm run --prefix api test:unit waitport 3000 curl -m 3 http://localhost:3000/ API_RESPONSE=$(curl -m 10 http://localhost:3000/ \ diff --git a/api/README.md b/api/README.md index 968e3a18..f83c088f 100644 --- a/api/README.md +++ b/api/README.md @@ -15,29 +15,29 @@ you can read [the api documentation here](/docs/api.md). this list is not final and keeps expanding over time! if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀). -| service | video + audio | only audio | only video | metadata | rich file names | -| :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | -| bilibili | ✅ | ✅ | ✅ | ➖ | ➖ | -| bluesky | ✅ | ✅ | ✅ | ➖ | ➖ | -| dailymotion | ✅ | ✅ | ✅ | ✅ | ✅ | -| instagram | ✅ | ✅ | ✅ | ➖ | ➖ | -| facebook | ✅ | ❌ | ✅ | ➖ | ➖ | -| loom | ✅ | ❌ | ✅ | ✅ | ➖ | -| newgrounds | ✅ | ✅ | ✅ | ✅ | ✅ | -| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ | -| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | -| reddit | ✅ | ✅ | ✅ | ❌ | ❌ | -| rutube | ✅ | ✅ | ✅ | ✅ | ✅ | -| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ | -| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ | -| streamable | ✅ | ✅ | ✅ | ➖ | ➖ | -| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ | -| tumblr | ✅ | ✅ | ✅ | ➖ | ➖ | -| twitch clips | ✅ | ✅ | ✅ | ✅ | ✅ | -| twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | -| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | -| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | -| youtube | ✅ | ✅ | ✅ | ✅ | ✅ | +| service | video + audio | only audio | only video | transcript only | metadata | rich file names | +| :-------- | :-----------: | :--------: | :--------: | :-------------: | :------: | :-------------: | +| bilibili | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| bluesky | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| dailymotion | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | +| instagram | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| facebook | ✅ | ❌ | ✅ | ❌ | ➖ | ➖ | +| loom | ✅ | ❌ | ✅ | ✅ | ✅ | ➖ | +| newgrounds | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | +| ok.ru | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | +| pinterest | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| reddit | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | +| rutube | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| snapchat | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| soundcloud | ➖ | ✅ | ➖ | ❌ | ✅ | ✅ | +| streamable | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| tiktok | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| tumblr | ✅ | ✅ | ✅ | ❌ | ➖ | ➖ | +| twitch clips | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | +| twitter/x | ✅ | ✅ | ✅ | ✅ | ➖ | ➖ | +| vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | +| youtube | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | emoji | meaning | | :-----: | :---------------------- | @@ -45,6 +45,8 @@ if the desired service isn't supported yet, feel free to create an appropriate i | ➖ | unreasonable/impossible | | ❌ | not supported | +standalone transcript downloads are supported for marked services with `downloadMode: "captions"`. + ### additional notes or features (per service) | service | notes or features | | :-------- | :----- | diff --git a/api/package.json b/api/package.json index 3166ba79..84de764b 100644 --- a/api/package.json +++ b/api/package.json @@ -11,6 +11,7 @@ "scripts": { "start": "node src/cobalt", "test": "node src/util/test", + "test:unit": "node --test src/processing/helpers/tests/*.test.js", "token:jwt": "node src/util/generate-jwt-secret" }, "repository": { diff --git a/api/src/processing/helpers/caption-formats.js b/api/src/processing/helpers/caption-formats.js new file mode 100644 index 00000000..f471bf63 --- /dev/null +++ b/api/src/processing/helpers/caption-formats.js @@ -0,0 +1,266 @@ +const namedEntities = { + amp: "&", + apos: "'", + gt: ">", + lt: "<", + nbsp: " ", + quot: '"', +}; + +const decodeEntities = text => text.replace( + /&(#(?:x[0-9a-f]+|\d+)|[a-z]+);/gi, + (entity, value) => { + if (value[0] !== "#") { + return namedEntities[value.toLowerCase()] ?? entity; + } + + const hex = value[1]?.toLowerCase() === "x"; + const codePoint = Number.parseInt(value.slice(hex ? 2 : 1), hex ? 16 : 10); + try { + return String.fromCodePoint(codePoint); + } catch { + return entity; + } + } +); + +const cleanCueText = text => decodeEntities( + text + .replace(/<\d{2}:\d{2}(?::\d{2})?\.\d{3}>/g, "") + .replace(//gi, "\n") + .replace(/<[^>]+>/g, "") +).split("\n") + .map(line => line.trim()) + .filter(Boolean) + .join("\n"); + +export const parseVtt = input => { + const normalized = input.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + const blocks = normalized.split(/\n{2,}/); + const cues = []; + + for (const block of blocks) { + const lines = block.split("\n").map(line => line.trimEnd()); + if (!lines.length || /^(WEBVTT|NOTE|STYLE|REGION)(?:\s|$)/.test(lines[0])) { + continue; + } + + const timingIndex = lines.findIndex(line => line.includes(" --> ")); + if (timingIndex < 0) continue; + + const timing = lines[timingIndex].match( + /^(\d{2}:)?\d{2}:\d{2}\.\d{3}\s+-->\s+(\d{2}:)?\d{2}:\d{2}\.\d{3}/ + )?.[0]; + if (!timing) continue; + + const [ start, end ] = timing.split(/\s+-->\s+/); + const text = cleanCueText(lines.slice(timingIndex + 1).join("\n")); + if (text) cues.push({ start, end, text }); + } + + return cues; +}; + +export const parseSrt = input => { + const normalized = input.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + const cues = []; + + for (const block of normalized.split(/\n{2,}/)) { + const lines = block.split("\n").map(line => line.trimEnd()); + const timingIndex = lines.findIndex(line => line.includes(" --> ")); + if (timingIndex < 0) continue; + + const timing = lines[timingIndex].match( + /^(\d{2}:)?\d{2}:\d{2},\d{3}\s+-->\s+(\d{2}:)?\d{2}:\d{2},\d{3}/ + )?.[0]; + if (!timing) continue; + + const [ start, end ] = timing.split(/\s+-->\s+/).map(value => + value.replace(",", ".") + ); + const text = cleanCueText(lines.slice(timingIndex + 1).join("\n")); + if (text) cues.push({ start, end, text }); + } + + return cues; +}; + +const timestampSeconds = timestamp => timestamp.split(":").reduce( + (seconds, part) => seconds * 60 + Number(part), 0 +); + +const vttTimestamp = seconds => { + const milliseconds = Math.round(seconds * 1000); + if (!Number.isFinite(milliseconds) || milliseconds < 0) { + throw new Error("invalid caption timestamp"); + } + const hours = Math.floor(milliseconds / 3600000); + const minutes = Math.floor(milliseconds / 60000) % 60; + const remaining = milliseconds % 60000; + return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:` + + `${(remaining / 1000).toFixed(3).padStart(6, "0")}`; +}; + +// HLS repeats boundary cues and may use a different local clock per segment. +// Keep the first segment's timeline while applying subsequent timestamp maps. +// https://www.rfc-editor.org/rfc/rfc8216#section-3.5 +export const mergeCaptionSegments = segments => { + const blocks = new Set(); + const metadata = new Set(); + const clockWrap = 2 ** 33; + let firstOffset, previousMpeg; + let adjustment; + let wraps = 0; + + for (const segment of segments) { + const normalized = segment.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + const hasHeader = /^WEBVTT(?:\s|$)/.test(normalized); + if (hasHeader) { + const header = normalized.split(/\n{2,}/)[0]; + const map = header.match(/^X-TIMESTAMP-MAP=(.+)$/m)?.[1]; + const local = map?.match(/LOCAL:([\d:.]+)/)?.[1]; + const mpeg = map?.match(/MPEGTS:(\d+)/)?.[1]; + if (map && (!local || !mpeg)) throw new Error("invalid caption timestamp map"); + const timestamp = Number(mpeg); + if (map && previousMpeg !== undefined + && timestamp - previousMpeg < -clockWrap / 2) wraps++; + if (map) previousMpeg = timestamp; + const offset = map + ? (timestamp + wraps * clockWrap) / 90000 - timestampSeconds(local) + : 0; + firstOffset ??= offset; + adjustment = offset - firstOffset; + } else if (adjustment === undefined) { + throw new Error("invalid VTT segment"); + } + + const segmentBlocks = normalized.split(/\n{2,}/).slice(hasHeader ? 1 : 0); + for (const block of segmentBlocks) { + if (/^(STYLE|REGION)(?:\s|$)/.test(block)) { + metadata.add(block.trimEnd()); + continue; + } + if (/^NOTE(?:\s|$)/.test(block)) continue; + const lines = block.trimEnd().split("\n"); + const timingIndex = lines.findIndex(line => line.includes(" --> ")); + if (timingIndex < 0) continue; + const timing = lines[timingIndex].match(/^(\S+)\s+-->\s+(\S+)(.*)$/); + if (!timing) throw new Error("invalid caption cue"); + const start = vttTimestamp(timestampSeconds(timing[1]) + adjustment); + const end = vttTimestamp(timestampSeconds(timing[2]) + adjustment); + // Identifiers can change across segments; timing and payload identify a cue. + blocks.add([ + `${start} --> ${end}${timing[3]}`, + ...lines.slice(timingIndex + 1), + ].join("\n")); + } + } + return `WEBVTT\n\n${[...metadata, ...blocks].join("\n\n")}\n`; +}; + +const collapseRollingCues = cues => { + const result = []; + let run = [], rows = [], rolling = false; + const flush = () => { + if (rolling) { + result.push({ ...run[0], end: run.at(-1).end, text: rows.join("\n") }); + } else { + for (const cue of run) result.push(cue); + } + run = []; + rows = []; + rolling = false; + }; + + for (const cue of cues) { + const nextRows = cue.text.split("\n"); + const previous = run.at(-1); + let shared = 0; + if (previous && timestampSeconds(cue.start) >= timestampSeconds(previous.start) + && timestampSeconds(cue.start) <= timestampSeconds(previous.end)) { + const previousRows = previous.text.split("\n"); + for (let count = Math.min(previousRows.length, nextRows.length); count > 0; count--) { + if (previousRows.slice(-count).every((row, index) => row === nextRows[index])) { + shared = count; + break; + } + } + // A changing multiline window distinguishes rolling text from repeated speech. + if (shared && previous.text !== cue.text + && (previousRows.length > 1 || nextRows.length > 1)) rolling = true; + } + if (!shared) flush(); + for (const row of nextRows.slice(shared)) rows.push(row); + run.push(cue); + } + flush(); + return result; +}; + +const transcriptLines = cues => { + const lines = []; + let previous; + for (const cue of collapseRollingCues(cues)) { + const text = cue.text.replace(/\s*\n\s*/g, " ").replace(/\s+/g, " ").trim(); + if (!text) continue; + + const overlaps = previous + && timestampSeconds(cue.start) >= timestampSeconds(previous.start) + && timestampSeconds(cue.start) < timestampSeconds(previous.end); + if (overlaps && (text === lines.at(-1) || text.startsWith(`${lines.at(-1)} `))) { + lines[lines.length - 1] = text; + } else { + lines.push(text); + } + previous = cue; + } + return lines; +}; + +const srtTimestamp = timestamp => { + const withHours = timestamp.split(":").length === 2 ? `00:${timestamp}` : timestamp; + return withHours.replace(".", ","); +}; + +const escapeMarkdown = value => value.replace(/[\\`*_{}\[\]<>#+.!|~-]/g, "\\$&"); + +export const convertCaptions = (input, format, metadata = {}, sourceFormat = "vtt") => { + if (!["vtt", "srt"].includes(sourceFormat)) throw new Error("unsupported caption source format"); + const cues = sourceFormat === "srt" ? parseSrt(input) : parseVtt(input); + if (!cues.length) throw new Error("caption file has no cues"); + + if (sourceFormat === "vtt" && format === "vtt") { + return input.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n"); + } + + if (format === "vtt") { + return "WEBVTT\n\n" + cues.map(cue => [ + `${cue.start} --> ${cue.end}`, + cue.text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"), + ].join("\n")).join("\n\n") + "\n"; + } + + if (format === "srt") { + return cues.map((cue, index) => [ + index + 1, + `${srtTimestamp(cue.start)} --> ${srtTimestamp(cue.end)}`, + cue.text, + ].join("\n")).join("\n\n") + "\n"; + } + + const transcript = transcriptLines(cues); + if (format === "txt") return transcript.join("\n\n") + "\n"; + + if (format === "md") { + const heading = `# ${escapeMarkdown(metadata.title || "Transcript")}`; + const details = [ + metadata.author && `author: ${escapeMarkdown(metadata.author)}`, + metadata.language && `language: ${escapeMarkdown(metadata.language)}`, + metadata.source && `source: <${metadata.source}>`, + ].filter(Boolean); + + return [ heading, ...details, ...transcript.map(escapeMarkdown) ].join("\n\n") + "\n"; + } + + throw new Error("unsupported caption format"); +}; diff --git a/api/src/processing/helpers/captions.js b/api/src/processing/helpers/captions.js new file mode 100644 index 00000000..306a2adc --- /dev/null +++ b/api/src/processing/helpers/captions.js @@ -0,0 +1,132 @@ +import { convertLanguageCode } from "../../misc/language-codes.js"; + +// Services use both ISO 639-1 and ISO 639-2, sometimes with a script or region. +export const normalizeCaptionLanguage = language => { + if (!language) return; + const [base, ...parts] = language.replaceAll("_", "-").split("-"); + const normalized = [ + base.length === 3 ? convertLanguageCode(base) || base : base, + ...parts, + ].join("-"); + try { + return new Intl.Locale(normalized).toString(); + } catch { + return normalized.toLowerCase(); + } +}; + +const languageRank = (language, requested) => { + if (!requested) return 0; + if (!language) return Infinity; + if (language === requested) return 0; + + try { + const track = new Intl.Locale(language); + const target = new Intl.Locale(requested); + if (track.language !== target.language) return Infinity; + // Bare languages accept any script. Explicit scripts or regions must + // not fall back to a different writing system, e.g. zh-Hant -> zh-Hans. + if ((target.script || target.region) + && track.maximize().script !== target.maximize().script) return Infinity; + return 1; + } catch { + return Infinity; + } +}; + +export const captionSelectionError = tracks => { + const available = tracks.filter(track => track.url); + if (!available.length) return { error: "captions.unavailable" }; + return { + error: "captions.language_unavailable", + context: { + languages: [...new Set(available.map(track => + normalizeCaptionLanguage(track.language) + ).filter(Boolean))], + }, + }; +}; + +export const createCaptionResponse = ({ + url, + format, + sourceFormat = "vtt", + language, + service, + id, + title, + author, + source, + headers, +}) => { + const resolvedTitle = title || `${service}_${id}`; + return { + type: "captions", + isCaptionOnly: true, + urls: url, + headers, + captionFormat: format, + captionSourceFormat: sourceFormat, + captionLanguage: language || "unknown", + filenameAttributes: { + service, + id, + title: resolvedTitle, + author, + }, + captionMetadata: { + title: resolvedTitle, + author, + language, + source, + }, + }; +}; + +const captionResource = (resource, playlistURL) => { + const output = { + url: new URL(resource.uri, playlistURL).toString(), + }; + if (resource.byterange) { + const { offset, length } = resource.byterange; + output.headers = { range: `bytes=${offset}-${offset + length - 1}` }; + } + return output; +}; + +export const captionSegmentResources = (segments, playlistURL) => { + const resources = []; + let previousMap; + + for (const segment of segments) { + if (segment.map) { + const map = captionResource(segment.map, playlistURL); + const mapKey = `${map.url}:${map.headers?.range || ""}`; + if (mapKey !== previousMap) { + resources.push(map); + previousMap = mapKey; + } + } else { + previousMap = undefined; + } + resources.push(captionResource(segment, playlistURL)); + } + return resources; +}; + +// Tracks share { url, language, automatic, sourceFormat }; adapters may add fields. +// Preserve source order when language rank and manual/automatic status are equal. +export const selectCaptionTrack = (tracks, language, allowAutomatic = true) => { + const requested = normalizeCaptionLanguage(language); + let selected, bestRank = Infinity; + for (const track of tracks) { + if (!track.url || (!allowAutomatic && track.automatic)) continue; + const rank = languageRank(normalizeCaptionLanguage(track.language), requested) * 2 + + Number(!!track.automatic); + if (rank < bestRank) { + selected = track; + bestRank = rank; + } + } + return selected; +}; diff --git a/api/src/processing/helpers/tests/captions.test.js b/api/src/processing/helpers/tests/captions.test.js new file mode 100644 index 00000000..d5ed9ff6 --- /dev/null +++ b/api/src/processing/helpers/tests/captions.test.js @@ -0,0 +1,176 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { captionSegmentResources, selectCaptionTrack } from "../captions.js"; +import { convertCaptions, mergeCaptionSegments, parseSrt, parseVtt } from "../caption-formats.js"; +import { apiSchema } from "../../schema.js"; + +const sample = `\uFEFFWEBVTT\r +\r +NOTE generated fixture\r +ignored\r +\r +first cue\r +00:00.000 --> 00:02.500 align:start position:0%\r +Hello & welcome\r +\r +00:02.500 --> 00:04.000\r +Hello & welcome back\r +\r +00:04.000 --> 00:06.250\r +Final line\r +`; + +const sampleSrt = `1\r +00:00:00,000 --> 00:00:02,500\r +Hello & welcome\r +\r +2\r +00:00:02,500 --> 00:00:04,000\r +Final line\r +`; + +test("parseVtt extracts and cleans cues", () => { + assert.deepEqual(parseVtt(sample), [ + { start: "00:00.000", end: "00:02.500", text: "Hello & welcome" }, + { start: "00:02.500", end: "00:04.000", text: "Hello & welcome back" }, + { start: "00:04.000", end: "00:06.250", text: "Final line" }, + ]); +}); + +test("SRT input converts to transcripts and VTT", () => { + assert.equal(parseSrt(sampleSrt).length, 2); + assert.equal( + convertCaptions(sampleSrt, "txt", {}, "srt"), + "Hello & welcome\n\nFinal line\n" + ); + assert.match( + convertCaptions(sampleSrt, "vtt", {}, "srt"), + /^WEBVTT\n\n00:00:00\.000 --> 00:00:02\.500/ + ); +}); + +test("plain text collapses growing automatic-caption cues", () => { + assert.equal( + convertCaptions(sample.replace("00:02.500 -->", "00:01.500 -->"), "txt"), + "Hello & welcome back\n\nFinal line\n" + ); +}); + +test("SRT output numbers cues and converts timestamps", () => { + assert.match(convertCaptions(sample, "srt"), /^1\n00:00:00,000 --> 00:00:02,500/); + assert.match(convertCaptions(sample, "srt"), /3\n00:00:04,000 --> 00:00:06,250\nFinal line\n$/); +}); + +test("Markdown output includes escaped metadata and transcript", () => { + const markdown = convertCaptions(sample, "md", { + title: "A *small* test", + author: "cobalt", + language: "en", + source: "https://www.youtube.com/watch?v=abc", + }); + + assert.match(markdown, /^# A \\\*small\\\* test/); + assert.match(markdown, /language: en/); + assert.match(markdown, /source: /); +}); + +test("VTT output is passed through with normalized newlines", () => { + const output = convertCaptions(sample, "vtt"); + assert.ok(output.startsWith("WEBVTT\n")); + assert.ok(!output.includes("\r")); +}); + +test("caption selection prefers manual tracks and matches base languages", () => { + const tracks = [ + { url: "auto", language: "en-US", automatic: true }, + { url: "manual", language: "en-GB" }, + { url: "french", language: "fr" }, + ]; + + assert.equal(selectCaptionTrack(tracks, "en", true), tracks[1]); + assert.equal(selectCaptionTrack(tracks, "en-AU", true), tracks[1]); + assert.equal(selectCaptionTrack(tracks, "en-US", false), tracks[1]); + assert.equal(selectCaptionTrack(tracks, undefined, true), tracks[1]); + assert.equal(selectCaptionTrack(tracks, "de", true), undefined); +}); + +test("empty and unknown caption formats fail", () => { + assert.throws(() => convertCaptions("WEBVTT\n", "txt")); + assert.throws(() => convertCaptions(sample, "json")); +}); + +test("caption API requests default to plain text and reject unknown formats", () => { + const parsed = apiSchema.safeParse({ + url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + downloadMode: "captions", + }); + + assert.equal(parsed.success, true); + assert.equal(parsed.data.captionFormat, "txt"); + assert.equal(parsed.data.downloadMode, "captions"); + + assert.equal(apiSchema.safeParse({ + url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + downloadMode: "captions", + captionFormat: "json", + }).success, false); +}); + +test("adjoining rolling windows collapse but separate repeated speech survives", () => { + const input = "WEBVTT\n\n" + + "00:00.000 --> 00:00.010\nA new game\n\n" + + "00:00.010 --> 00:02.000\nA new game\n\n" + + "00:02.000 --> 00:04.000\nA new game\nwith a large world\n\n" + + "00:04.000 --> 00:06.000\nwith a large world\n\n" + + "00:10.000 --> 00:11.000\nNo.\n\n" + + "00:11.000 --> 00:12.000\nNo.\n"; + assert.equal(convertCaptions(input, "txt"), "A new game with a large world\n\nNo.\n\nNo.\n"); + assert.equal(convertCaptions(input, "md"), "# Transcript\n\nA new game with a large world\n\nNo\\.\n\nNo\\.\n"); + assert.equal(parseVtt(convertCaptions(input, "vtt")).length, 6); + assert.equal(parseSrt(convertCaptions(input, "srt")).length, 6); +}); + +test("HLS merging preserves cue settings and markup and removes repeated boundary cues", () => { + const cue = "00:00:09.000 --> 00:00:12.000 align:start\nHello & welcome"; + const merged = mergeCaptionSegments([ + `WEBVTT\n\nSTYLE\n::cue { color: white; }\n\nfirst-id\n${cue}\n`, + `WEBVTT\n\nnew-id\n${cue}\n\n00:00:12.000 --> 00:00:14.000\nSecond segment\n`, + "WEBVTT\n", + ]); + assert.equal(parseVtt(merged).length, 2); + assert.ok(merged.includes(cue)); + assert.ok(merged.includes("STYLE\n::cue { color: white; }")); + assert.equal(convertCaptions(merged, "txt"), "Hello & welcome\n\nSecond segment\n"); +}); + +test("HLS merging accepts media fragments after an initialization section", () => { + const merged = mergeCaptionSegments([ + "WEBVTT\nX-TIMESTAMP-MAP=LOCAL:00:00:00.000,MPEGTS:900000\n", + "00:00:00.000 --> 00:00:02.000\nFirst fragment\n", + "00:00:02.000 --> 00:00:04.000\nSecond fragment\n", + ]); + assert.equal(convertCaptions(merged, "txt"), "First fragment\n\nSecond fragment\n"); +}); + +test("HLS resources include changed initialization sections and byte ranges", () => { + const map = { uri: "captions.vtt", byterange: { offset: 0, length: 100 } }; + const resources = captionSegmentResources([ + { uri: "captions.vtt", byterange: { offset: 100, length: 50 }, map }, + { uri: "captions.vtt", byterange: { offset: 150, length: 60 }, map }, + { + uri: "captions.vtt", + byterange: { offset: 310, length: 70 }, + map: { uri: "captions.vtt", byterange: { offset: 210, length: 100 } }, + }, + ], "https://example.com/path/playlist.m3u8"); + + assert.deepEqual(resources.map(resource => resource.headers.range), [ + "bytes=0-99", + "bytes=100-149", + "bytes=150-209", + "bytes=210-309", + "bytes=310-379", + ]); + assert.ok(resources.every(resource => resource.url === "https://example.com/path/captions.vtt")); +}); diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index ff7175c9..6120cfa5 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -34,12 +34,16 @@ export default function({ requestIP, originalRequest: r.originalRequest, subtitles: r.subtitles, + captionFormat: r.captionFormat, + captionSourceFormat: r.captionSourceFormat, + captionMetadata: !disableMetadata ? r.captionMetadata : {}, cover: !disableMetadata ? r.cover : false, cropCover: !disableMetadata ? r.cropCover : false, }, params = {}; - if (r.isPhoto) action = "photo"; + if (r.isCaptionOnly) action = "captions"; + else if (r.isPhoto) action = "photo"; else if (r.picker) action = "picker" else if (r.isGif && convertGif) action = "gif"; else if (isAudioOnly) action = "audio"; @@ -70,6 +74,37 @@ export default function({ params = { type: "proxy" }; break; + case "captions": { + const extension = r.captionFormat; + const language = r.captionLanguage; + const baseName = createFilename( + { + ...r.filenameAttributes, + extension, + qualityLabel: language, + }, + filenameStyle, + false, + false + ); + const textName = createFilename( + { + ...r.filenameAttributes, + extension: "txt", + qualityLabel: language, + }, + filenameStyle, + false, + false + ); + params = { + type: "captions", + filename: baseName, + captionTextFilename: textName, + }; + break; + } + case "gif": params = { type: "gif" }; break; @@ -252,7 +287,7 @@ export default function({ // TODO: add support for HLS // (very painful) - if (!params.isHLS && responseType !== "picker") { + if (action !== "captions" && !params.isHLS && responseType !== "picker") { const isPreferredWithExtra = localProcessing === "preferred" && extraProcessingTypes.has(params.type); diff --git a/api/src/processing/match.js b/api/src/processing/match.js index e97f7600..de38fcd2 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -7,6 +7,7 @@ import { testers } from "./service-patterns.js"; import matchAction from "./match-action.js"; import { friendlyServiceName } from "./service-alias.js"; +import { captionOnlyServices } from "./service-config.js"; import bilibili from "./services/bilibili.js"; import reddit from "./services/reddit.js"; @@ -49,7 +50,8 @@ export default async function({ host, patternMatch, params, authType }) { try { let r, isAudioOnly = params.downloadMode === "audio", - isAudioMuted = params.downloadMode === "mute"; + isAudioMuted = params.downloadMode === "mute", + isCaptionOnly = params.downloadMode === "captions"; if (!testers[host]) { return createResponse("error", { @@ -65,6 +67,12 @@ export default async function({ host, patternMatch, params, authType }) { }); } + if (isCaptionOnly && !captionOnlyServices.has(host)) { + return createResponse("error", { + code: "error.api.service.captions_not_supported" + }); + } + // youtubeHLS will be fully removed in the future let youtubeHLS = params.youtubeHLS; const hlsEnv = env.enableDeprecatedYoutubeHls; @@ -84,7 +92,10 @@ export default async function({ host, patternMatch, params, authType }) { toGif: !!params.convertGif, alwaysProxy: params.alwaysProxy, dispatcher, - subtitleLang + subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage || subtitleLang, + captionFormat: params.captionFormat, }); break; @@ -95,6 +106,9 @@ export default async function({ host, patternMatch, params, authType }) { accessKey: patternMatch.accessKey, quality: params.videoQuality, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage || subtitleLang, + captionFormat: params.captionFormat, }); break; @@ -121,9 +135,12 @@ export default async function({ host, patternMatch, params, authType }) { dubLang: params.youtubeDubLang, youtubeHLS, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage, + captionFormat: params.captionFormat, } - if (url.hostname === "music.youtube.com" || isAudioOnly) { + if (!isCaptionOnly && (url.hostname === "music.youtube.com" || isAudioOnly)) { fetchInfo.quality = "1080"; fetchInfo.codec = "vp9"; fetchInfo.isAudioOnly = true; @@ -153,6 +170,9 @@ export default async function({ host, patternMatch, params, authType }) { h265: params.allowH265, alwaysProxy: params.alwaysProxy, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage || subtitleLang, + captionFormat: params.captionFormat, }); break; @@ -171,6 +191,9 @@ export default async function({ host, patternMatch, params, authType }) { quality: params.videoQuality, isAudioOnly, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage || subtitleLang, + captionFormat: params.captionFormat, }); break; @@ -223,6 +246,9 @@ export default async function({ host, patternMatch, params, authType }) { quality: params.videoQuality, isAudioOnly, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage || subtitleLang, + captionFormat: params.captionFormat, }); break; @@ -241,6 +267,9 @@ export default async function({ host, patternMatch, params, authType }) { r = await loom({ id: patternMatch.id, subtitleLang, + isCaptionOnly, + captionLanguage: params.captionLanguage || subtitleLang, + captionFormat: params.captionFormat, }); break; @@ -284,7 +313,7 @@ export default async function({ host, patternMatch, params, authType }) { } if (r.error) { - let context; + let context = r.context; switch(r.error) { case "content.too_long": context = { diff --git a/api/src/processing/request.js b/api/src/processing/request.js index b9ebca05..cec8783d 100644 --- a/api/src/processing/request.js +++ b/api/src/processing/request.js @@ -48,6 +48,15 @@ export function createResponse(responseType, responseData) { url: createStream(responseData), filename: responseData?.filename } + if (responseData?.type === "captions") { + response.text = responseData.captionFormat === "txt" + ? response.url + : createStream({ + ...responseData, + captionFormat: "txt", + filename: responseData.captionTextFilename, + }); + } break; case "local-processing": diff --git a/api/src/processing/schema.js b/api/src/processing/schema.js index 7570a8b0..e9c8f7d6 100644 --- a/api/src/processing/schema.js +++ b/api/src/processing/schema.js @@ -15,9 +15,13 @@ export const apiSchema = z.object({ ).default("mp3"), downloadMode: z.enum( - ["auto", "audio", "mute"] + ["auto", "audio", "mute", "captions"] ).default("auto"), + captionFormat: z.enum( + ["txt", "vtt", "srt", "md"] + ).default("txt"), + filenameStyle: z.enum( ["classic", "pretty", "basic", "nerdy"] ).default("basic"), @@ -50,6 +54,12 @@ export const apiSchema = z.object({ .regex(/^[0-9a-zA-Z\-]+$/) .optional(), + captionLanguage: z.string() + .min(2) + .max(8) + .regex(/^[0-9a-zA-Z\-]+$/) + .optional(), + disableMetadata: z.boolean().default(false), allowH265: z.boolean().default(false), diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index fbe25805..dcfc391e 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -1,6 +1,7 @@ import UrlPattern from "url-pattern"; export const audioIgnore = new Set(["vk", "ok", "loom"]); +export const captionOnlyServices = new Set(["loom", "rutube", "tiktok", "twitter", "vimeo", "vk", "youtube"]); export const hlsExceptions = new Set(["dailymotion", "vimeo", "rutube", "bsky", "youtube"]); export const services = { diff --git a/api/src/processing/services/loom.js b/api/src/processing/services/loom.js index 64382108..a949ad12 100644 --- a/api/src/processing/services/loom.js +++ b/api/src/processing/services/loom.js @@ -1,4 +1,9 @@ import { genericUserAgent } from "../../config.js"; +import { + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; const craftHeaders = id => ({ "user-agent": genericUserAgent, @@ -80,12 +85,39 @@ async function getTranscript(id) { .then(r => r.status === 200 && r.json()) .catch(() => {}); - if (gql?.data?.fetchVideoTranscript?.captions_source_url?.includes('.vtt?')) { - return gql.data.fetchVideoTranscript.captions_source_url; + const transcript = gql?.data?.fetchVideoTranscript; + if (transcript?.captions_source_url?.includes('.vtt?')) { + return { + url: transcript.captions_source_url, + language: transcript.language, + }; } } -export default async function({ id, subtitleLang }) { +export default async function({ + id, + subtitleLang, + isCaptionOnly, + captionLanguage, + captionFormat, +}) { + if (isCaptionOnly) { + const transcript = await getTranscript(id); + const tracks = transcript ? [transcript] : []; + if (!selectCaptionTrack(tracks, captionLanguage)) { + return captionSelectionError(tracks); + } + + return createCaptionResponse({ + url: transcript.url, + format: captionFormat, + language: transcript.language, + service: "loom", + id, + source: `https://www.loom.com/share/${id}`, + }); + } + let url = await fromTranscodedURL(id); url ??= await fromRawURL(id); @@ -96,7 +128,7 @@ export default async function({ id, subtitleLang }) { let subtitles; if (subtitleLang) { const transcript = await getTranscript(id); - if (transcript) subtitles = transcript; + if (transcript) subtitles = transcript.url; } return { diff --git a/api/src/processing/services/rutube.js b/api/src/processing/services/rutube.js index 95cb2e07..5b69a1a4 100644 --- a/api/src/processing/services/rutube.js +++ b/api/src/processing/services/rutube.js @@ -1,5 +1,10 @@ import HLS from "hls-parser"; import { env } from "../../config.js"; +import { + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; async function requestJSON(url) { try { @@ -12,6 +17,8 @@ const delta = (a, b) => Math.abs(a - b); export default async function(obj) { if (obj.yappyId) { + if (obj.isCaptionOnly) return captionSelectionError([]); + const yappy = await requestJSON( `https://rutube.ru/pangolin/api/web/yappy/v4/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=1` ) @@ -40,6 +47,30 @@ export default async function(obj) { if (play.detail || !play.video_balancer) return { error: "fetch.empty" }; if (play.live_streams?.hls) return { error: "content.video.live" }; + if (obj.isCaptionOnly) { + const tracks = (play.captions || []) + .filter(caption => ["webvtt", "srt"].includes(caption.format)) + .map(caption => ({ + url: caption.file, + language: caption.code, + sourceFormat: caption.format === "webvtt" ? "vtt" : "srt", + })); + const caption = selectCaptionTrack(tracks, obj.captionLanguage); + if (!caption) return captionSelectionError(tracks); + + return createCaptionResponse({ + url: caption.url, + format: obj.captionFormat, + sourceFormat: caption.sourceFormat, + language: caption.language, + service: "rutube", + id: obj.id, + title: play.title?.trim(), + author: play.author?.name?.trim(), + source: `https://rutube.ru/video/${obj.id}/`, + }); + } + if (play.duration > env.durationLimit * 1000) return { error: "content.too_long" }; diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 1e1605e0..67541b80 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -5,6 +5,12 @@ import { genericUserAgent } from "../../config.js"; import { updateCookie } from "../cookie/manager.js"; import { createStream } from "../../stream/manage.js"; import { convertLanguageCode } from "../../misc/language-codes.js"; +import { + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, + normalizeCaptionLanguage, +} from "../helpers/captions.js"; const shortDomain = "https://vt.tiktok.com/"; @@ -72,6 +78,32 @@ export default async function(obj) { return { error: "fetch.empty" }; } + if (obj.isCaptionOnly) { + const tracks = (detail.video?.subtitleInfos || []) + .filter(subtitle => subtitle.Format === "webvtt") + .map(subtitle => ({ + url: subtitle.Url, + language: normalizeCaptionLanguage(subtitle.LanguageCodeName), + })); + const subtitle = selectCaptionTrack(tracks, obj.captionLanguage); + if (!subtitle) return captionSelectionError(tracks); + + return createCaptionResponse({ + url: subtitle.url, + format: obj.captionFormat, + language: subtitle.language, + service: "tiktok", + id: postId, + title: detail.desc?.trim(), + author: detail.author.uniqueId, + source: `https://www.tiktok.com/@${detail.author.uniqueId}/video/${postId}`, + headers: { + cookie: cookie.toString(), + "user-agent": genericUserAgent, + }, + }); + } + let video, videoFilename, audioFilename, audio, images, filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`, bestAudio; // will get defaulted to m4a later on in match-action diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index 463b0589..883d842e 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -2,6 +2,12 @@ import HLS from "hls-parser"; import { genericUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; +import { + captionSegmentResources, + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail'; const tokenURL = 'https://api.x.com/1.1/guest/activate.json'; @@ -207,7 +213,17 @@ const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) = return (repostedTweet?.media || baseTweet?.extended_entities?.media); } -export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) { +export default async function({ + id, + index, + toGif, + dispatcher, + alwaysProxy, + subtitleLang, + isCaptionOnly, + captionLanguage, + captionFormat, +}) { const cookie = await getCookie('twitter'); let guestToken = await getGuestToken(dispatcher); @@ -260,28 +276,54 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy, subti url, filename, }); - const extractSubtitles = async (hlsUrl) => { + const getSubtitleTracks = async hlsUrl => { const mainHls = await fetch(hlsUrl).then(r => r.text()).catch(() => {}); - if (!mainHls) return; + if (!mainHls) return []; + return (HLS.parse(mainHls)?.variants?.[0]?.subtitles || []).map(subtitle => ({ + url: new URL(subtitle.uri, hlsUrl).toString(), + language: subtitle.language, + })); + }; - const subtitle = HLS.parse(mainHls)?.variants[0]?.subtitles?.find( - s => s.language.startsWith(subtitleLang) - ); + const resolveSubtitle = async (subtitle, allSegments = false) => { if (!subtitle) return; - - const subtitleUrl = new URL(subtitle.uri, hlsUrl).toString(); - const subtitleHls = await fetch(subtitleUrl).then(r => r.text()); + const subtitleHls = await fetch(subtitle.url).then(r => r.text()).catch(() => {}); if (!subtitleHls) return; - const finalSubtitlePath = HLS.parse(subtitleHls)?.segments?.[0].uri; - if (!finalSubtitlePath) return; - - const finalSubtitleUrl = new URL(finalSubtitlePath, hlsUrl).toString(); - + const segments = HLS.parse(subtitleHls)?.segments; + if (!segments?.length) return; + const urls = allSegments + ? captionSegmentResources(segments, subtitle.url) + : new URL(segments[0].uri, subtitle.url).toString(); return { - url: finalSubtitleUrl, + url: urls, language: subtitle.language, }; + }; + + if (isCaptionOnly) { + const tracks = []; + for (const mediaItem of media) { + if (mediaItem.type !== "video") continue; + const hlsVariant = mediaItem.video_info?.variants?.find( + variant => variant.content_type === "application/x-mpegURL" + ); + if (hlsVariant) tracks.push(...await getSubtitleTracks(hlsVariant.url)); + } + const selected = selectCaptionTrack(tracks, captionLanguage); + if (!selected) return captionSelectionError(tracks); + const subtitles = await resolveSubtitle(selected, true); + if (!subtitles) return { error: "captions.unavailable" }; + + return createCaptionResponse({ + url: subtitles.url, + format: captionFormat, + language: subtitles.language, + service: "twitter", + id, + source: `https://x.com/i/status/${id}`, + headers: { "user-agent": genericUserAgent }, + }); } switch (media?.length) { @@ -308,7 +350,9 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy, subti v => v.content_type === "application/x-mpegURL" ); if (hlsVariant) { - const { url, language } = await extractSubtitles(hlsVariant.url) || {}; + const { url, language } = await resolveSubtitle(selectCaptionTrack( + await getSubtitleTracks(hlsVariant.url), subtitleLang + )) || {}; subtitles = url; if (language) fileMetadata = { sublanguage: language }; } diff --git a/api/src/processing/services/vimeo.js b/api/src/processing/services/vimeo.js index c12269bc..493b74cf 100644 --- a/api/src/processing/services/vimeo.js +++ b/api/src/processing/services/vimeo.js @@ -2,6 +2,11 @@ import HLS from "hls-parser"; import { env } from "../../config.js"; import { merge } from '../../misc/utils.js'; import { getCookie } from "../cookie/manager.js"; +import { + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; const resolutionMatch = { "3840": 2160, @@ -76,6 +81,15 @@ const compareQuality = (rendition, requestedQuality) => { return Math.abs(quality - requestedQuality); } +const getPlayerConfig = (id, password) => { + const url = new URL(`https://player.vimeo.com/video/${id}/config`); + if (password) url.searchParams.set("h", password); + + return fetch(url) + .then(r => r.json()) + .catch(() => {}); +} + const getDirectLink = async (data, quality, subtitleLang) => { if (!data.files) return; @@ -102,7 +116,9 @@ const getDirectLink = async (data, quality, subtitleLang) => { subtitles = config.request.text_tracks.find( t => t.lang.startsWith(subtitleLang) ); - subtitles = new URL(subtitles.url, "https://player.vimeo.com/").toString(); + if (subtitles) { + subtitles = new URL(subtitles.url, "https://player.vimeo.com/").toString(); + } } } @@ -183,6 +199,28 @@ const getHLS = async (configURL, obj) => { } export default async function(obj) { + if (obj.isCaptionOnly) { + const config = await getPlayerConfig(obj.id, obj.password); + if (!config) return { error: "fetch.fail" }; + const tracks = (config.request?.text_tracks || []).map(track => ({ + url: track.url, + language: track.lang, + })); + const track = selectCaptionTrack(tracks, obj.captionLanguage); + if (!track) return captionSelectionError(tracks); + + return createCaptionResponse({ + url: new URL(track.url, "https://player.vimeo.com/").toString(), + format: obj.captionFormat, + language: track.language, + service: "vimeo", + id: obj.id, + title: config.video?.title, + author: config.video?.owner?.name, + source: `https://vimeo.com/${obj.id}`, + }); + } + let quality = obj.quality === "max" ? 9000 : Number(obj.quality); if (quality < 240) quality = 240; if (!quality || obj.isAudioOnly) quality = 9000; diff --git a/api/src/processing/services/vk.js b/api/src/processing/services/vk.js index 4e2d8548..2bb90808 100644 --- a/api/src/processing/services/vk.js +++ b/api/src/processing/services/vk.js @@ -1,4 +1,9 @@ import { env } from "../../config.js"; +import { + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"]; @@ -80,7 +85,16 @@ const getVideo = async (ownerId, videoId, accessKey) => { return video; } -export default async function ({ ownerId, videoId, accessKey, quality, subtitleLang }) { +export default async function ({ + ownerId, + videoId, + accessKey, + quality, + subtitleLang, + isCaptionOnly, + captionLanguage, + captionFormat, +}) { const token = await getToken(); if (!token) return { error: "fetch.fail" }; @@ -103,6 +117,26 @@ export default async function ({ ownerId, videoId, accessKey, quality, subtitleL return { error: "content.video.unavailable" }; } + if (isCaptionOnly) { + const tracks = (video.subtitles || []) + .filter(subtitle => subtitle.title?.endsWith(".vtt")) + .map(subtitle => ({ url: subtitle.url, language: subtitle.lang })); + const subtitle = selectCaptionTrack(tracks, captionLanguage); + if (!subtitle) return captionSelectionError(tracks); + + const id = `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`; + return createCaptionResponse({ + url: subtitle.url, + format: captionFormat, + language: subtitle.language, + service: "vk", + id, + title: video.title?.trim(), + source: `https://vk.com/video${id}`, + headers: { "user-agent": vkClientAgent }, + }); + } + if (!video.files || !video.duration) { return { error: "fetch.fail" }; } diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 55385971..7db3566f 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -7,6 +7,12 @@ import { Innertube, Platform, Session } from "youtubei.js"; import { env } from "../../config.js"; import { getCookie } from "../cookie/manager.js"; import { getYouTubeSession } from "../helpers/youtube-session.js"; +import { + captionSegmentResources, + createCaptionResponse, + selectCaptionTrack, + captionSelectionError, +} from "../helpers/captions.js"; // https://github.com/LuanRT/YouTube.js/pull/1052 Platform.shim.eval = async (data) => { @@ -135,12 +141,14 @@ const getHlsVariants = async (hlsManifest, dispatcher) => { return variants; } -const getSubtitles = async (info, dispatcher, subtitleLang) => { - const preferredCap = info.captions.caption_tracks.find(caption => - caption.kind !== 'asr' && caption.language_code.startsWith(subtitleLang) - ); +const captionTracks = info => (info.captions?.caption_tracks || []).map(track => ({ + url: track.base_url, + language: track.language_code, + automatic: track.kind === "asr", +})); - const captionsUrl = preferredCap?.base_url; +const getSubtitles = async (info, dispatcher, preferredCap, allSegments = false) => { + const captionsUrl = preferredCap?.url; if (!captionsUrl) return; if (!captionsUrl.includes("exp=xpe")) { @@ -149,7 +157,7 @@ const getSubtitles = async (info, dispatcher, subtitleLang) => { return { url: url.toString(), - language: preferredCap.language_code, + language: preferredCap.language, } } @@ -158,7 +166,7 @@ const getSubtitles = async (info, dispatcher, subtitleLang) => { // so instead we just use subtitles from HLS const hlsVariants = await getHlsVariants( - info.streaming_data.hls_manifest_url, + info.streaming_data?.hls_manifest_url, dispatcher ); if (hlsVariants?.error) return; @@ -167,22 +175,28 @@ const getSubtitles = async (info, dispatcher, subtitleLang) => { const hlsSubtitles = hlsVariants[0]?.subtitles; if (!hlsSubtitles?.length) return; - const preferredHls = hlsSubtitles.find( - subtitle => subtitle.language.startsWith(subtitleLang) + const preferredHls = selectCaptionTrack( + hlsSubtitles.map(subtitle => ({ ...subtitle, url: subtitle.uri })), + preferredCap.language ); if (!preferredHls) return; + const playlistURL = new URL(preferredHls.uri, info.streaming_data.hls_manifest_url).toString(); const fetchedHlsSubs = - await fetch(preferredHls.uri, { dispatcher }) + await fetch(playlistURL, { dispatcher }) .then(r => r.status === 200 ? r.text() : undefined) .catch(() => {}); + if (!fetchedHlsSubs) return; const parsedSubs = HLS.parse(fetchedHlsSubs); - if (!parsedSubs) return; + if (!parsedSubs?.segments?.length) return; + const urls = allSegments + ? captionSegmentResources(parsedSubs.segments, playlistURL) + : new URL(parsedSubs.segments[0].uri, playlistURL).toString(); return { - url: parsedSubs.segments[0]?.uri, + url: urls, language: preferredHls.language, } } @@ -217,7 +231,7 @@ export default async function (o) { ); // we can get subtitles reliably only from the iOS client - if (o.subtitleLang) { + if (o.subtitleLang || o.isCaptionOnly) { innertubeClient = "IOS"; useSession = false; } @@ -324,7 +338,7 @@ export default async function (o) { return { error: "content.video.live" }; } - if (basicInfo.duration > env.durationLimit) { + if (!o.isCaptionOnly && basicInfo.duration > env.durationLimit) { return { error: "content.too_long" }; } @@ -337,6 +351,26 @@ export default async function (o) { } } + if (o.isCaptionOnly) { + const tracks = captionTracks(info); + const selectedTrack = selectCaptionTrack(tracks, o.captionLanguage || o.subtitleLang); + if (!selectedTrack) return captionSelectionError(tracks); + + const captions = await getSubtitles(info, o.dispatcher, selectedTrack, true); + if (!captions?.url) return { error: "captions.unavailable" }; + + return createCaptionResponse({ + url: captions.url, + format: o.captionFormat, + language: captions.language, + service: "youtube", + id: o.id, + title: basicInfo.title.trim(), + author: basicInfo.author.replace("- Topic", "").trim(), + source: `https://www.youtube.com/watch?v=${o.id}`, + }); + } + const normalizeQuality = res => { const shortestSide = Math.min(res.height, res.width); return videoQualities.find(qual => qual >= shortestSide); @@ -504,7 +538,9 @@ export default async function (o) { } if (o.subtitleLang && !o.isAudioOnly && info.captions?.caption_tracks?.length) { - const videoSubtitles = await getSubtitles(info, o.dispatcher, o.subtitleLang); + const videoSubtitles = await getSubtitles( + info, o.dispatcher, selectCaptionTrack(captionTracks(info), o.subtitleLang, false) + ); if (videoSubtitles) { subtitles = videoSubtitles; } diff --git a/api/src/stream/captions.js b/api/src/stream/captions.js new file mode 100644 index 00000000..10412d46 --- /dev/null +++ b/api/src/stream/captions.js @@ -0,0 +1,81 @@ +import { Agent, request } from "undici"; +import { create as contentDisposition } from "content-disposition-header"; + +import { closeRequest, closeResponse } from "./shared.js"; +import { destroyInternalStream } from "./manage.js"; +import { convertCaptions, mergeCaptionSegments } from "../processing/helpers/caption-formats.js"; + +const defaultAgent = new Agent(); +const maxCaptionSize = 8 * 1024 * 1024; + +const contentTypes = { + md: "text/markdown; charset=utf-8", + srt: "application/x-subrip; charset=utf-8", + txt: "text/plain; charset=utf-8", + vtt: "text/vtt; charset=utf-8", +}; + +export default async function captions(streamInfo, res) { + const abortController = new AbortController(); + const urls = [streamInfo.urls].flat(); + const cleanup = () => { + closeRequest(abortController); + urls.forEach(destroyInternalStream); + }; + res.once("close", cleanup); + res.once("error", cleanup); + + try { + if (res.destroyed) return; + const segments = []; + let size = 0; + for (const url of urls) { + const response = await request(url, { + signal: abortController.signal, + maxRedirections: 4, + dispatcher: defaultAgent, + headers: streamInfo.headers, + }); + + if (response.statusCode < 200 || response.statusCode > 299) { + return closeResponse(res); + } + + const declaredSize = Number(response.headers["content-length"]); + if (size + declaredSize > maxCaptionSize) { + res.status(413).end(); + return; + } + + const chunks = []; + for await (const chunk of response.body) { + size += chunk.length; + if (size > maxCaptionSize) { + res.status(413).end(); + return; + } + chunks.push(chunk); + } + segments.push(Buffer.concat(chunks).toString("utf8")); + } + + const body = convertCaptions( + segments.length === 1 ? segments[0] : mergeCaptionSegments(segments), + streamInfo.captionFormat, + streamInfo.captionMetadata, + streamInfo.captionSourceFormat + ); + + res.setHeader("Cross-Origin-Resource-Policy", "cross-origin"); + res.setHeader("Content-Disposition", contentDisposition(streamInfo.filename)); + res.setHeader("Content-Type", contentTypes[streamInfo.captionFormat]); + res.setHeader("Content-Length", Buffer.byteLength(body)); + res.status(200).send(body); + } catch { + if (!res.destroyed) closeResponse(res); + } finally { + cleanup(); + res.off("close", cleanup); + res.off("error", cleanup); + } +} diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index 93e9e652..4e1bd74f 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -45,6 +45,10 @@ export function createStream(obj) { // url to a subtitle file subtitles: obj.subtitles, + + captionFormat: obj.captionFormat, + captionSourceFormat: obj.captionSourceFormat, + captionMetadata: obj.captionMetadata, }; // FIXME: this is now a Promise, but it is not awaited @@ -259,17 +263,32 @@ const transplantTunnel = async function (dispatcher) { function wrapStream(streamInfo) { const url = streamInfo.urls; + const createWrappedStream = resource => { + if (typeof resource === "string") { + return createInternalStream(resource, streamInfo, streamInfo.type === "captions"); + } + if (streamInfo.type !== "captions" || typeof resource?.url !== "string") { + throw "invalid url"; + } + return createInternalStream( + resource.url, + { + ...streamInfo, + headers: { ...streamInfo.headers, ...resource.headers }, + }, + true + ); + }; + if (streamInfo.originalRequest) { streamInfo.transplant = transplantTunnel.bind(streamInfo); } if (typeof url === 'string') { - streamInfo.urls = createInternalStream(url, streamInfo); + streamInfo.urls = createWrappedStream(url); } else if (Array.isArray(url)) { for (const idx in streamInfo.urls) { - streamInfo.urls[idx] = createInternalStream( - streamInfo.urls[idx], streamInfo - ); + streamInfo.urls[idx] = createWrappedStream(streamInfo.urls[idx]); } } else throw 'invalid urls'; diff --git a/api/src/stream/stream.js b/api/src/stream/stream.js index 1290c029..5e620571 100644 --- a/api/src/stream/stream.js +++ b/api/src/stream/stream.js @@ -1,5 +1,6 @@ import proxy from "./proxy.js"; import ffmpeg from "./ffmpeg.js"; +import captions from "./captions.js"; import { closeResponse } from "./shared.js"; import { internalStream } from "./internal.js"; @@ -10,6 +11,9 @@ export default async function(res, streamInfo) { case "proxy": return await proxy(streamInfo, res); + case "captions": + return await captions(streamInfo, res); + case "internal": return await internalStream(streamInfo.data, res); diff --git a/api/src/util/tests/loom.json b/api/src/util/tests/loom.json index 0ebb4580..4dd38479 100644 --- a/api/src/util/tests/loom.json +++ b/api/src/util/tests/loom.json @@ -30,6 +30,18 @@ "status": "error" } }, + { + "name": "standalone transcript", + "url": "https://www.loom.com/share/aa3d8b08bee74d05af5b42989e9f33e9", + "params": { + "downloadMode": "captions", + "captionFormat": "txt" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "video with no transcodedUrl", "url": "https://www.loom.com/share/aa3d8b08bee74d05af5b42989e9f33e9", diff --git a/api/src/util/tests/rutube.json b/api/src/util/tests/rutube.json index 1052aa92..38173df7 100644 --- a/api/src/util/tests/rutube.json +++ b/api/src/util/tests/rutube.json @@ -8,6 +8,19 @@ "status": "tunnel" } }, + { + "name": "standalone transcript", + "url": "https://rutube.ru/video/76ee456fa0fe79a8e0013b6110cb0168/", + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "captionLanguage": "ru" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "vertical video (isAudioMuted)", "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/", diff --git a/api/src/util/tests/tiktok.json b/api/src/util/tests/tiktok.json index c8dbce8c..bf39bc99 100644 --- a/api/src/util/tests/tiktok.json +++ b/api/src/util/tests/tiktok.json @@ -8,6 +8,19 @@ "status": "tunnel" } }, + { + "name": "standalone English transcript", + "url": "https://www.tiktok.com/@tedtoks/video/7168241231892319531", + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "captionLanguage": "en" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "images", "url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526", @@ -44,4 +57,4 @@ "status": "error" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 512eb376..f6c7c070 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -10,6 +10,20 @@ "status": "redirect" } }, + { + "name": "standalone English transcript", + "url": "https://twitter.com/TwitterA11y/status/1395427851860684801", + "canFail": true, + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "captionLanguage": "en" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "video with mobile web mediaviewer", "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011¤tTweetUser=X¤tTweet=1697304622749086011¤tTweetUser=X", diff --git a/api/src/util/tests/vimeo.json b/api/src/util/tests/vimeo.json index e28af888..3668af3b 100644 --- a/api/src/util/tests/vimeo.json +++ b/api/src/util/tests/vimeo.json @@ -21,6 +21,19 @@ "status": "redirect" } }, + { + "name": "standalone English transcript", + "url": "https://vimeo.com/76979871", + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "captionLanguage": "en" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "1080p dash parcel", "url": "https://vimeo.com/967252742", @@ -63,4 +76,4 @@ "status": "redirect" } } -] \ No newline at end of file +] diff --git a/api/src/util/tests/vk.json b/api/src/util/tests/vk.json index 71720af5..24ae60a2 100644 --- a/api/src/util/tests/vk.json +++ b/api/src/util/tests/vk.json @@ -61,6 +61,19 @@ "status": "tunnel" } }, + { + "name": "standalone transcript", + "url": "https://vkvideo.ru/video-26006257_456245538", + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "captionLanguage": "ru" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "ancient video (fallback to 240p)", "url": "https://vk.com/video-1959_28496479", diff --git a/api/src/util/tests/youtube.json b/api/src/util/tests/youtube.json index cb4964be..4a83fb18 100644 --- a/api/src/util/tests/youtube.json +++ b/api/src/util/tests/youtube.json @@ -240,5 +240,19 @@ "code": 200, "status": "tunnel" } + }, + { + "name": "standalone English transcript", + "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, + "params": { + "downloadMode": "captions", + "captionFormat": "txt", + "subtitleLang": "en" + }, + "expected": { + "code": 200, + "status": "tunnel" + } } -] \ No newline at end of file +] diff --git a/docs/api.md b/docs/api.md index 4ad2839b..3900a4eb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -73,7 +73,9 @@ all keys except for `url` are optional. value options are separated by `/`. | `url` | `string` | source URL | *required* | | `audioBitrate` | `string` | `320 / 256 / 128 / 96 / 64 / 8` (kbps) | `128` | | `audioFormat` | `string` | `best / mp3 / ogg / wav / opus` | `mp3` | -| `downloadMode` | `string` | `auto / audio / mute` | `auto` | +| `downloadMode` | `string` | `auto / audio / mute / captions` | `auto` | +| `captionFormat` | `string` | `txt / vtt / srt / md` | `txt` | +| `captionLanguage` | `string` | any valid ISO 639-1 language code | *none* | | `filenameStyle` | `string` | `classic / pretty / basic / nerdy` | `basic` | | `videoQuality` | `string` | `max / 4320 / 2160 / 1440 / 1080 / 720 / 480 / 360 / 240 / 144` | `1080` | | `disableMetadata` | `boolean` | title, artist, and other info will not be added to the file | `false` | @@ -93,6 +95,44 @@ all keys except for `url` are optional. value options are separated by `/`. | `youtubeBetterAudio` | `boolean` | prefer higher quality youtube audio if possible | `false` | | `youtubeHLS` | `boolean` | use HLS formats when downloading from youtube | `false` | +#### standalone captions + +set `downloadMode` to `captions` to download a caption or transcript file without +downloading video or audio. supported services are listed in the +[api readme](/api/README.md#supported-services). + +`captionLanguage` selects the caption language, falling back to `subtitleLang` when +omitted. if both are omitted, cobalt chooses an available track. when a service +identifies automatic captions, manually provided tracks take priority. + +when a language is provided, an exact match takes priority over compatible regional +variants, with manual captions preferred within each tier. explicitly requested +writing systems are preserved, so `zh-Hant` will not fall back to `zh-Hans`. + +if captions exist but none match, the api returns +`error.api.captions.language_unavailable`. omit both language fields to retry with an +available track. if no downloadable tracks exist, the error is +`error.api.captions.unavailable`. + +`txt` and `md` produce readable transcripts without timestamps. `vtt` preserves the +original webvtt cues, while `srt` converts those cues to the subrip format. + +example request: + +```json +{ + "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + "downloadMode": "captions", + "captionFormat": "txt", + "subtitleLang": "en" +} +``` + +the response is a regular [`tunnel`](#tunnelredirect-response) whose filename uses +the selected extension. caption tunnels use the same expiry and signature checks as +media tunnels. caption responses also include a `text` tunnel for copying a plain, +untimed transcript. for `txt` requests, `url` and `text` are the same. + ### response body type: `application/json` @@ -109,6 +149,7 @@ the response will always be a JSON object containing the `status` key, which is | `status` | `string` | `tunnel / redirect` | | `url` | `string` | url for the cobalt tunnel, or redirect to an external link | | `filename` | `string` | cobalt-generated filename for the file being downloaded | +| `text` | `string` | plain-text transcript tunnel for caption responses (optional) | ### local processing response | key | type | value | @@ -186,6 +227,7 @@ all keys in this table are optional. |:-------------|:---------|:----------------------------------------------------------------------------| | `service` | `string` | origin service (optional) | | `limit` | `number` | the maximum downloadable video duration or the rate limit window (optional) | +| `languages` | `string[]` | available caption languages when the requested language is missing (optional) | ## POST `/session` used for generating JWT tokens, if enabled. currently, cobalt only supports diff --git a/web/i18n/en/button.json b/web/i18n/en/button.json index 643f4b8e..5be8adea 100644 --- a/web/i18n/en/button.json +++ b/web/i18n/en/button.json @@ -7,6 +7,7 @@ "download": "download", "share": "share", "copy": "copy", + "copy.text": "copy text", "copy.section": "copy the section link", "copied": "copied", "import": "import", @@ -25,5 +26,6 @@ "clear_cache": "clear cache", "remove": "remove", "retry": "retry", - "delete": "delete" + "delete": "delete", + "download.transcript_any_language": "download anyway" } diff --git a/web/i18n/en/dialog.json b/web/i18n/en/dialog.json index 44e92dab..70304ee6 100644 --- a/web/i18n/en/dialog.json +++ b/web/i18n/en/dialog.json @@ -10,6 +10,7 @@ "saving.title": "choose how to save", "saving.blocked": "cobalt tried opening the file in a new tab, but your browser blocked it. you can allow pop-ups for cobalt to prevent this from happening next time.", "saving.timeout": "cobalt tried saving the file automatically, but your browser stopped it. you have to select a preferred method manually.", + "saving.copy_failed": "your browser wouldn't let cobalt copy the transcript. check clipboard permissions and try again.", "safety.title": "important safety notice", diff --git a/web/i18n/en/error/api.json b/web/i18n/en/error/api.json index 186b45a4..5329500a 100644 --- a/web/i18n/en/error/api.json +++ b/web/i18n/en/error/api.json @@ -25,6 +25,7 @@ "service.unsupported": "this service is not supported yet. have you pasted the right link?", "service.disabled": "this service is generally supported by cobalt, but it's disabled on this processing instance. try a link from another service!", "service.audio_not_supported": "this service doesn't support audio extraction. try a link from another service!", + "service.captions_not_supported": "this service doesn't support standalone transcripts.", "link.invalid": "your link is invalid or this service is not supported yet. have you pasted the right link?", "link.unsupported": "{{ service }} is supported, but i couldn't recognize your link. have you pasted the right one?", @@ -59,5 +60,9 @@ "youtube.api_error": "youtube updated something about its api and i couldn't get any info about this video. try again in a few seconds, but if this issue sticks, please report it!", "youtube.disabled_main_instance": "youtube downloading is disabled on the main instance due to restrictions from youtube's side and infinite maintenance cost at scale.\n\nwe apologize for the inconvenience and encourage you to host your own API instance for this.", "youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!", - "youtube.no_session_tokens": "couldn't get required session tokens for youtube. this may be caused by a restriction on youtube's side. try again in a few seconds, but if this issue sticks, please report it!" + "youtube.no_session_tokens": "couldn't get required session tokens for youtube. this may be caused by a restriction on youtube's side. try again in a few seconds, but if this issue sticks, please report it!", + "youtube.captions_unavailable": "this video doesn't have any captions i can download.", + "youtube.caption_language_unavailable": "this video doesn't have captions in the selected language. available languages: {{ languages }}.", + "captions.unavailable": "this video doesn't have any captions i can download.", + "captions.language_unavailable": "this video doesn't have captions in your preferred language. you can download an available transcript instead." } diff --git a/web/i18n/en/save.json b/web/i18n/en/save.json index 91716ab4..603dcfb0 100644 --- a/web/i18n/en/save.json +++ b/web/i18n/en/save.json @@ -4,6 +4,7 @@ "auto": "auto", "audio": "audio", "mute": "mute", + "captions": "transcript", "input.placeholder": "paste the link here", "terms.note.agreement": "by continuing, you agree to", "terms.note.link": "terms and ethics of use", diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index 6f0463bd..6f3be3fe 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -65,8 +65,19 @@ "subtitles": "subtitles", "subtitles.title": "preferred subtitle language", - "subtitles.description": "cobalt will add subtitles to the downloaded file in the preferred language if they're available.\n\nsome services don't have a language selection, and if that's the case, cobalt will add the only available subtitle track if you have any language selected.", + "subtitles.description": "cobalt will add subtitles to downloaded media in the preferred language when they're available.", "subtitles.none": "none", + "captions": "transcripts", + "captions.format": "transcript file format", + "captions.format.description": "plain text and markdown omit timestamps. webvtt and subrip keep caption timing.", + "captions.format.txt": "plain text (.txt)", + "captions.format.vtt": "webvtt (.vtt)", + "captions.format.srt": "subrip (.srt)", + "captions.format.md": "markdown (.md)", + "captions.method": "transcript saving method", + "captions.method.description": "how to save transcripts from the save button. \"ask\" shows both options every time.", + "captions.language": "preferred transcript language", + "captions.language.description": "pick a preferred transcript language, or let cobalt choose an available track automatically. if your preferred language isn't available, cobalt will ask before downloading another language.", "audio.youtube.better_audio": "youtube audio quality", "audio.youtube.better_audio.title": "prefer better quality", @@ -157,5 +168,6 @@ "tabs": "navigation", "tabs.hide_remux": "hide the remux tab", - "tabs.hide_remux.description": "if you don't use the remux tool, you can hide it from the navigation bar." + "tabs.hide_remux.description": "if you don't use the remux tool, you can hide it from the navigation bar.", + "captions.language.automatic": "automatic" } diff --git a/web/src/components/dialog/DialogButton.svelte b/web/src/components/dialog/DialogButton.svelte index 69aaf722..f1aa3203 100644 --- a/web/src/components/dialog/DialogButton.svelte +++ b/web/src/components/dialog/DialogButton.svelte @@ -55,7 +55,9 @@ .popup-button { width: 100%; - height: 40px; + min-height: 40px; + height: auto; + padding-block: 8px; transition: 0.2s opacity; } diff --git a/web/src/components/dialog/DialogContainer.svelte b/web/src/components/dialog/DialogContainer.svelte index dc1bfb76..bb1a8fd2 100644 --- a/web/src/components/dialog/DialogContainer.svelte +++ b/web/src/components/dialog/DialogContainer.svelte @@ -22,7 +22,7 @@ // check if dialog parent is still present if (dialogParent) { dialogParent.close(); - killDialog(); + killDialog(id); } }, 150); } diff --git a/web/src/components/dialog/DialogHolder.svelte b/web/src/components/dialog/DialogHolder.svelte index b519340d..363e8018 100644 --- a/web/src/components/dialog/DialogHolder.svelte +++ b/web/src/components/dialog/DialogHolder.svelte @@ -15,7 +15,7 @@ -->
- {#each $dialogs as dialog} + {#each $dialogs as dialog (dialog.id)} {#if dialog.type === "small"} {:else if dialog.type === "picker"} diff --git a/web/src/components/dialog/NoScriptDialog.svelte b/web/src/components/dialog/NoScriptDialog.svelte index c9ba9c2e..45ef1595 100644 --- a/web/src/components/dialog/NoScriptDialog.svelte +++ b/web/src/components/dialog/NoScriptDialog.svelte @@ -4,7 +4,7 @@ {#if !browser} -
{/if} + + {#if copyFailed} +
+ {$t("dialog.saving.copy_failed")} +
+ {/if} ; @@ -223,6 +224,14 @@ {$t("save.mute")} + + + {$t("save.captions")} + diff --git a/web/src/lib/api/saving-handler.ts b/web/src/lib/api/saving-handler.ts index 94c0319e..6fb168e7 100644 --- a/web/src/lib/api/saving-handler.ts +++ b/web/src/lib/api/saving-handler.ts @@ -11,6 +11,7 @@ import { downloadButtonState } from "$lib/state/omnibox"; import { createSavePipeline } from "$lib/task-manager/queue"; import type { CobaltSaveRequestBody } from "$lib/types/api"; +import type { DialogButton } from "$lib/types/dialog"; type SavingHandlerArgs = { url?: string, @@ -18,18 +19,19 @@ type SavingHandlerArgs = { oldTaskId?: string } -export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerArgs) => { +export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerArgs): Promise => { downloadButtonState.set("think"); - const error = (errorText: string) => { + const error = (errorText: string, retry?: DialogButton) => { return createDialog({ id: "save-error", type: "small", meowbalt: "error", buttons: [ + ...(retry ? [retry] : []), { text: get(t)("button.gotit"), - main: true, + main: !retry, action: () => {}, }, ], @@ -41,7 +43,7 @@ export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerAr if (!request && !url) return; - const selectedRequest = request || { + const selectedRequest = request ? { ...request } : { url: url!, // not lazy cuz default depends on device capabilities @@ -49,6 +51,7 @@ export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerAr alwaysProxy: getSetting("save", "alwaysProxy"), downloadMode: getSetting("save", "downloadMode"), + captionFormat: getSetting("save", "captionFormat"), subtitleLang: getSetting("save", "subtitleLang"), filenameStyle: getSetting("save", "filenameStyle"), @@ -69,6 +72,15 @@ export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerAr convertGif: getSetting("save", "convertGif"), } + if (!request && selectedRequest.downloadMode === "captions") { + // Transcript preferences are independent of embedded media subtitles. + delete selectedRequest.subtitleLang; + const captionLang = getSetting("save", "captionLang"); + if (captionLang && captionLang !== "none") { + selectedRequest.captionLanguage = captionLang; + } + } + const response = await API.request(selectedRequest); if (!response) { @@ -79,8 +91,23 @@ export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerAr if (response.status === "error") { downloadButtonState.set("error"); + const canRetry = selectedRequest.downloadMode === "captions" + && [ + "error.api.captions.language_unavailable", + "error.api.youtube.caption_language_unavailable", + ].includes(response.error.code); return error( - get(t)(response.error.code, response?.error?.context) + get(t)(response.error.code, response?.error?.context), + canRetry ? { + text: get(t)("button.download.transcript_any_language"), + main: true, + action: () => { + const retry = { ...selectedRequest }; + delete retry.captionLanguage; + delete retry.subtitleLang; + return savingHandler({ request: retry, oldTaskId }); + }, + } : undefined, ); } @@ -103,6 +130,12 @@ export const savingHandler = async ({ url, request, oldTaskId }: SavingHandlerAr return downloadFile({ url: response.url, + method: selectedRequest.downloadMode === "captions" + ? get(settings).save.transcriptMethod + : undefined, + copyTextURL: selectedRequest.downloadMode === "captions" + ? response.text || response.url + : undefined, }); } else { downloadButtonState.set("error"); diff --git a/web/src/lib/download.ts b/web/src/lib/download.ts index efcf50ee..52dd4d47 100644 --- a/web/src/lib/download.ts +++ b/web/src/lib/download.ts @@ -8,11 +8,14 @@ import { createDialog } from "$lib/state/dialogs"; import type { DialogInfo } from "$lib/types/dialog"; import type { CobaltFileUrlType } from "$lib/types/api"; +import type { CobaltSettings } from "$lib/types/settings"; type DownloadFileParams = { url?: string, file?: File, urlType?: CobaltFileUrlType, + copyTextURL?: string, + method?: CobaltSettings['save']['transcriptMethod'], } type SavingDialogParams = { @@ -20,15 +23,17 @@ type SavingDialogParams = { file?: File, body?: string, urlType?: CobaltFileUrlType, + copyTextURL?: string, } -const openSavingDialog = ({ url, file, body, urlType }: SavingDialogParams) => { +const openSavingDialog = ({ url, file, body, urlType, copyTextURL }: SavingDialogParams) => { const dialogData: DialogInfo = { type: "saving", id: "saving", file, url, urlType, + copyTextURL, } if (body) dialogData.bodyText = body; @@ -75,13 +80,20 @@ export const copyURL = async (url: string) => { return await navigator?.clipboard?.writeText(url); } -export const downloadFile = ({ url, file, urlType }: DownloadFileParams) => { +export const copyTextFromURL = async (url: string) => { + if (!navigator?.clipboard?.writeText) throw new Error("clipboard unavailable"); + const response = await fetch(url); + if (!response.ok) throw new Error("couldn't fetch transcript"); + return await navigator.clipboard.writeText(await response.text()); +} + +export const downloadFile = async ({ url, file, urlType, copyTextURL, method }: DownloadFileParams) => { if (!url && !file) throw new Error("attempted to download void"); - const pref = get(settings).save.savingMethod; + const pref = method ?? get(settings).save.savingMethod; if (pref === "ask") { - return openSavingDialog({ url, file, urlType }); + return openSavingDialog({ url, file, urlType, copyTextURL }); } /* @@ -100,7 +112,8 @@ export const downloadFile = ({ url, file, urlType }: DownloadFileParams) => { url, file, body: get(t)("dialog.saving.timeout"), - urlType + urlType, + copyTextURL, }); } @@ -134,10 +147,12 @@ export const downloadFile = ({ url, file, urlType }: DownloadFileParams) => { && !(device.is.iOS && urlType === "redirect")) { return openURL(url); } else if (pref === "copy" && !file) { - return copyURL(url); + return copyTextURL + ? await copyTextFromURL(copyTextURL) + : await copyURL(url); } } } catch { /* catch & ignore */ } - return openSavingDialog({ url, file, urlType }); + return openSavingDialog({ url, file, urlType, copyTextURL }); } diff --git a/web/src/lib/i18n/translations.ts b/web/src/lib/i18n/translations.ts index 9a65c761..a48a2bc1 100644 --- a/web/src/lib/i18n/translations.ts +++ b/web/src/lib/i18n/translations.ts @@ -29,6 +29,7 @@ const config: Config<{ formats?: string; limit?: number; service?: string; + languages?: string[]; }> = { fallbackLocale: defaultLocale, translations: Object.keys(parsedLocfiles).reduce((obj, lang) => { diff --git a/web/src/lib/settings/defaults.ts b/web/src/lib/settings/defaults.ts index ce3114ec..9b58bfa5 100644 --- a/web/src/lib/settings/defaults.ts +++ b/web/src/lib/settings/defaults.ts @@ -3,7 +3,7 @@ import { defaultLocale } from "$lib/i18n/translations"; import type { CobaltSettings } from "$lib/types/settings"; const defaultSettings: CobaltSettings = { - schemaVersion: 6, + schemaVersion: 7, advanced: { debug: false, useWebCodecs: false, @@ -28,6 +28,9 @@ const defaultSettings: CobaltSettings = { audioFormat: "mp3", disableMetadata: false, downloadMode: "auto", + captionFormat: "txt", + captionLang: "none", + transcriptMethod: "download", filenameStyle: "basic", savingMethod: "download", allowH265: false, diff --git a/web/src/lib/settings/migrate.ts b/web/src/lib/settings/migrate.ts index d033a6f9..8df26235 100644 --- a/web/src/lib/settings/migrate.ts +++ b/web/src/lib/settings/migrate.ts @@ -6,6 +6,7 @@ import type { CobaltSettingsV4, CobaltSettingsV5, CobaltSettingsV6, + CobaltSettingsV7, } from "$lib/types/settings"; import { getBrowserLanguage } from "$lib/settings/audio-sub-language"; @@ -95,6 +96,12 @@ const migrations: Record = { return out as AllPartialSettingsWithSchema; }, + + [7]: (settings: AllPartialSettingsWithSchema) => { + const out = settings as RecursivePartial; + out.schemaVersion = 7; + return out as AllPartialSettingsWithSchema; + }, }; export const migrate = (settings: AllPartialSettingsWithSchema): PartialSettings => { diff --git a/web/src/lib/settings/validate.ts b/web/src/lib/settings/validate.ts index a711f9ae..73d5acb3 100644 --- a/web/src/lib/settings/validate.ts +++ b/web/src/lib/settings/validate.ts @@ -7,9 +7,11 @@ import { themeOptions, videoQualityOptions, youtubeVideoCodecOptions, + captionFormatOptions, + transcriptMethodOptions, type PartialSettings, } from '$lib/types/settings'; -import { youtubeDubLanguages } from '$lib/settings/audio-sub-language'; +import { subtitleLanguages, youtubeDubLanguages } from '$lib/settings/audio-sub-language'; function validateTypes(input: unknown, reference = defaultSettings as unknown) { if (typeof input === 'undefined') @@ -77,6 +79,9 @@ export function validateSettings(settings: PartialSettings) { && validateLiterals([ [ settings?.appearance?.theme , themeOptions ], [ settings?.save?.downloadMode , downloadModeOptions ], + [ settings?.save?.captionFormat , captionFormatOptions ], + [ settings?.save?.captionLang , subtitleLanguages ], + [ settings?.save?.transcriptMethod , transcriptMethodOptions ], [ settings?.save?.filenameStyle , filenameStyleOptions ], [ settings?.save?.videoQuality , videoQualityOptions ], [ settings?.save?.youtubeVideoCodec, youtubeVideoCodecOptions ], diff --git a/web/src/lib/state/dialogs.ts b/web/src/lib/state/dialogs.ts index a186be04..08b40d16 100644 --- a/web/src/lib/state/dialogs.ts +++ b/web/src/lib/state/dialogs.ts @@ -2,6 +2,7 @@ import { readable, type Updater } from "svelte/store"; import type { DialogInfo } from "$lib/types/dialog"; let update: (_: Updater) => void; +let nextId = 0; export default readable( [], @@ -10,14 +11,15 @@ export default readable( export function createDialog(newData: DialogInfo) { update((popups) => { - popups.push(newData); + popups.push({ ...newData, id: `${newData.id}-${nextId++}` }); return popups; }); } -export function killDialog() { +export function killDialog(id?: string) { update((popups) => { - popups.pop() + const index = id ? popups.findIndex(popup => popup.id === id) : popups.length - 1; + if (index >= 0) popups.splice(index, 1); return popups; }); } diff --git a/web/src/lib/types/api.ts b/web/src/lib/types/api.ts index f127d02e..4729049a 100644 --- a/web/src/lib/types/api.ts +++ b/web/src/lib/types/api.ts @@ -15,6 +15,7 @@ export type CobaltErrorResponse = { context?: { service?: string, limit?: number, + languages?: string[], } }, }; @@ -41,6 +42,7 @@ type CobaltRedirectResponse = { type CobaltTunnelResponse = { status: CobaltResponseType.Tunnel, + text?: string, } & CobaltPartialURLResponse; export const CobaltFileMetadataKeys = [ @@ -113,7 +115,7 @@ export type CobaltServerInfo = { // this allows for extra properties, which is not ideal, // but i couldn't figure out how to make a strict partial :( export type CobaltSaveRequestBody = - { url: string } & Partial>; + { url: string, captionLanguage?: string } & Partial>; export type CobaltSessionResponse = CobaltSession | CobaltErrorResponse; export type CobaltServerInfoResponse = CobaltServerInfo | CobaltErrorResponse; diff --git a/web/src/lib/types/dialog.ts b/web/src/lib/types/dialog.ts index c0721f97..0110fb5b 100644 --- a/web/src/lib/types/dialog.ts +++ b/web/src/lib/types/dialog.ts @@ -46,6 +46,7 @@ type SavingDialog = Dialog & { url?: string, file?: File, urlType?: CobaltFileUrlType, + copyTextURL?: string, }; export type DialogInfo = SmallDialog | PickerDialog | SavingDialog; diff --git a/web/src/lib/types/settings.ts b/web/src/lib/types/settings.ts index 1d184eec..6825a4eb 100644 --- a/web/src/lib/types/settings.ts +++ b/web/src/lib/types/settings.ts @@ -4,16 +4,19 @@ import type { CobaltSettingsV3 } from "$lib/types/settings/v3"; import type { CobaltSettingsV4 } from "$lib/types/settings/v4"; import type { CobaltSettingsV5 } from "$lib/types/settings/v5"; import type { CobaltSettingsV6 } from "$lib/types/settings/v6"; +import type { CobaltSettingsV7 } from "$lib/types/settings/v7"; export * from "$lib/types/settings/v2"; export * from "$lib/types/settings/v3"; export * from "$lib/types/settings/v4"; export * from "$lib/types/settings/v5"; export * from "$lib/types/settings/v6"; +export * from "$lib/types/settings/v7"; +export { downloadModeOptions } from "$lib/types/settings/v7"; -export type CobaltSettings = CobaltSettingsV6; +export type CobaltSettings = CobaltSettingsV7; -export type AnyCobaltSettings = CobaltSettingsV5 | CobaltSettingsV4 | CobaltSettingsV3 | CobaltSettingsV2 | CobaltSettings; +export type AnyCobaltSettings = CobaltSettingsV6 | CobaltSettingsV5 | CobaltSettingsV4 | CobaltSettingsV3 | CobaltSettingsV2 | CobaltSettings; export type PartialSettings = RecursivePartial; diff --git a/web/src/lib/types/settings/v7.ts b/web/src/lib/types/settings/v7.ts new file mode 100644 index 00000000..799da71f --- /dev/null +++ b/web/src/lib/types/settings/v7.ts @@ -0,0 +1,17 @@ +import type { SubtitleLang } from "$lib/settings/audio-sub-language"; +import type { CobaltSettingsV6 } from "$lib/types/settings/v6"; +import { downloadModeOptions as previousDownloadModes } from "$lib/types/settings/v2"; + +export const downloadModeOptions = [...previousDownloadModes, "captions"] as const; +export const captionFormatOptions = ["txt", "vtt", "srt", "md"] as const; +export const transcriptMethodOptions = ["ask", "download", "copy"] as const; + +export type CobaltSettingsV7 = Omit & { + schemaVersion: 7, + save: Omit & { + downloadMode: typeof downloadModeOptions[number], + captionFormat: typeof captionFormatOptions[number], + captionLang: SubtitleLang, + transcriptMethod: typeof transcriptMethodOptions[number], + }, +}; diff --git a/web/src/routes/settings/metadata/+page.svelte b/web/src/routes/settings/metadata/+page.svelte index dfe02eef..0431a3f2 100644 --- a/web/src/routes/settings/metadata/+page.svelte +++ b/web/src/routes/settings/metadata/+page.svelte @@ -3,7 +3,12 @@ import { t } from "$lib/i18n/translations"; import { namedSubtitleLanguages } from "$lib/settings/audio-sub-language"; - import { filenameStyleOptions, savingMethodOptions } from "$lib/types/settings"; + import { + captionFormatOptions, + filenameStyleOptions, + savingMethodOptions, + transcriptMethodOptions, + } from "$lib/types/settings"; import SettingsCategory from "$components/settings/SettingsCategory.svelte"; import Switcher from "$components/buttons/Switcher.svelte"; @@ -13,6 +18,16 @@ import SettingsDropdown from "$components/settings/SettingsDropdown.svelte"; const displayLangs = namedSubtitleLanguages($t); + const displayCaptionLangs: Record = { + ...displayLangs, + none: $t("settings.captions.language.automatic"), + }; + const displayCaptionFormats = Object.fromEntries( + captionFormatOptions.map(format => [format, $t(`settings.captions.format.${format}`)]) + ); + const displayTranscriptMethods = Object.fromEntries( + transcriptMethodOptions.map(method => [method, $t(`settings.saving.${method}`)]) + ); @@ -35,6 +50,39 @@ + + + + + + {#each savingMethodOptions as value}