diff --git a/api/README.md b/api/README.md index 968e3a18..c65bbfbc 100644 --- a/api/README.md +++ b/api/README.md @@ -26,6 +26,7 @@ if the desired service isn't supported yet, feel free to create an appropriate i | newgrounds | ✅ | ✅ | ✅ | ✅ | ✅ | | ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | +| pr0gramm (pr0.app)| ✅ | ✅ | ✅ | ➖ | ✅ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | | rutube | ✅ | ✅ | ✅ | ✅ | ✅ | | snapchat | ✅ | ✅ | ✅ | ➖ | ➖ | @@ -51,6 +52,7 @@ if the desired service isn't supported yet, feel free to create an appropriate i | instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. | | facebook | supports public accessible videos content only. | | pinterest | supports photos, gifs, videos and stories. | +| pr0gramm | supports images and videos, in the h264 and vp9 variants that the site itself offers. also accepts pr0.app links. | | reddit | supports gifs and videos. | | snapchat | supports spotlights and stories. lets you pick what to save from stories. | | rutube | supports yappy & private links. | diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index ff7175c9..206fa9ba 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -183,6 +183,13 @@ export default function({ params = { type: "proxy" }; break; + /* pr0gramm media is served over plain cdn urls, so both a + redirect and a tunnel work. the redirect is preferred and + `alwaysProxy` turns it into a tunnel when needed. */ + case "pr0gramm": + responseType = "redirect"; + break; + case "facebook": case "instagram": case "tumblr": diff --git a/api/src/processing/match.js b/api/src/processing/match.js index e97f7600..1aca84af 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -29,6 +29,7 @@ import loom from "./services/loom.js"; import facebook from "./services/facebook.js"; import bluesky from "./services/bluesky.js"; import newgrounds from "./services/newgrounds.js"; +import pr0gramm from "./services/pr0gramm.js"; let freebind; @@ -266,6 +267,14 @@ export default async function({ host, patternMatch, params, authType }) { }); break; + case "pr0gramm": + r = await pr0gramm({ + ...patternMatch, + quality: params.videoQuality, + isAudioOnly, + }); + break; + default: return createResponse("error", { code: "error.api.service.unsupported" diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index fbe25805..586440c7 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -75,6 +75,19 @@ export const services = { "url_shortener/:shortLink" ], }, + pr0gramm: { + /* pr0gramm.com and pr0.app share the same url layout, and an item link + is *always* "…/" — no matter whether it was shared from a + feed (/top/, /new/, /junk/), a profile + (/user//) or a collection (/collections//). + + pr0.app additionally redirects unknown path segments to /new/ + (it puts that into its og:url meta tag), so the pattern stays as + permissive as pr0gramm itself and the id is validated numerially in + service-patterns.js instead. */ + patterns: ["**/:id"], + altDomains: ["pr0.app"], + }, newgrounds: { patterns: [ "portal/view/:id", diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index 07bef989..ad4a2007 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -36,6 +36,10 @@ export const testers = { pattern.id?.length <= 128 || pattern.shortLink?.length <= 32, + "pr0gramm": pattern => + // item ids are numeric and can be anywhere up to int64 + pattern.id?.length <= 19 && /^\d+$/.test(pattern.id || ""), + "reddit": pattern => pattern.id?.length <= 16 && !pattern.sub && !pattern.user || (pattern.sub?.length <= 22 && pattern.id?.length <= 16) || diff --git a/api/src/processing/services/pr0gramm.js b/api/src/processing/services/pr0gramm.js new file mode 100644 index 00000000..72576b92 --- /dev/null +++ b/api/src/processing/services/pr0gramm.js @@ -0,0 +1,201 @@ +import { strict as assert } from "node:assert"; + +import { genericUserAgent } from "../../config.js"; + +/* + pr0gramm is an imageboard. every upload ("item") is a single image or a video, + and all media is served from plain cdn hosts without any login. + + the api returns bare paths (e.g. `2026/10/09/-h264-sd.mp4`), so the host + has to be picked based on what is being downloaded: + + - `image` / `fullsize` -> img.pr0gramm.com (images) + - `image` (when the item is a video) / `variants` -> vid.pr0gramm.com + - `thumb` -> thumb.pr0gramm.com (smaller re-encode, jpeg/webp) + + videos come as h264 and vp9 variants. h264 always has audio, vp9 usually + doesn't, so h264 is used for audio downloads. +*/ + +const mediaHosts = { + image: "https://img.pr0gramm.com/", + video: "https://vid.pr0gramm.com/", + thumb: "https://thumb.pr0gramm.com/", +}; + +export const mediaURL = (path, host = "image") => { + assert(mediaHosts[host]); + return new URL(path, mediaHosts[host]).toString(); +}; + +const videoExtensions = ["mp4", "webm", "mov", "mkv"]; + +const fileExtension = (path) => path?.split(".").pop().toLowerCase(); + +// videos live on vid. and images on img., and both hosts are happy to serve +// the other type — but only with the extension the path actually has, so it +// has to be taken from the path itself +export const mediaURLFor = (path) => mediaURL( + path, + videoExtensions.includes(fileExtension(path)) ? "video" : "image" +); + +/* + pr0gramm does not always keep its media paths up to date: an item that was + converted after its upload has the old path in `fullsize` (e.g. a `.png` + that is really served as `.jpg`), and a video without any h264/vp9 variant + has only its `image` path. so the reachable one is probed instead of + trusting whichever field happens to be filled. +*/ +const pickReachableMedia = async (candidates) => { + for (const path of candidates) { + if (!path) continue; + + const url = mediaURLFor(path), + reachable = await fetch(url, { + method: "HEAD", + headers: { "user-agent": genericUserAgent } + }).then(r => r.ok).catch(() => false); + + if (reachable) { + return { url, extension: fileExtension(path) }; + } + } +}; + +/* + variants are named `-`, and the height classes are + consistent across qualities (sd is always 480p, hd always 720p, ...), so they + are a more reliable source for picking a quality than the api heights. +*/ +const heightClasses = { + low_res: 360, + sd: 480, + hd: 720, + full_hd: 1080, + ultra_hd: 2160, + "4k": 2160, + "8k": 4320, +}; + +const videoTypes = [ + { mimeType: "video/mp4", extension: "mp4", codecs: ["h264", "hevc", "av1"] }, + { mimeType: "video/webm", extension: "webm", codecs: ["vp9", "av1"] }, +]; + +const getVideoType = (variant) => + videoTypes.find(type => type.mimeType === variant?.mimeType); + +export const selectVariant = (variants, quality, codec = "h264") => { + const parsed = (variants || []) + .filter(variant => variant?.path && getVideoType(variant)) + .map(variant => ({ + ...variant, + height: heightClasses[variant.name?.split("-").pop()] + || variant.height + || 0, + })) + .sort((a, b) => a.height - b.height); + + if (!parsed.length) return; + + // prefer the requested codec, but never fail just because it's missing + const sameCodec = parsed.filter(variant => variant.codec === codec); + const candidates = sameCodec.length ? sameCodec : parsed; + + const target = quality === "max" ? Infinity : parseInt(quality) || 1080; + return candidates.findLast(variant => variant.height <= target) + ?? candidates[0]; +}; + +const getFilename = (item, extension, qualityLabel, resolution) => ({ + filenameAttributes: { + service: "pr0gramm", + id: `${item.id}`, + title: `pr0gramm ${item.id}`, + ...(item.user && { author: item.user }), + ...(extension && { extension }), + ...(qualityLabel && { qualityLabel: `${qualityLabel}p` }), + ...(resolution?.width && { + resolution: `${resolution.width}x${resolution.height}`, + }), + }, + fileMetadata: { + title: `${item.user ? `${item.user} - ` : ""}pr0gramm ${item.id}`, + ...(item.user && { artist: item.user }), + ...(item.created && { + date: new Date(item.created * 1000).toISOString().split("T")[0], + }), + }, +}); + +async function getItem(id) { + const api = new URL("https://pr0gramm.com/api/items/get"); + api.searchParams.set("id", id); + // `flags` is required by the api. 1 = sfw, the only flag that is + // visible without an account + api.searchParams.set("flags", "1"); + + const response = await fetch(api, { + headers: { "user-agent": genericUserAgent } + }).catch(() => {}); + + if (!response?.ok) return; + + const data = await response.json().catch(() => {}); + if (data?.error) return; + + // the api returns items *starting* at the requested id, so it has to be + // looked up explicitly. `id` is a number in the response + return data?.items?.find(item => item.id === +id); +} + +export default async function ({ id, quality, isAudioOnly }) { + const item = await getItem(id); + if (!item) return { error: "fetch.empty" }; + + const video = selectVariant( + item.variants, + isAudioOnly ? "max" : quality, + "h264" + ); + + if (isAudioOnly) { + if (!video) return { error: "fetch.empty" }; + + return { + urls: mediaURL(video.path, "video"), + audioFilename: `pr0gramm_${item.id}_audio`, + fileMetadata: getFilename(item).fileMetadata, + }; + } + + if (!video) { + // uploads without any video variant are images + const photo = await pickReachableMedia([ item.image, item.fullsize ]); + if (!photo) return { error: "fetch.empty" }; + + return { + urls: photo.url, + // photos have no quality labels, so a plain filename reads better + // than the "… ()" that the filename styles would produce + filename: `pr0gramm_${item.id}.${photo.extension}`, + // the rare video without any variants is still a video, and + // shouldn't be saved as a photo + ...(!videoExtensions.includes(photo.extension) && { isPhoto: true }), + }; + } + + const type = getVideoType(video), + // pr0gramm transcribes audio uploads and renders the default subtitle + // against the sd variant + subtitle = item.audio + ? item.subtitles?.find(sub => sub.isDefault) + : undefined; + + return { + urls: mediaURL(video.path, "video"), + ...getFilename(item, type.extension, video.height, video), + ...(subtitle && { subtitles: mediaURL(subtitle.path, "video") }), + }; +} diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 26df685b..29adbbb3 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -31,6 +31,12 @@ function aliasURL(url) { } break; + case "pr0": + if (services.pr0gramm.altDomains.includes(url.hostname)) { + url.hostname = 'pr0gramm.com'; + } + break; + case "pin": if (url.hostname === 'pin.it' && parts.length === 2) { url = new URL(`https://pinterest.com/url_shortener/${ diff --git a/api/src/util/tests/pr0gramm.json b/api/src/util/tests/pr0gramm.json new file mode 100644 index 00000000..39e9d910 --- /dev/null +++ b/api/src/util/tests/pr0gramm.json @@ -0,0 +1,118 @@ +[ + { + "name": "video", + "url": "https://pr0gramm.com/top/7167415", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "video on pr0.app", + "url": "https://pr0.app/top/7167415", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "video, quality capped at 360p", + "url": "https://pr0gramm.com/top/7167415", + "params": { + "videoQuality": "360" + }, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "video, muted", + "url": "https://pr0gramm.com/top/7167415", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "video, audio only", + "url": "https://pr0gramm.com/top/7167415", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "photo", + "url": "https://pr0gramm.com/top/7167378", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "photo on pr0.app", + "url": "https://pr0.app/top/7167378", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "photo whose fullsize path is stale", + "url": "https://pr0gramm.com/top/7167411", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "user link", + "url": "https://pr0gramm.com/user/dex0r/7167415", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "arbitrary path with an item id", + "url": "https://pr0gramm.com/quatsch/foobar/7167415", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } + }, + { + "name": "non numeric id", + "url": "https://pr0gramm.com/top/dex0r", + "params": {}, + "expected": { + "code": 400, + "status": "error", + "errorCode": "error.api.link.unsupported" + } + }, + { + "name": "inexistent item", + "url": "https://pr0gramm.com/top/999999999", + "params": {}, + "expected": { + "code": 400, + "status": "error", + "errorCode": "error.api.fetch.empty" + } + } +]