mirror of
https://github.com/imputnet/cobalt.git
synced 2026-10-25 03:22:19 +00:00
api/pr0gramm: add support for pr0gramm
pr0gramm is an imageboard, and its new frontend pr0.app uses the same public
api. every upload is a single image or a video, served over plain cdn urls
(img./vid./thumb.pr0gramm.com) that need no login for sfw items.
the extractor reads the item from the public api and picks what the site itself
offers:
- videos exist as h264 (mp4, always with audio) and vp9 (webm, usually
without audio), in several height classes; videoQuality caps the height
- subtitles (.vtt) are attached for uploads that have audio
- items without any video variant are images, and are proxied
the media path pr0gramm reports isn't always up to date: an item that was
converted after its upload keeps the old path in `fullsize` (a `.png` that is
really served as `.jpg`), and a video without any variant has only its `image`
path. rather than trusting whichever field is filled, the reachable candidate
is probed. reviewers who prefer not to make HEAD requests here can simply keep
the `fullsize || image` order instead.
item links from both pr0gramm.com and pr0.app are accepted (/top/<id>,
/new/<id>, /user/<user>/<id>, ... -- the item id is always the last path
segment), and pr0.app is canonicalized to pr0gramm.com.
the media is served with `access-control-allow-origin: https://pr0.app` and
without referer checks, so a plain redirect response works for both the api and
the web ui and no local processing is needed.
this change was written with Hermes Agent and reviewed by me before submitting.
This commit is contained in:
propagare
parent
a636575b09
commit
cba14785e6
8 changed files with 360 additions and 0 deletions
|
|
@ -26,6 +26,7 @@ if the desired service isn't supported yet, feel free to create an appropriate i
|
|||
| newgrounds | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ |
|
||||
| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| pr0gramm (pr0.app)| ✅ | ✅ | ✅ | ➖ | ✅ |
|
||||
| reddit | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| rutube | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| snapchat | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
|
|
@ -51,6 +52,7 @@ if the desired service isn't supported yet, feel free to create an appropriate i
|
|||
| instagram | supports reels, photos, and videos. lets you pick what to save from multi-media posts. |
|
||||
| facebook | supports public accessible videos content only. |
|
||||
| pinterest | supports photos, gifs, videos and stories. |
|
||||
| pr0gramm | supports images and videos, in the h264 and vp9 variants that the site itself offers. also accepts pr0.app links. |
|
||||
| reddit | supports gifs and videos. |
|
||||
| snapchat | supports spotlights and stories. lets you pick what to save from stories. |
|
||||
| rutube | supports yappy & private links. |
|
||||
|
|
|
|||
|
|
@ -183,6 +183,13 @@ export default function({
|
|||
params = { type: "proxy" };
|
||||
break;
|
||||
|
||||
/* pr0gramm media is served over plain cdn urls, so both a
|
||||
redirect and a tunnel work. the redirect is preferred and
|
||||
`alwaysProxy` turns it into a tunnel when needed. */
|
||||
case "pr0gramm":
|
||||
responseType = "redirect";
|
||||
break;
|
||||
|
||||
case "facebook":
|
||||
case "instagram":
|
||||
case "tumblr":
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import loom from "./services/loom.js";
|
|||
import facebook from "./services/facebook.js";
|
||||
import bluesky from "./services/bluesky.js";
|
||||
import newgrounds from "./services/newgrounds.js";
|
||||
import pr0gramm from "./services/pr0gramm.js";
|
||||
|
||||
let freebind;
|
||||
|
||||
|
|
@ -266,6 +267,14 @@ export default async function({ host, patternMatch, params, authType }) {
|
|||
});
|
||||
break;
|
||||
|
||||
case "pr0gramm":
|
||||
r = await pr0gramm({
|
||||
...patternMatch,
|
||||
quality: params.videoQuality,
|
||||
isAudioOnly,
|
||||
});
|
||||
break;
|
||||
|
||||
default:
|
||||
return createResponse("error", {
|
||||
code: "error.api.service.unsupported"
|
||||
|
|
|
|||
|
|
@ -75,6 +75,19 @@ export const services = {
|
|||
"url_shortener/:shortLink"
|
||||
],
|
||||
},
|
||||
pr0gramm: {
|
||||
/* pr0gramm.com and pr0.app share the same url layout, and an item link
|
||||
is *always* "…/<item id>" — no matter whether it was shared from a
|
||||
feed (/top/<id>, /new/<id>, /junk/<id>), a profile
|
||||
(/user/<user>/<id>) or a collection (/collections/<name>/<id>).
|
||||
|
||||
pr0.app additionally redirects unknown path segments to /new/<id>
|
||||
(it puts that into its og:url meta tag), so the pattern stays as
|
||||
permissive as pr0gramm itself and the id is validated numerially in
|
||||
service-patterns.js instead. */
|
||||
patterns: ["**/:id"],
|
||||
altDomains: ["pr0.app"],
|
||||
},
|
||||
newgrounds: {
|
||||
patterns: [
|
||||
"portal/view/:id",
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ export const testers = {
|
|||
pattern.id?.length <= 128 ||
|
||||
pattern.shortLink?.length <= 32,
|
||||
|
||||
"pr0gramm": pattern =>
|
||||
// item ids are numeric and can be anywhere up to int64
|
||||
pattern.id?.length <= 19 && /^\d+$/.test(pattern.id || ""),
|
||||
|
||||
"reddit": pattern =>
|
||||
pattern.id?.length <= 16 && !pattern.sub && !pattern.user ||
|
||||
(pattern.sub?.length <= 22 && pattern.id?.length <= 16) ||
|
||||
|
|
|
|||
201
api/src/processing/services/pr0gramm.js
Normal file
201
api/src/processing/services/pr0gramm.js
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
import { strict as assert } from "node:assert";
|
||||
|
||||
import { genericUserAgent } from "../../config.js";
|
||||
|
||||
/*
|
||||
pr0gramm is an imageboard. every upload ("item") is a single image or a video,
|
||||
and all media is served from plain cdn hosts without any login.
|
||||
|
||||
the api returns bare paths (e.g. `2026/10/09/<hash>-h264-sd.mp4`), so the host
|
||||
has to be picked based on what is being downloaded:
|
||||
|
||||
- `image` / `fullsize` -> img.pr0gramm.com (images)
|
||||
- `image` (when the item is a video) / `variants` -> vid.pr0gramm.com
|
||||
- `thumb` -> thumb.pr0gramm.com (smaller re-encode, jpeg/webp)
|
||||
|
||||
videos come as h264 and vp9 variants. h264 always has audio, vp9 usually
|
||||
doesn't, so h264 is used for audio downloads.
|
||||
*/
|
||||
|
||||
const mediaHosts = {
|
||||
image: "https://img.pr0gramm.com/",
|
||||
video: "https://vid.pr0gramm.com/",
|
||||
thumb: "https://thumb.pr0gramm.com/",
|
||||
};
|
||||
|
||||
export const mediaURL = (path, host = "image") => {
|
||||
assert(mediaHosts[host]);
|
||||
return new URL(path, mediaHosts[host]).toString();
|
||||
};
|
||||
|
||||
const videoExtensions = ["mp4", "webm", "mov", "mkv"];
|
||||
|
||||
const fileExtension = (path) => path?.split(".").pop().toLowerCase();
|
||||
|
||||
// videos live on vid. and images on img., and both hosts are happy to serve
|
||||
// the other type — but only with the extension the path actually has, so it
|
||||
// has to be taken from the path itself
|
||||
export const mediaURLFor = (path) => mediaURL(
|
||||
path,
|
||||
videoExtensions.includes(fileExtension(path)) ? "video" : "image"
|
||||
);
|
||||
|
||||
/*
|
||||
pr0gramm does not always keep its media paths up to date: an item that was
|
||||
converted after its upload has the old path in `fullsize` (e.g. a `.png`
|
||||
that is really served as `.jpg`), and a video without any h264/vp9 variant
|
||||
has only its `image` path. so the reachable one is probed instead of
|
||||
trusting whichever field happens to be filled.
|
||||
*/
|
||||
const pickReachableMedia = async (candidates) => {
|
||||
for (const path of candidates) {
|
||||
if (!path) continue;
|
||||
|
||||
const url = mediaURLFor(path),
|
||||
reachable = await fetch(url, {
|
||||
method: "HEAD",
|
||||
headers: { "user-agent": genericUserAgent }
|
||||
}).then(r => r.ok).catch(() => false);
|
||||
|
||||
if (reachable) {
|
||||
return { url, extension: fileExtension(path) };
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
variants are named `<codec>-<height class>`, and the height classes are
|
||||
consistent across qualities (sd is always 480p, hd always 720p, ...), so they
|
||||
are a more reliable source for picking a quality than the api heights.
|
||||
*/
|
||||
const heightClasses = {
|
||||
low_res: 360,
|
||||
sd: 480,
|
||||
hd: 720,
|
||||
full_hd: 1080,
|
||||
ultra_hd: 2160,
|
||||
"4k": 2160,
|
||||
"8k": 4320,
|
||||
};
|
||||
|
||||
const videoTypes = [
|
||||
{ mimeType: "video/mp4", extension: "mp4", codecs: ["h264", "hevc", "av1"] },
|
||||
{ mimeType: "video/webm", extension: "webm", codecs: ["vp9", "av1"] },
|
||||
];
|
||||
|
||||
const getVideoType = (variant) =>
|
||||
videoTypes.find(type => type.mimeType === variant?.mimeType);
|
||||
|
||||
export const selectVariant = (variants, quality, codec = "h264") => {
|
||||
const parsed = (variants || [])
|
||||
.filter(variant => variant?.path && getVideoType(variant))
|
||||
.map(variant => ({
|
||||
...variant,
|
||||
height: heightClasses[variant.name?.split("-").pop()]
|
||||
|| variant.height
|
||||
|| 0,
|
||||
}))
|
||||
.sort((a, b) => a.height - b.height);
|
||||
|
||||
if (!parsed.length) return;
|
||||
|
||||
// prefer the requested codec, but never fail just because it's missing
|
||||
const sameCodec = parsed.filter(variant => variant.codec === codec);
|
||||
const candidates = sameCodec.length ? sameCodec : parsed;
|
||||
|
||||
const target = quality === "max" ? Infinity : parseInt(quality) || 1080;
|
||||
return candidates.findLast(variant => variant.height <= target)
|
||||
?? candidates[0];
|
||||
};
|
||||
|
||||
const getFilename = (item, extension, qualityLabel, resolution) => ({
|
||||
filenameAttributes: {
|
||||
service: "pr0gramm",
|
||||
id: `${item.id}`,
|
||||
title: `pr0gramm ${item.id}`,
|
||||
...(item.user && { author: item.user }),
|
||||
...(extension && { extension }),
|
||||
...(qualityLabel && { qualityLabel: `${qualityLabel}p` }),
|
||||
...(resolution?.width && {
|
||||
resolution: `${resolution.width}x${resolution.height}`,
|
||||
}),
|
||||
},
|
||||
fileMetadata: {
|
||||
title: `${item.user ? `${item.user} - ` : ""}pr0gramm ${item.id}`,
|
||||
...(item.user && { artist: item.user }),
|
||||
...(item.created && {
|
||||
date: new Date(item.created * 1000).toISOString().split("T")[0],
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
async function getItem(id) {
|
||||
const api = new URL("https://pr0gramm.com/api/items/get");
|
||||
api.searchParams.set("id", id);
|
||||
// `flags` is required by the api. 1 = sfw, the only flag that is
|
||||
// visible without an account
|
||||
api.searchParams.set("flags", "1");
|
||||
|
||||
const response = await fetch(api, {
|
||||
headers: { "user-agent": genericUserAgent }
|
||||
}).catch(() => {});
|
||||
|
||||
if (!response?.ok) return;
|
||||
|
||||
const data = await response.json().catch(() => {});
|
||||
if (data?.error) return;
|
||||
|
||||
// the api returns items *starting* at the requested id, so it has to be
|
||||
// looked up explicitly. `id` is a number in the response
|
||||
return data?.items?.find(item => item.id === +id);
|
||||
}
|
||||
|
||||
export default async function ({ id, quality, isAudioOnly }) {
|
||||
const item = await getItem(id);
|
||||
if (!item) return { error: "fetch.empty" };
|
||||
|
||||
const video = selectVariant(
|
||||
item.variants,
|
||||
isAudioOnly ? "max" : quality,
|
||||
"h264"
|
||||
);
|
||||
|
||||
if (isAudioOnly) {
|
||||
if (!video) return { error: "fetch.empty" };
|
||||
|
||||
return {
|
||||
urls: mediaURL(video.path, "video"),
|
||||
audioFilename: `pr0gramm_${item.id}_audio`,
|
||||
fileMetadata: getFilename(item).fileMetadata,
|
||||
};
|
||||
}
|
||||
|
||||
if (!video) {
|
||||
// uploads without any video variant are images
|
||||
const photo = await pickReachableMedia([ item.image, item.fullsize ]);
|
||||
if (!photo) return { error: "fetch.empty" };
|
||||
|
||||
return {
|
||||
urls: photo.url,
|
||||
// photos have no quality labels, so a plain filename reads better
|
||||
// than the "… ()" that the filename styles would produce
|
||||
filename: `pr0gramm_${item.id}.${photo.extension}`,
|
||||
// the rare video without any variants is still a video, and
|
||||
// shouldn't be saved as a photo
|
||||
...(!videoExtensions.includes(photo.extension) && { isPhoto: true }),
|
||||
};
|
||||
}
|
||||
|
||||
const type = getVideoType(video),
|
||||
// pr0gramm transcribes audio uploads and renders the default subtitle
|
||||
// against the sd variant
|
||||
subtitle = item.audio
|
||||
? item.subtitles?.find(sub => sub.isDefault)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
urls: mediaURL(video.path, "video"),
|
||||
...getFilename(item, type.extension, video.height, video),
|
||||
...(subtitle && { subtitles: mediaURL(subtitle.path, "video") }),
|
||||
};
|
||||
}
|
||||
|
|
@ -31,6 +31,12 @@ function aliasURL(url) {
|
|||
}
|
||||
break;
|
||||
|
||||
case "pr0":
|
||||
if (services.pr0gramm.altDomains.includes(url.hostname)) {
|
||||
url.hostname = 'pr0gramm.com';
|
||||
}
|
||||
break;
|
||||
|
||||
case "pin":
|
||||
if (url.hostname === 'pin.it' && parts.length === 2) {
|
||||
url = new URL(`https://pinterest.com/url_shortener/${
|
||||
|
|
|
|||
118
api/src/util/tests/pr0gramm.json
Normal file
118
api/src/util/tests/pr0gramm.json
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
[
|
||||
{
|
||||
"name": "video",
|
||||
"url": "https://pr0gramm.com/top/7167415",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video on pr0.app",
|
||||
"url": "https://pr0.app/top/7167415",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video, quality capped at 360p",
|
||||
"url": "https://pr0gramm.com/top/7167415",
|
||||
"params": {
|
||||
"videoQuality": "360"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video, muted",
|
||||
"url": "https://pr0gramm.com/top/7167415",
|
||||
"params": {
|
||||
"downloadMode": "mute"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "video, audio only",
|
||||
"url": "https://pr0gramm.com/top/7167415",
|
||||
"params": {
|
||||
"downloadMode": "audio"
|
||||
},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "photo",
|
||||
"url": "https://pr0gramm.com/top/7167378",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "photo on pr0.app",
|
||||
"url": "https://pr0.app/top/7167378",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "photo whose fullsize path is stale",
|
||||
"url": "https://pr0gramm.com/top/7167411",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "tunnel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user link",
|
||||
"url": "https://pr0gramm.com/user/dex0r/7167415",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "arbitrary path with an item id",
|
||||
"url": "https://pr0gramm.com/quatsch/foobar/7167415",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "non numeric id",
|
||||
"url": "https://pr0gramm.com/top/dex0r",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error",
|
||||
"errorCode": "error.api.link.unsupported"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inexistent item",
|
||||
"url": "https://pr0gramm.com/top/999999999",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 400,
|
||||
"status": "error",
|
||||
"errorCode": "error.api.fetch.empty"
|
||||
}
|
||||
}
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue