From d0538d4481edd880e09396d9b8a80db8b8843ee9 Mon Sep 17 00:00:00 2001 From: wolfmanster <68727977+wolfmanster@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:14:56 +0800 Subject: [PATCH] Organize downloaded media by post metadata --- apps/server/src/media-paths.test.ts | 43 ++++++++++++++++++++++++ apps/server/src/media-paths.ts | 51 +++++++++++++++++++++++++++++ apps/server/src/queue.ts | 11 +++++-- apps/server/src/store.ts | 13 +++++++- 4 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 apps/server/src/media-paths.test.ts create mode 100644 apps/server/src/media-paths.ts diff --git a/apps/server/src/media-paths.test.ts b/apps/server/src/media-paths.test.ts new file mode 100644 index 00000000..e6e8211c --- /dev/null +++ b/apps/server/src/media-paths.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; +import { buildAuthorFolder, buildMediaFilename, buildPostFolder, formatPublishedDate } from './media-paths.js'; + +describe('media path naming', () => { + it('formats the publication date and appends the tweet ID', () => { + expect(formatPublishedDate('2026-08-18T16:02:03.000Z')).toBe('2026-08-19'); + expect(buildPostFolder({ publishedAt: '2026-08-19T01:02:03.000Z', text: 'hello world' }, '1234567890')) + .toBe('2026-08-19_hello world_1234567890'); + }); + + it('uses the China Standard Time date around UTC midnight', () => { + expect(formatPublishedDate('2026-08-19T15:59:59.000Z')).toBe('2026-08-19'); + expect(formatPublishedDate('2026-08-19T16:00:00.000Z')).toBe('2026-08-20'); + }); + + it('sanitizes Windows path characters while preserving the tweet ID suffix', () => { + const folder = buildPostFolder( + { publishedAt: '2026-08-19T01:02:03.000Z', text: 'a/b:c?'.repeat(60) + }, + '1234567890', + ); + + expect(folder).toMatch(/^2026-08-19_a_b_c_.*_1234567890$/); + expect(folder).not.toMatch(/[<>:"/\\|?*]/); + }); + + it('builds the author folder from display name and user ID', () => { + expect(buildAuthorFolder({ authorName: 'A/B', userId: '42' }, 'fallback')).toBe('A_B_42'); + }); + + it.each([ + [{ kind: 'image', filename: 'source.jpg' }, '1-pic.jpg'], + [{ kind: 'video', filename: 'source.mp4' }, '2-vdo.mp4'], + [{ kind: 'gif', filename: 'source.gif' }, '3-pic.gif'], + ] as const)('names media by tweet order and type', (media, expected) => { + expect(buildMediaFilename({ + ...media, + downloadedBytes: 0, + id: 'media', + sourceUrl: 'https://example.com/media', + }, Number(expected[0]) - 1)).toBe(expected); + }); +}); diff --git a/apps/server/src/media-paths.ts b/apps/server/src/media-paths.ts new file mode 100644 index 00000000..503c6f03 --- /dev/null +++ b/apps/server/src/media-paths.ts @@ -0,0 +1,51 @@ +import path from 'node:path'; +import type { MediaItem, PostMetadata } from './types.js'; + +const POST_TIME_ZONE = 'Asia/Shanghai'; + +function sanitizePathSegment(value: string, fallback: string, maxLength = 180): string { + let segment = value + .replace(/[\u0000-\u001F\u007F-\u009F]/g, '') + .replace(/[<>:"/\\|?*]/g, '_') + .trim() + .replace(/[. ]+$/g, ''); + + if (!segment) segment = fallback; + if (/^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$/i.test(segment)) segment = `_${segment}`; + + // Keep each Windows path component below the usual 255-character limit. + segment = Array.from(segment).slice(0, maxLength).join('').replace(/[. ]+$/g, ''); + return segment || fallback; +} + +export function formatPublishedDate(isoDate: string | undefined): string { + if (!isoDate) return 'unknown-date'; + const date = new Date(isoDate); + if (Number.isNaN(date.getTime())) return 'unknown-date'; + + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: POST_TIME_ZONE, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).formatToParts(date); + const values = Object.fromEntries(parts.map(({ type, value }) => [type, value])); + return `${values.year}-${values.month}-${values.day}`; +} + +export function buildAuthorFolder(metadata: Pick, fallback: string): string { + return sanitizePathSegment(`${metadata.authorName}_${metadata.userId}`, fallback); +} + +export function buildPostFolder(metadata: Pick | undefined, tweetId: string): string { + const date = formatPublishedDate(metadata?.publishedAt); + // Reserve room for the date and tweet ID so the ID is always retained. + const text = sanitizePathSegment(metadata?.text?.trim() || tweetId, tweetId, 140); + return sanitizePathSegment(`${date}_${text}_${tweetId}`, `${date}_${tweetId}_${tweetId}`); +} + +export function buildMediaFilename(media: MediaItem, index: number): string { + const extension = path.extname(media.filename).slice(1).toLowerCase() || (media.kind === 'video' ? 'mp4' : 'jpg'); + const suffix = media.kind === 'video' ? 'vdo' : 'pic'; + return `${index + 1}-${suffix}.${extension}`; +} diff --git a/apps/server/src/queue.ts b/apps/server/src/queue.ts index 8a63c8a8..d69895fe 100644 --- a/apps/server/src/queue.ts +++ b/apps/server/src/queue.ts @@ -6,6 +6,7 @@ import { Readable, Transform } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { fetchPostMetadata } from './metadata.js'; import { resolveMedia } from './cobalt.js'; +import { buildAuthorFolder, buildMediaFilename, buildPostFolder } from './media-paths.js'; import type { DownloadJob, MediaItem } from './types.js'; import { JobStore } from './store.js'; @@ -143,12 +144,18 @@ export class DownloadQueue extends EventEmitter { if (contentType && EXTENSIONS[contentType]) { media.filename = media.filename.replace(/\.[a-z0-9]+$/i, `.${EXTENSIONS[contentType]}`); } + media.filename = buildMediaFilename(media, index); media.contentType = contentType; media.totalBytes = total; - const jobDir = path.join(this.store.mediaDir, job.id); + const metadata = job.metadata; + const authorFolder = metadata + ? buildAuthorFolder(metadata, job.tweetId) + : job.tweetId; + const postFolder = buildPostFolder(metadata, job.tweetId); + const jobDir = path.join(this.store.mediaDir, authorFolder, postFolder); await fsp.mkdir(jobDir, { recursive: true }); - const finalPath = path.join(jobDir, media.filename.replace(/[^a-zA-Z0-9._-]/g, '_')); + const finalPath = path.join(jobDir, media.filename); const partialPath = `${finalPath}.part`; media.localPath = path.relative(this.store.dataDir, finalPath); diff --git a/apps/server/src/store.ts b/apps/server/src/store.ts index 35b81383..bdf3fa61 100644 --- a/apps/server/src/store.ts +++ b/apps/server/src/store.ts @@ -64,7 +64,18 @@ export class JobStore { for (const [id, job] of this.jobs) { if (['completed', 'failed', 'canceled'].includes(job.status)) { this.jobs.delete(id); - mediaDirectories.push(path.join(this.mediaDir, id)); + const mediaRoot = path.resolve(this.mediaDir); + const directories = new Set( + job.media + .map((media) => { + if (!media.localPath) return undefined; + const directory = path.resolve(this.dataDir, path.dirname(media.localPath)); + return directory.startsWith(`${mediaRoot}${path.sep}`) ? directory : undefined; + }) + .filter((directory): directory is string => Boolean(directory)), + ); + if (!directories.size) directories.add(path.join(this.mediaDir, id)); + mediaDirectories.push(...directories); count += 1; } }