diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 8168136685..e08ecd79e8 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -127,6 +127,20 @@ export type TFile = { * See Codex P1 review on PR #12934. */ textFormat?: 'html' | 'text' | null; + /** + * Lifecycle of the inline preview rendered from `text`. `'pending'` + * while background HTML extraction is in flight (phase-2 of the + * code-execution flow), `'ready'` once `text`/`textFormat` are set, + * `'failed'` if extraction errored or hit the 60s ceiling. `undefined` + * for legacy records and for files that never expect a preview — + * clients MUST treat that as `'ready'`. + */ + status?: 'pending' | 'ready' | 'failed'; + /** + * Short machine-readable failure reason when `status === 'failed'`. + * Suitable for tooltip text but not user-facing prose. + */ + previewError?: string; metadata?: { fileIdentifier?: string }; createdAt?: string | Date; updatedAt?: string | Date; @@ -136,6 +150,28 @@ export type TFileUpload = TFile & { temp_file_id: string; }; +/** + * Shape returned by `GET /api/files/:file_id/preview`. The two-phase + * code-execution flow polls this until status is terminal: + * - `pending`: phase-2 HTML extraction is still running. No `text`. + * - `ready`: extraction succeeded; `text` + `textFormat` populated + * iff the file produced inline preview content (binary/oversized + * files reach `ready` with no text — render download-only). + * - `failed`: extraction errored or hit the 60s ceiling; + * `previewError` carries the short reason (`timeout`, + * `parser-error`, `orphaned`, etc.). + * + * Legacy records pre-dating the field are surfaced as `'ready'` server- + * side so existing attachments keep rendering normally. + */ +export type TFilePreview = { + file_id: string; + status: 'pending' | 'ready' | 'failed'; + text?: string; + textFormat?: 'html' | 'text' | null; + previewError?: string; +}; + export type AvatarUploadResponse = { url: string; }; diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.ts index 390b6a8f5c..a96dc3c424 100644 --- a/packages/data-schemas/src/methods/file.spec.ts +++ b/packages/data-schemas/src/methods/file.spec.ts @@ -529,4 +529,101 @@ describe('File Methods', () => { await expect(fileMethods.batchUpdateFiles([])).resolves.toBeUndefined(); }); }); + + describe('sweepOrphanedPreviews', () => { + /* The two-phase preview flow runs phase-2 in-process. If the + * backend restarts mid-extraction, records stay at `status: 'pending'` + * forever. The boot-time sweep transitions stale ones to 'failed' + * with `previewError: 'orphaned'` so the frontend stops polling. */ + const userId = new mongoose.Types.ObjectId(); + + /** + * Stamp a precise `updatedAt` on a file record. Mongoose timestamps + * insist on the current time during create, so we backdate via a + * direct collection write afterwards. + */ + async function makeFile(opts: { + ageMs: number; + status?: 'pending' | 'ready' | 'failed'; + }): Promise { + const fileId = uuidv4(); + await fileMethods.createFile({ + file_id: fileId, + user: userId, + filename: `${fileId}.xlsx`, + filepath: `/uploads/${fileId}.xlsx`, + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 1024, + ...(opts.status ? { status: opts.status } : {}), + }); + const backdated = new Date(Date.now() - opts.ageMs); + await File.collection.updateOne({ file_id: fileId }, { $set: { updatedAt: backdated } }); + return fileId; + } + + it('marks stale pending records as failed with previewError:orphaned', async () => { + const stale = await makeFile({ ageMs: 10 * 60 * 1000, status: 'pending' }); + const fresh = await makeFile({ ageMs: 30 * 1000, status: 'pending' }); + + const count = await fileMethods.sweepOrphanedPreviews(); + expect(count).toBe(1); + + const staleAfter = (await fileMethods.findFileById(stale)) as { + status?: string; + previewError?: string; + } | null; + expect(staleAfter?.status).toBe('failed'); + expect(staleAfter?.previewError).toBe('orphaned'); + + const freshAfter = (await fileMethods.findFileById(fresh)) as { + status?: string; + } | null; + expect(freshAfter?.status).toBe('pending'); + }); + + it('does not touch records that are already ready or failed (idempotent)', async () => { + const ready = await makeFile({ ageMs: 60 * 60 * 1000, status: 'ready' }); + const failed = await makeFile({ ageMs: 60 * 60 * 1000, status: 'failed' }); + + const count = await fileMethods.sweepOrphanedPreviews(); + expect(count).toBe(0); + + const readyAfter = (await fileMethods.findFileById(ready)) as { status?: string } | null; + const failedAfter = (await fileMethods.findFileById(failed)) as { status?: string } | null; + expect(readyAfter?.status).toBe('ready'); + expect(failedAfter?.status).toBe('failed'); + }); + + it('does not touch legacy records with no status field (back-compat)', async () => { + const legacy = await makeFile({ ageMs: 60 * 60 * 1000 }); // no status set + const count = await fileMethods.sweepOrphanedPreviews(); + expect(count).toBe(0); + const after = (await fileMethods.findFileById(legacy)) as { status?: string } | null; + expect(after?.status).toBeUndefined(); + }); + + it('respects a custom maxAgeMs cutoff', async () => { + const old10s = await makeFile({ ageMs: 10 * 1000, status: 'pending' }); + const old1m = await makeFile({ ageMs: 60 * 1000, status: 'pending' }); + + // Cutoff = 30s — only the 60s-old record should be swept. + const count = await fileMethods.sweepOrphanedPreviews(30 * 1000); + expect(count).toBe(1); + + const tenSecAfter = (await fileMethods.findFileById(old10s)) as { status?: string } | null; + const oneMinAfter = (await fileMethods.findFileById(old1m)) as { + status?: string; + previewError?: string; + } | null; + expect(tenSecAfter?.status).toBe('pending'); + expect(oneMinAfter?.status).toBe('failed'); + expect(oneMinAfter?.previewError).toBe('orphaned'); + }); + + it('returns 0 when there are no stale pending records', async () => { + await makeFile({ ageMs: 30 * 1000, status: 'pending' }); + const count = await fileMethods.sweepOrphanedPreviews(); + expect(count).toBe(0); + }); + }); }); diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.ts index 4c0969afb3..e7af55dd03 100644 --- a/packages/data-schemas/src/methods/file.ts +++ b/packages/data-schemas/src/methods/file.ts @@ -368,6 +368,38 @@ export function createFileMethods(mongoose: typeof import('mongoose')) { return results.filter((result): result is IMongoFile => result != null); } + /** + * Mark stale `status: 'pending'` file records as `'failed'` with + * `previewError: 'orphaned'`. Recovers from the only case the + * in-process two-phase preview flow can't handle: a backend restart + * mid-extraction loses the in-memory phase-2 promise, leaving the + * record stuck pending forever. + * + * Cheap to run on boot — the `status` field is indexed and the typical + * cutoff (5 min) bounds the candidate set to whatever was in flight at + * the prior shutdown. The 60s phase-2 timeout means anything older + * than a few minutes that's still pending is definitively orphaned. + * + * @param maxAgeMs - Cutoff in milliseconds; records whose `updatedAt` + * is older than `now - maxAgeMs` are marked failed. Defaults to 5 + * minutes (well above the 60s phase-2 ceiling). + * @returns Number of records updated. + */ + async function sweepOrphanedPreviews(maxAgeMs: number = 5 * 60 * 1000): Promise { + const File = mongoose.models.File as Model; + const cutoff = new Date(Date.now() - maxAgeMs); + const result = await File.updateMany( + { status: 'pending', updatedAt: { $lt: cutoff } }, + { $set: { status: 'failed', previewError: 'orphaned' } }, + ); + if (result.modifiedCount > 0) { + logger.info( + `[sweepOrphanedPreviews] Marked ${result.modifiedCount} stale 'pending' files as 'failed' (cutoff: ${cutoff.toISOString()})`, + ); + } + return result.modifiedCount ?? 0; + } + return { findFileById, getFiles, @@ -383,6 +415,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')) { deleteFileByFilter, batchUpdateFiles, updateFilesUsage, + sweepOrphanedPreviews, }; } diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts index 4098c83eb9..e7eb479db0 100644 --- a/packages/data-schemas/src/schema/file.ts +++ b/packages/data-schemas/src/schema/file.ts @@ -64,6 +64,18 @@ const file: Schema = new Schema( type: String, enum: ['html', 'text'], }, + status: { + /* Two-phase code-execution flow: phase-1 emits the attachment + * record immediately with 'pending'; background phase-2 (HTML + * extraction) updates to 'ready' or 'failed'. Absent on legacy + * records and on file kinds that never expect a preview. */ + type: String, + enum: ['pending', 'ready', 'failed'], + index: true, + }, + previewError: { + type: String, + }, context: { type: String, }, diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts index 9541d5abec..3e26dd3c6b 100644 --- a/packages/data-schemas/src/types/file.ts +++ b/packages/data-schemas/src/types/file.ts @@ -19,6 +19,26 @@ export interface IMongoFile extends Omit { * become executable markup. See Codex P1 review on PR #12934. */ textFormat?: 'html' | 'text'; + /** + * Lifecycle of the inline preview rendered from `text`. Tracks the + * two-phase code-execution flow (PR #12951 follow-up): phase-1 saves + * the file blob and emits the attachment record immediately with + * `status: 'pending'`; phase-2 runs HTML extraction in the background + * and updates the record to `'ready'` (with `text` + `textFormat`) + * or `'failed'` (with `previewError`). Decouples the agent's final + * response from CPU-heavy office-format rendering. + * + * Absent for legacy records and for files that never expect a preview + * (RAG uploads, images, plain-text artifacts). Clients MUST treat + * `undefined` as `'ready'` so prior-version records render normally. + */ + status?: 'pending' | 'ready' | 'failed'; + /** + * Short machine-readable reason when `status === 'failed'` — + * `'timeout'`, `'parser-error'`, `'oversized'`, `'orphaned'`. UI hint + * for tooltip text; not user-facing prose. Absent otherwise. + */ + previewError?: string; filename: string; filepath: string; object: 'file';