From 7d0d7753bb705a2429d74b213df1953b277f3f48 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 29 Jul 2026 04:16:17 +0200 Subject: [PATCH] fix(import): expire imported assets with their conversations, and report skips Under retentionMode all the batch builder gives imported conversations and messages a deadline, but attachments got none - so an attachment outlived the chat that showed it, in exactly the deployments that require everything to expire. disableTTL only suppresses the short upload TTL, which is a different field. Re-importing a finished export is all skips, and the skip path advanced the counter without publishing it, so the bar sat at its old value for the whole run and then jumped to completed. --- api/server/routes/convos.js | 3 ++ packages/api/src/import/assets.spec.ts | 51 +++++++++++++++++++++++ packages/api/src/import/assets.ts | 10 +++++ packages/api/src/import/claude/service.ts | 1 + packages/api/src/import/service.ts | 5 +++ packages/api/src/import/sink.ts | 4 ++ 6 files changed, 74 insertions(+) diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index a45d32c212..8b0ec0a1e1 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -493,6 +493,9 @@ async function runImportJob(context, job) { filepath: job.filepath, userId, tenantId, + /** The same deadline the builder stamps on this import's conversations + * and messages, so an attachment expires with the chat that shows it. */ + expiredAt: batch.getRetentionFields().expiredAt, format: target.format, defaultModel, deps: { saveBuffer, createFile: db.createFile, deleteFile: releaseAsset }, diff --git a/packages/api/src/import/assets.spec.ts b/packages/api/src/import/assets.spec.ts index 30255b70ef..65f25e8431 100644 --- a/packages/api/src/import/assets.spec.ts +++ b/packages/api/src/import/assets.spec.ts @@ -818,3 +818,54 @@ describe('ingestAssets storage strategy', () => { expect(sources).toEqual(['document_backend']); }); }); + +/** Under `retentionMode: all` the batch builder gives imported conversations + * and messages a deadline. An attachment without the same deadline outlives + * the chat that shows it, in exactly the deployments that require everything + * to expire. */ +describe('ingestAssets retention', () => { + const archive: Archive = { + entries: [{ name: 'png-one.dat', bytes: 4 }], + read: async () => Buffer.from([0x89, 0x50, 0x4e, 0x47]), + close: () => undefined, + }; + + async function ingestWith(expiredAt?: Date) { + const rows: Array<{ expiredAt?: Date }> = []; + await ingestAssets({ + archive, + layout: resolveLayout(archive.entries, null), + userId: 'u1', + tenantId: undefined, + expiredAt, + pointers: ['file-service://png-one'], + deps: { + saveBuffer: async ({ fileName }) => ({ + filepath: `/uploads/u1/${fileName}`, + source: 'local', + }), + createFile: async (data) => { + rows.push(data); + return { file_id: data.file_id }; + }, + }, + }); + return rows; + } + + it('stamps the retention deadline on the imported file row', async () => { + const expiredAt = new Date('2030-01-01T00:00:00.000Z'); + + const rows = await ingestWith(expiredAt); + + expect(rows).toHaveLength(1); + expect(rows[0].expiredAt).toEqual(expiredAt); + }); + + it('leaves the row without a deadline when the deployment sets none', async () => { + const rows = await ingestWith(undefined); + + expect(rows).toHaveLength(1); + expect(rows[0]).not.toHaveProperty('expiredAt'); + }); +}); diff --git a/packages/api/src/import/assets.ts b/packages/api/src/import/assets.ts index 0f61f64699..8510ac2e46 100644 --- a/packages/api/src/import/assets.ts +++ b/packages/api/src/import/assets.ts @@ -39,6 +39,8 @@ export interface CreateFileInput { source: string; context: string; tenantId?: string; + /** Retention deadline, when the deployment sets one. */ + expiredAt?: Date; } export type CreateFileFn = ( @@ -63,6 +65,13 @@ export interface IngestInput { userId: string; tenantId: string | undefined; pointers: string[]; + /** The retention deadline the conversations and messages of this import are + * getting. An imported attachment has to expire with the conversation that + * references it: without it the row and its storage object outlive the chat + * they belong to, in exactly the deployments that require everything to + * expire. `disableTTL` only suppresses the short upload TTL, which is a + * different field. */ + expiredAt?: Date; deps: AssetDeps; /** Attachment metadata keyed by the raw id (pointer with its scheme * stripped). Supplies the authoritative `mime_type` the export recorded @@ -304,6 +313,7 @@ async function ingestOne( source, context: FileContext.message_attachment, tenantId, + ...(input.expiredAt ? { expiredAt: input.expiredAt } : {}), }, true, ); diff --git a/packages/api/src/import/claude/service.ts b/packages/api/src/import/claude/service.ts index b5f4d56fd6..291ce1f926 100644 --- a/packages/api/src/import/claude/service.ts +++ b/packages/api/src/import/claude/service.ts @@ -130,6 +130,7 @@ export async function runClaudeImport(context: ProviderImportContext): Promise { layout, userId: input.userId, tenantId: input.tenantId, + expiredAt: input.expiredAt, pointers: scan.pointers, attachments: scan.attachments, references: scan.references, @@ -374,6 +375,10 @@ export async function runImport(input: RunImportInput): Promise { if (input.existingExternalIds.has(conv.conversation_id)) { report.skipped += 1; progress.conversations.done += 1; + /** Published like any other advance: re-importing a finished export + * is all skips, and without this the bar sits at its old value for + * the whole run and then jumps straight to completed. */ + await input.onProgress?.(progress); continue; } diff --git a/packages/api/src/import/sink.ts b/packages/api/src/import/sink.ts index 8c56266eba..28390a6885 100644 --- a/packages/api/src/import/sink.ts +++ b/packages/api/src/import/sink.ts @@ -57,6 +57,10 @@ export interface RunImportInput { filepath: string; userId: string; tenantId?: string; + /** Retention deadline for everything this run writes, when the deployment + * sets one. The batch builder already applies it to conversations and + * messages; assets have to carry it too or they outlive the chats. */ + expiredAt?: Date; defaultModel: string; /** The format `inspectExport` already identified. Omit it and `runImport` * re-reads the first shard to detect it. */