fix(import): expire imported assets with their conversations, and report skips

Under retentionMode all the batch builder gives imported conversations and
messages a deadline, but attachments got none - so an attachment outlived
the chat that showed it, in exactly the deployments that require everything
to expire. disableTTL only suppresses the short upload TTL, which is a
different field.

Re-importing a finished export is all skips, and the skip path advanced the
counter without publishing it, so the bar sat at its old value for the whole
run and then jumped to completed.
This commit is contained in:
Marco Beretta 2026-07-29 04:16:17 +02:00
parent 430f46ecbe
commit 7d0d7753bb
No known key found for this signature in database
GPG key ID: D918033D8E74CC11
6 changed files with 74 additions and 0 deletions

View file

@ -493,6 +493,9 @@ async function runImportJob(context, job) {
filepath: job.filepath,
userId,
tenantId,
/** The same deadline the builder stamps on this import's conversations
* and messages, so an attachment expires with the chat that shows it. */
expiredAt: batch.getRetentionFields().expiredAt,
format: target.format,
defaultModel,
deps: { saveBuffer, createFile: db.createFile, deleteFile: releaseAsset },

View file

@ -818,3 +818,54 @@ describe('ingestAssets storage strategy', () => {
expect(sources).toEqual(['document_backend']);
});
});
/** Under `retentionMode: all` the batch builder gives imported conversations
* and messages a deadline. An attachment without the same deadline outlives
* the chat that shows it, in exactly the deployments that require everything
* to expire. */
describe('ingestAssets retention', () => {
const archive: Archive = {
entries: [{ name: 'png-one.dat', bytes: 4 }],
read: async () => Buffer.from([0x89, 0x50, 0x4e, 0x47]),
close: () => undefined,
};
async function ingestWith(expiredAt?: Date) {
const rows: Array<{ expiredAt?: Date }> = [];
await ingestAssets({
archive,
layout: resolveLayout(archive.entries, null),
userId: 'u1',
tenantId: undefined,
expiredAt,
pointers: ['file-service://png-one'],
deps: {
saveBuffer: async ({ fileName }) => ({
filepath: `/uploads/u1/${fileName}`,
source: 'local',
}),
createFile: async (data) => {
rows.push(data);
return { file_id: data.file_id };
},
},
});
return rows;
}
it('stamps the retention deadline on the imported file row', async () => {
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
const rows = await ingestWith(expiredAt);
expect(rows).toHaveLength(1);
expect(rows[0].expiredAt).toEqual(expiredAt);
});
it('leaves the row without a deadline when the deployment sets none', async () => {
const rows = await ingestWith(undefined);
expect(rows).toHaveLength(1);
expect(rows[0]).not.toHaveProperty('expiredAt');
});
});

View file

@ -39,6 +39,8 @@ export interface CreateFileInput {
source: string;
context: string;
tenantId?: string;
/** Retention deadline, when the deployment sets one. */
expiredAt?: Date;
}
export type CreateFileFn = (
@ -63,6 +65,13 @@ export interface IngestInput {
userId: string;
tenantId: string | undefined;
pointers: string[];
/** The retention deadline the conversations and messages of this import are
* getting. An imported attachment has to expire with the conversation that
* references it: without it the row and its storage object outlive the chat
* they belong to, in exactly the deployments that require everything to
* expire. `disableTTL` only suppresses the short upload TTL, which is a
* different field. */
expiredAt?: Date;
deps: AssetDeps;
/** Attachment metadata keyed by the raw id (pointer with its scheme
* stripped). Supplies the authoritative `mime_type` the export recorded
@ -304,6 +313,7 @@ async function ingestOne(
source,
context: FileContext.message_attachment,
tenantId,
...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
},
true,
);

View file

@ -130,6 +130,7 @@ export async function runClaudeImport(context: ProviderImportContext): Promise<v
if (input.existingExternalIds.has(conv.uuid)) {
report.skipped += 1;
progress.conversations.done += 1;
await input.onProgress?.(progress);
continue;
}

View file

@ -329,6 +329,7 @@ export async function runImport(input: RunImportInput): Promise<ImportReport> {
layout,
userId: input.userId,
tenantId: input.tenantId,
expiredAt: input.expiredAt,
pointers: scan.pointers,
attachments: scan.attachments,
references: scan.references,
@ -374,6 +375,10 @@ export async function runImport(input: RunImportInput): Promise<ImportReport> {
if (input.existingExternalIds.has(conv.conversation_id)) {
report.skipped += 1;
progress.conversations.done += 1;
/** Published like any other advance: re-importing a finished export
* is all skips, and without this the bar sits at its old value for
* the whole run and then jumps straight to completed. */
await input.onProgress?.(progress);
continue;
}

View file

@ -57,6 +57,10 @@ export interface RunImportInput {
filepath: string;
userId: string;
tenantId?: string;
/** Retention deadline for everything this run writes, when the deployment
* sets one. The batch builder already applies it to conversations and
* messages; assets have to carry it too or they outlive the chats. */
expiredAt?: Date;
defaultModel: string;
/** The format `inspectExport` already identified. Omit it and `runImport`
* re-reads the first shard to detect it. */