mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-08-04 14:57:42 +00:00
fix(import): expire imported assets with their conversations, and report skips
Under retentionMode all the batch builder gives imported conversations and messages a deadline, but attachments got none - so an attachment outlived the chat that showed it, in exactly the deployments that require everything to expire. disableTTL only suppresses the short upload TTL, which is a different field. Re-importing a finished export is all skips, and the skip path advanced the counter without publishing it, so the bar sat at its old value for the whole run and then jumped to completed.
This commit is contained in:
parent
430f46ecbe
commit
7d0d7753bb
6 changed files with 74 additions and 0 deletions
|
|
@ -493,6 +493,9 @@ async function runImportJob(context, job) {
|
|||
filepath: job.filepath,
|
||||
userId,
|
||||
tenantId,
|
||||
/** The same deadline the builder stamps on this import's conversations
|
||||
* and messages, so an attachment expires with the chat that shows it. */
|
||||
expiredAt: batch.getRetentionFields().expiredAt,
|
||||
format: target.format,
|
||||
defaultModel,
|
||||
deps: { saveBuffer, createFile: db.createFile, deleteFile: releaseAsset },
|
||||
|
|
|
|||
|
|
@ -818,3 +818,54 @@ describe('ingestAssets storage strategy', () => {
|
|||
expect(sources).toEqual(['document_backend']);
|
||||
});
|
||||
});
|
||||
|
||||
/** Under `retentionMode: all` the batch builder gives imported conversations
|
||||
* and messages a deadline. An attachment without the same deadline outlives
|
||||
* the chat that shows it, in exactly the deployments that require everything
|
||||
* to expire. */
|
||||
describe('ingestAssets retention', () => {
|
||||
const archive: Archive = {
|
||||
entries: [{ name: 'png-one.dat', bytes: 4 }],
|
||||
read: async () => Buffer.from([0x89, 0x50, 0x4e, 0x47]),
|
||||
close: () => undefined,
|
||||
};
|
||||
|
||||
async function ingestWith(expiredAt?: Date) {
|
||||
const rows: Array<{ expiredAt?: Date }> = [];
|
||||
await ingestAssets({
|
||||
archive,
|
||||
layout: resolveLayout(archive.entries, null),
|
||||
userId: 'u1',
|
||||
tenantId: undefined,
|
||||
expiredAt,
|
||||
pointers: ['file-service://png-one'],
|
||||
deps: {
|
||||
saveBuffer: async ({ fileName }) => ({
|
||||
filepath: `/uploads/u1/${fileName}`,
|
||||
source: 'local',
|
||||
}),
|
||||
createFile: async (data) => {
|
||||
rows.push(data);
|
||||
return { file_id: data.file_id };
|
||||
},
|
||||
},
|
||||
});
|
||||
return rows;
|
||||
}
|
||||
|
||||
it('stamps the retention deadline on the imported file row', async () => {
|
||||
const expiredAt = new Date('2030-01-01T00:00:00.000Z');
|
||||
|
||||
const rows = await ingestWith(expiredAt);
|
||||
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].expiredAt).toEqual(expiredAt);
|
||||
});
|
||||
|
||||
it('leaves the row without a deadline when the deployment sets none', async () => {
|
||||
const rows = await ingestWith(undefined);
|
||||
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).not.toHaveProperty('expiredAt');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ export interface CreateFileInput {
|
|||
source: string;
|
||||
context: string;
|
||||
tenantId?: string;
|
||||
/** Retention deadline, when the deployment sets one. */
|
||||
expiredAt?: Date;
|
||||
}
|
||||
|
||||
export type CreateFileFn = (
|
||||
|
|
@ -63,6 +65,13 @@ export interface IngestInput {
|
|||
userId: string;
|
||||
tenantId: string | undefined;
|
||||
pointers: string[];
|
||||
/** The retention deadline the conversations and messages of this import are
|
||||
* getting. An imported attachment has to expire with the conversation that
|
||||
* references it: without it the row and its storage object outlive the chat
|
||||
* they belong to, in exactly the deployments that require everything to
|
||||
* expire. `disableTTL` only suppresses the short upload TTL, which is a
|
||||
* different field. */
|
||||
expiredAt?: Date;
|
||||
deps: AssetDeps;
|
||||
/** Attachment metadata keyed by the raw id (pointer with its scheme
|
||||
* stripped). Supplies the authoritative `mime_type` the export recorded
|
||||
|
|
@ -304,6 +313,7 @@ async function ingestOne(
|
|||
source,
|
||||
context: FileContext.message_attachment,
|
||||
tenantId,
|
||||
...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@ export async function runClaudeImport(context: ProviderImportContext): Promise<v
|
|||
if (input.existingExternalIds.has(conv.uuid)) {
|
||||
report.skipped += 1;
|
||||
progress.conversations.done += 1;
|
||||
await input.onProgress?.(progress);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -329,6 +329,7 @@ export async function runImport(input: RunImportInput): Promise<ImportReport> {
|
|||
layout,
|
||||
userId: input.userId,
|
||||
tenantId: input.tenantId,
|
||||
expiredAt: input.expiredAt,
|
||||
pointers: scan.pointers,
|
||||
attachments: scan.attachments,
|
||||
references: scan.references,
|
||||
|
|
@ -374,6 +375,10 @@ export async function runImport(input: RunImportInput): Promise<ImportReport> {
|
|||
if (input.existingExternalIds.has(conv.conversation_id)) {
|
||||
report.skipped += 1;
|
||||
progress.conversations.done += 1;
|
||||
/** Published like any other advance: re-importing a finished export
|
||||
* is all skips, and without this the bar sits at its old value for
|
||||
* the whole run and then jumps straight to completed. */
|
||||
await input.onProgress?.(progress);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,10 @@ export interface RunImportInput {
|
|||
filepath: string;
|
||||
userId: string;
|
||||
tenantId?: string;
|
||||
/** Retention deadline for everything this run writes, when the deployment
|
||||
* sets one. The batch builder already applies it to conversations and
|
||||
* messages; assets have to carry it too or they outlive the chats. */
|
||||
expiredAt?: Date;
|
||||
defaultModel: string;
|
||||
/** The format `inspectExport` already identified. Omit it and `runImport`
|
||||
* re-reads the first shard to detect it. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue