mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🆖 fix: Recognize Confirmed RAG Embedding Refusals (#18404)
* 🆖 fix: Recognize Confirmed RAG Embedding Refusals
* test: Separate Resource Lock Retry Readiness Budget
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
b2244f7b53
commit
dcbccb4341
9 changed files with 581 additions and 89 deletions
|
|
@ -1,13 +1,8 @@
|
|||
const fs = require('fs');
|
||||
const axios = require('axios');
|
||||
const FormData = require('form-data');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { FileSources } = require('librechat-data-provider');
|
||||
const {
|
||||
logAxiosError,
|
||||
generateShortLivedToken,
|
||||
createVectorFileDeleter,
|
||||
dispatchFileStorageOperation,
|
||||
createVectorFileUploader,
|
||||
} = require('@librechat/api');
|
||||
|
||||
const deleteVectors = (req, file) =>
|
||||
|
|
@ -19,87 +14,12 @@ const deleteVectors = (req, file) =>
|
|||
namespaceConcurrency: req.config?.fileConfig?.vectorDeletionConcurrency,
|
||||
})(req, file);
|
||||
|
||||
/**
|
||||
* Uploads a file to the configured Vector database
|
||||
*
|
||||
* @param {Object} params - The params object.
|
||||
* @param {Object} params.req - The request object from Express. It should have a `user` property with an `id` representing the user
|
||||
* @param {Express.Multer.File} params.file - The file object, which is part of the request. The file object should
|
||||
* have a `path` property that points to the location of the uploaded file.
|
||||
* @param {string} params.file_id - The file ID.
|
||||
* @param {string} [params.entity_id] - The entity ID for shared resources.
|
||||
* @param {Object} [params.storageMetadata] - Storage metadata for dual storage pattern.
|
||||
*
|
||||
* @returns {Promise<{ filepath: string, bytes: number }>}
|
||||
* A promise that resolves to an object containing:
|
||||
* - filepath: The path where the file is saved.
|
||||
* - bytes: The size of the file in bytes.
|
||||
*/
|
||||
async function uploadVectors({
|
||||
req,
|
||||
file,
|
||||
file_id,
|
||||
entity_id,
|
||||
storageMetadata,
|
||||
signal,
|
||||
onUploadStart,
|
||||
}) {
|
||||
if (!process.env.RAG_API_URL) {
|
||||
throw new Error('RAG_API_URL not defined');
|
||||
}
|
||||
|
||||
try {
|
||||
const jwtToken = generateShortLivedToken(req.user.id);
|
||||
const formData = new FormData();
|
||||
formData.append('file_id', file_id);
|
||||
formData.append('file', fs.createReadStream(file.path));
|
||||
if (entity_id != null && entity_id) {
|
||||
formData.append('entity_id', entity_id);
|
||||
}
|
||||
|
||||
// Include storage metadata for RAG API to store with embeddings
|
||||
if (storageMetadata) {
|
||||
formData.append('storage_metadata', JSON.stringify(storageMetadata));
|
||||
}
|
||||
|
||||
const formHeaders = formData.getHeaders();
|
||||
|
||||
const response = await dispatchFileStorageOperation(onUploadStart, () =>
|
||||
axios.post(`${process.env.RAG_API_URL}/embed`, formData, {
|
||||
...(signal ? { signal } : {}),
|
||||
headers: {
|
||||
Authorization: `Bearer ${jwtToken}`,
|
||||
accept: 'application/json',
|
||||
...formHeaders,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const responseData = response.data;
|
||||
logger.debug('Response from embedding file', responseData);
|
||||
|
||||
if (responseData.known_type === false) {
|
||||
throw new Error(`File embedding failed. The filetype ${file.mimetype} is not supported`);
|
||||
}
|
||||
|
||||
if (!responseData.status) {
|
||||
throw new Error('File embedding failed.');
|
||||
}
|
||||
|
||||
return {
|
||||
bytes: file.size,
|
||||
filename: file.originalname,
|
||||
filepath: FileSources.vectordb,
|
||||
embedded: Boolean(responseData.known_type),
|
||||
};
|
||||
} catch (error) {
|
||||
logAxiosError({
|
||||
error,
|
||||
message: 'Error uploading vectors',
|
||||
});
|
||||
throw new Error(error.message || 'An error occurred during file upload.', { cause: error });
|
||||
}
|
||||
}
|
||||
const uploadVectors = (params) =>
|
||||
createVectorFileUploader({
|
||||
baseUrl: process.env.RAG_API_URL,
|
||||
httpClient: axios,
|
||||
generateToken: generateShortLivedToken,
|
||||
})(params);
|
||||
|
||||
module.exports = {
|
||||
deleteVectors,
|
||||
|
|
|
|||
62
api/server/services/Files/VectorDB/crud.spec.js
Normal file
62
api/server/services/Files/VectorDB/crud.spec.js
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const axios = require('axios');
|
||||
const { mkdtemp, writeFile, rm } = require('node:fs/promises');
|
||||
const { RagEmbeddingError, isDefinitiveFileStorageFailure } = require('@librechat/api');
|
||||
const { uploadVectors } = require('./crud');
|
||||
|
||||
let directory;
|
||||
let params;
|
||||
const previousUrl = process.env.RAG_API_URL;
|
||||
|
||||
beforeAll(async () => {
|
||||
directory = await mkdtemp(path.join(os.tmpdir(), 'rag-wiring-'));
|
||||
const filepath = path.join(directory, 'notes.txt');
|
||||
await writeFile(filepath, 'notes');
|
||||
params = {
|
||||
req: { user: { id: 'user' } },
|
||||
file: { path: filepath, originalname: 'notes.txt', mimetype: 'text/plain', size: 5 },
|
||||
file_id: 'file',
|
||||
};
|
||||
process.env.RAG_API_URL = 'http://rag.test';
|
||||
});
|
||||
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
afterAll(async () => {
|
||||
if (previousUrl === undefined) delete process.env.RAG_API_URL;
|
||||
else process.env.RAG_API_URL = previousUrl;
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('wires a successful embedding through the real package export', async () => {
|
||||
jest.spyOn(axios, 'post').mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: { status: true, known_type: true, file_id: 'file' },
|
||||
});
|
||||
await expect(uploadVectors(params)).resolves.toMatchObject({ embedded: true, bytes: 5 });
|
||||
});
|
||||
|
||||
it('preserves the refusal type across the CommonJS package boundary', async () => {
|
||||
jest.spyOn(axios, 'post').mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: {
|
||||
status: false,
|
||||
file_id: 'file',
|
||||
error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'none' },
|
||||
},
|
||||
});
|
||||
const error = await uploadVectors(params).catch((error) => error);
|
||||
expect(error).toBeInstanceOf(RagEmbeddingError);
|
||||
expect(isDefinitiveFileStorageFailure(error)).toBe(true);
|
||||
});
|
||||
|
||||
it('does not promote legacy refusal flags to proof of remote absence', async () => {
|
||||
jest.spyOn(axios, 'post').mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: { status: false, known_type: false, message: 'private-document' },
|
||||
});
|
||||
const error = await uploadVectors(params).catch((error) => error);
|
||||
expect(error.code).toBe('RAG_EMBEDDING_OUTCOME_UNKNOWN');
|
||||
expect(isDefinitiveFileStorageFailure(error)).toBe(false);
|
||||
expect(String(error)).not.toContain('private-document');
|
||||
});
|
||||
|
|
@ -26,6 +26,8 @@ export * from './upload/fallback';
|
|||
export * from './rag';
|
||||
export * from './rag/context';
|
||||
export * from './rag/search';
|
||||
export * from './rag/errors';
|
||||
export * from './rag/upload';
|
||||
export * from './regexEngine';
|
||||
export * from './resize';
|
||||
export * from './retention';
|
||||
|
|
|
|||
|
|
@ -1,11 +1,18 @@
|
|||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import mongoose from 'mongoose';
|
||||
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { FileContext, FileSources } from 'librechat-data-provider';
|
||||
import { createMethods, createModels } from '@librechat/data-schemas';
|
||||
import type { CodeFileCommitData, FileProvisionCommit, IMongoFile } from '@librechat/data-schemas';
|
||||
import type { AxiosInstance } from 'axios';
|
||||
import { createFileProvisionRecovery, withFileProvisioning, withFileDeletion } from './operations';
|
||||
import { createVectorFileUploader } from './rag/upload';
|
||||
|
||||
let mongo: MongoMemoryServer;
|
||||
let directory: string;
|
||||
let uploadPath: string;
|
||||
let File: mongoose.Model<IMongoFile>;
|
||||
let methods: ReturnType<typeof createMethods>;
|
||||
const owner = new mongoose.Types.ObjectId();
|
||||
|
|
@ -19,7 +26,10 @@ const ref = {
|
|||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryServer.create();
|
||||
directory = await mkdtemp(path.join(os.tmpdir(), 'rag-lifecycle-'));
|
||||
uploadPath = path.join(directory, 'notes.txt');
|
||||
await writeFile(uploadPath, 'notes');
|
||||
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
|
||||
createModels(mongoose);
|
||||
await mongoose.connect(mongo.getUri());
|
||||
File = mongoose.models.File as mongoose.Model<IMongoFile>;
|
||||
|
|
@ -29,6 +39,7 @@ beforeAll(async () => {
|
|||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongo?.stop();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
|
|
@ -311,3 +322,213 @@ it('does not overwrite fresh output bytes while deletion intent remains after pr
|
|||
(await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.deletionPending,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
const refusalReceipt = {
|
||||
status: false,
|
||||
file_id: file.file_id,
|
||||
known_type: true,
|
||||
error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'none' },
|
||||
};
|
||||
|
||||
function ragProvision(data: unknown) {
|
||||
const storage = { ...methods, recovery: createFileProvisionRecovery() };
|
||||
const post = jest.fn().mockResolvedValue({ status: 200, data });
|
||||
const upload = createVectorFileUploader({
|
||||
baseUrl: 'http://rag.test',
|
||||
httpClient: { post } as Pick<AxiosInstance, 'post'>,
|
||||
generateToken: () => 'test-token',
|
||||
});
|
||||
const options = {
|
||||
file,
|
||||
userId: scope.userId,
|
||||
storage,
|
||||
destination: 'search:user',
|
||||
operation: async (_claim: FileProvisionCommit['claim'], onUploadStart: () => void) => ({
|
||||
value: await upload({
|
||||
req: { user: { id: scope.userId } },
|
||||
file: { path: uploadPath, originalname: 'notes.txt', size: 5 },
|
||||
file_id: file.file_id,
|
||||
onUploadStart,
|
||||
}),
|
||||
commit: { embedded: true, entityId: scope.userId },
|
||||
}),
|
||||
};
|
||||
return { options, storage, post };
|
||||
}
|
||||
|
||||
it('releases a confirmed RAG refusal and allows a fresh successful retry', async () => {
|
||||
const { options, post } = ragProvision(refusalReceipt);
|
||||
await expect(withFileProvisioning(options)).rejects.toMatchObject({
|
||||
code: 'RAG_EMBEDDING_REFUSED',
|
||||
});
|
||||
const refused = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(refused?.storageOperations?.provisioning).toEqual([]);
|
||||
expect(refused?.embedded).not.toBe(true);
|
||||
expect(refused?.metadata?.embeddedEntities).toBeUndefined();
|
||||
|
||||
post.mockResolvedValueOnce({
|
||||
status: 200,
|
||||
data: { status: true, known_type: true, file_id: file.file_id },
|
||||
});
|
||||
await expect(withFileProvisioning(options)).resolves.toMatchObject({ embedded: true });
|
||||
expect(post).toHaveBeenCalledTimes(2);
|
||||
const stored = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(stored?.storageOperations?.provisioning).toEqual([]);
|
||||
expect(stored?.embedded).toBe(true);
|
||||
expect(stored?.metadata?.embeddedEntities).toContain(scope.userId);
|
||||
});
|
||||
|
||||
it('allows owner-scoped deletion after refusal without publishing remote pointers', async () => {
|
||||
const { options, storage } = ragProvision(refusalReceipt);
|
||||
await expect(withFileProvisioning(options)).rejects.toMatchObject({
|
||||
code: 'RAG_EMBEDDING_REFUSED',
|
||||
});
|
||||
const stored = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(
|
||||
await methods.acquireFileDeletion({
|
||||
...scope,
|
||||
userId: new mongoose.Types.ObjectId().toString(),
|
||||
token: 'foreign',
|
||||
updatedAt: stored?.updatedAt,
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(
|
||||
await methods.acquireFileDeletion({
|
||||
...scope,
|
||||
tenantId: 'other',
|
||||
token: 'foreign',
|
||||
updatedAt: stored?.updatedAt,
|
||||
}),
|
||||
).toBeNull();
|
||||
const remove = jest.fn(async () => 'deleted');
|
||||
await expect(
|
||||
withFileDeletion({
|
||||
file: { ...file, updatedAt: stored?.updatedAt },
|
||||
userId: scope.userId,
|
||||
storage,
|
||||
operation: remove,
|
||||
}),
|
||||
).resolves.toBe('deleted');
|
||||
expect(remove).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([{ status: false, known_type: true }, { status: true, known_type: false }, {}, null])(
|
||||
'retains the RAG claim and blocks retry and deletion for uncertain response %j',
|
||||
async (data) => {
|
||||
const { options, storage, post } = ragProvision(data);
|
||||
await expect(withFileProvisioning(options)).rejects.toThrow('outcome is unknown');
|
||||
const pending = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(pending?.storageOperations?.provisioning).toHaveLength(1);
|
||||
await expect(withFileProvisioning(options)).rejects.toThrow('outcome is unknown');
|
||||
const remove = jest.fn();
|
||||
await expect(
|
||||
withFileDeletion({ file, userId: scope.userId, storage, operation: remove }),
|
||||
).rejects.toThrow('outcome is unknown');
|
||||
// A restarted process cannot take over the durable claim either.
|
||||
expect(
|
||||
await methods.acquireFileDeletion({
|
||||
...scope,
|
||||
token: 'restart',
|
||||
updatedAt: pending?.updatedAt,
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(remove).not.toHaveBeenCalled();
|
||||
expect(post).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
(await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.provisioning,
|
||||
).toEqual(pending?.storageOperations?.provisioning);
|
||||
},
|
||||
);
|
||||
|
||||
it('never clears another provisioning request or its committed namespace on refusal', async () => {
|
||||
const { options } = ragProvision(refusalReceipt);
|
||||
const peer = { ...scope, token: 'other-request' };
|
||||
expect(await methods.acquireFileProvision(peer)).toBe(true);
|
||||
expect(
|
||||
await methods.commitFileProvision({ claim: peer, embedded: true, entityId: 'agent-peer' }),
|
||||
).toBe(true);
|
||||
await expect(withFileProvisioning(options)).rejects.toMatchObject({
|
||||
code: 'RAG_EMBEDDING_REFUSED',
|
||||
});
|
||||
const stored = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(stored?.storageOperations?.provisioning).toEqual([peer.token]);
|
||||
expect(stored?.metadata?.embeddedEntities).toEqual(['agent-peer']);
|
||||
expect(stored?.embedded).toBe(true);
|
||||
expect(
|
||||
await methods.acquireFileDeletion({ ...scope, token: 'blocked', updatedAt: stored?.updatedAt }),
|
||||
).toBeNull();
|
||||
expect(await methods.releaseFileProvision(peer)).toBe(true);
|
||||
expect(
|
||||
await methods.acquireFileDeletion({ ...scope, token: 'delete', updatedAt: stored?.updatedAt }),
|
||||
).toEqual({ resumed: false });
|
||||
});
|
||||
|
||||
it('blocks deletion while refusal is in flight and releases only after its receipt', async () => {
|
||||
const { options, post } = ragProvision(refusalReceipt);
|
||||
let respond!: (value: { status: number; data: unknown }) => void;
|
||||
let dispatched!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
dispatched = resolve;
|
||||
});
|
||||
post.mockImplementationOnce(() => {
|
||||
dispatched();
|
||||
return new Promise((resolve) => {
|
||||
respond = resolve;
|
||||
});
|
||||
});
|
||||
const attempt = withFileProvisioning(options);
|
||||
const rejected = attempt.catch((error: unknown) => error);
|
||||
await started;
|
||||
const pending = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(
|
||||
await methods.acquireFileDeletion({
|
||||
...scope,
|
||||
token: 'blocked',
|
||||
updatedAt: pending?.updatedAt,
|
||||
}),
|
||||
).toBeNull();
|
||||
respond({ status: 200, data: refusalReceipt });
|
||||
await expect(rejected).resolves.toMatchObject({ code: 'RAG_EMBEDDING_REFUSED' });
|
||||
const stored = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(
|
||||
await methods.acquireFileDeletion({ ...scope, token: 'delete', updatedAt: stored?.updatedAt }),
|
||||
).toEqual({ resumed: false });
|
||||
});
|
||||
|
||||
it('recovers only its refused claim when release acknowledgement is lost', async () => {
|
||||
const { options, storage } = ragProvision(refusalReceipt);
|
||||
const release = jest
|
||||
.spyOn(storage, 'releaseFileProvision')
|
||||
.mockRejectedValueOnce(new Error('database unavailable'));
|
||||
await expect(withFileProvisioning(options)).rejects.toMatchObject({
|
||||
code: 'RAG_EMBEDDING_REFUSED',
|
||||
});
|
||||
const pending = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(pending?.storageOperations?.provisioning).toHaveLength(1);
|
||||
await storage.recovery.recover(scope, storage);
|
||||
expect(release).toHaveBeenCalledTimes(2);
|
||||
expect(release.mock.calls[0][0]).toEqual(release.mock.calls[1][0]);
|
||||
expect(
|
||||
(await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.provisioning,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['ETIMEDOUT', 'ECONNRESET', 'ERR_CANCELED'])(
|
||||
'keeps the durable claim on RAG transport uncertainty: %s',
|
||||
async (code) => {
|
||||
const { options, storage, post } = ragProvision(refusalReceipt);
|
||||
post.mockRejectedValueOnce(Object.assign(new Error('private upstream payload'), { code }));
|
||||
await expect(withFileProvisioning(options)).rejects.toThrow('outcome is unknown');
|
||||
const stored = await File.findOne({ file_id: file.file_id }).lean();
|
||||
expect(stored?.storageOperations?.provisioning).toHaveLength(1);
|
||||
await expect(storage.recovery.recover(scope, storage)).rejects.toThrow('outcome is unknown');
|
||||
expect(
|
||||
await methods.acquireFileDeletion({
|
||||
...scope,
|
||||
token: 'delete',
|
||||
updatedAt: stored?.updatedAt,
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(post).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import type {
|
|||
FileProvisionCommit,
|
||||
} from '@librechat/data-schemas';
|
||||
|
||||
import { RagEmbeddingError } from './rag/errors';
|
||||
|
||||
export type FileStorageUploadObserver = (outcome?: 'rejected') => void;
|
||||
|
||||
export type RawProvisionStorage = Pick<
|
||||
|
|
@ -90,6 +92,9 @@ export function isDefinitiveFileStorageFailure(error: unknown): boolean {
|
|||
// RAG wraps the HTTP failure; inspect at most two nested causes.
|
||||
for (let depth = 0; depth < 3; depth++) {
|
||||
if (typeof current !== 'object' || current === null) return false;
|
||||
if (current instanceof RagEmbeddingError) {
|
||||
return current.code === 'RAG_EMBEDDING_REFUSED';
|
||||
}
|
||||
let status: unknown;
|
||||
let hasStatus = true;
|
||||
if (
|
||||
|
|
|
|||
16
packages/api/src/files/rag/errors.ts
Normal file
16
packages/api/src/files/rag/errors.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
export type RagEmbeddingFailureCode = 'RAG_EMBEDDING_REFUSED' | 'RAG_EMBEDDING_OUTCOME_UNKNOWN';
|
||||
|
||||
/** Only a validated no-side-effect receipt establishes a definitive refusal. */
|
||||
export class RagEmbeddingError extends Error {
|
||||
readonly code: RagEmbeddingFailureCode;
|
||||
|
||||
constructor(code: RagEmbeddingFailureCode) {
|
||||
super(
|
||||
code === 'RAG_EMBEDDING_REFUSED'
|
||||
? 'File embedding was refused. You can retry or delete the file.'
|
||||
: 'File embedding was not confirmed.',
|
||||
);
|
||||
this.name = 'RagEmbeddingError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
149
packages/api/src/files/rag/upload.spec.ts
Normal file
149
packages/api/src/files/rag/upload.spec.ts
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import FormData from 'form-data';
|
||||
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
|
||||
import type { AxiosInstance } from 'axios';
|
||||
import { isDefinitiveFileStorageFailure } from '../operations';
|
||||
import { createVectorFileUploader } from './upload';
|
||||
import { RagEmbeddingError } from './errors';
|
||||
|
||||
let directory: string;
|
||||
let file: { path: string; originalname: string; size: number; mimetype: string };
|
||||
const fileId = 'file';
|
||||
const receipt = {
|
||||
status: false,
|
||||
file_id: fileId,
|
||||
error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'none' },
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
directory = await mkdtemp(path.join(os.tmpdir(), 'rag-upload-'));
|
||||
file = {
|
||||
path: path.join(directory, 'notes.txt'),
|
||||
originalname: 'notes.txt',
|
||||
size: 5,
|
||||
mimetype: 'text/plain',
|
||||
};
|
||||
await writeFile(file.path, 'notes');
|
||||
});
|
||||
afterAll(async () => rm(directory, { recursive: true, force: true }));
|
||||
|
||||
function setup(data: unknown, status = 200) {
|
||||
const post = jest.fn().mockResolvedValue({ status, data });
|
||||
const generateToken = jest.fn(() => 'test-token');
|
||||
const upload = createVectorFileUploader({
|
||||
baseUrl: 'http://rag.test',
|
||||
httpClient: { post } as Pick<AxiosInstance, 'post'>,
|
||||
generateToken,
|
||||
});
|
||||
const onUploadStart = jest.fn();
|
||||
const params = { req: { user: { id: 'user' } }, file, file_id: fileId, onUploadStart };
|
||||
return { post, generateToken, upload, params, onUploadStart };
|
||||
}
|
||||
|
||||
it('preserves successful embedding, multipart fields and cancellation', async () => {
|
||||
const { upload, params, post, generateToken, onUploadStart } = setup({
|
||||
status: true,
|
||||
known_type: true,
|
||||
file_id: fileId,
|
||||
});
|
||||
const signal = new AbortController().signal;
|
||||
await expect(
|
||||
upload({ ...params, entity_id: 'agent', storageMetadata: { source: 'local' }, signal }),
|
||||
).resolves.toEqual({ bytes: 5, filename: 'notes.txt', filepath: 'vectordb', embedded: true });
|
||||
expect(generateToken).toHaveBeenCalledWith('user');
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
'http://rag.test/embed',
|
||||
expect.any(FormData),
|
||||
expect.objectContaining({
|
||||
signal,
|
||||
headers: expect.objectContaining({ Authorization: 'Bearer test-token' }),
|
||||
}),
|
||||
);
|
||||
expect(onUploadStart.mock.calls).toEqual([[]]);
|
||||
});
|
||||
|
||||
it('recognizes only a request-matched no-side-effect refusal receipt', async () => {
|
||||
const { upload, params, onUploadStart } = setup(receipt);
|
||||
const error = await upload(params).catch((error: Error) => error);
|
||||
expect(error).toBeInstanceOf(RagEmbeddingError);
|
||||
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_REFUSED' });
|
||||
expect(isDefinitiveFileStorageFailure(error)).toBe(true);
|
||||
expect(onUploadStart.mock.calls).toEqual([[], ['rejected']]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
null,
|
||||
undefined,
|
||||
'',
|
||||
[],
|
||||
{},
|
||||
{ status: true },
|
||||
{ status: 1, known_type: true },
|
||||
{ status: true, known_type: 'true' },
|
||||
{ status: true, known_type: false },
|
||||
{ status: false, known_type: true },
|
||||
{ status: false, known_type: false },
|
||||
{ status: false, message: 'RAG_EMBEDDING_REFUSED side_effects none' },
|
||||
{ ...receipt, file_id: 'another-file' },
|
||||
{ ...receipt, file_id: undefined },
|
||||
{ ...receipt, status: true },
|
||||
{ ...receipt, error: { code: 'RAG_EMBEDDING_REFUSED' } },
|
||||
{ ...receipt, error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'unknown' } },
|
||||
{ ...receipt, error: { code: 'other', side_effects: 'none' } },
|
||||
{ status: true, known_type: true, error: receipt.error },
|
||||
{ status: true, known_type: true, file_id: 'another-file' },
|
||||
])('retains uncertainty for a legacy, contradictory or malformed response: %j', async (data) => {
|
||||
const { upload, params, onUploadStart } = setup(data);
|
||||
const error = await upload(params).catch((error: Error) => error);
|
||||
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
|
||||
expect(isDefinitiveFileStorageFailure(error)).toBe(false);
|
||||
expect(onUploadStart.mock.calls).toEqual([[]]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ code: 'ETIMEDOUT' },
|
||||
{ code: 'ECONNRESET' },
|
||||
{ code: 'ERR_CANCELED' },
|
||||
{ response: { status: 400 } },
|
||||
{ response: { status: 422 } },
|
||||
{ response: { status: 500 } },
|
||||
{ code: 'RAG_EMBEDDING_REFUSED' },
|
||||
])('never infers remote absence from a rejected request: %j', async (details) => {
|
||||
const { upload, params, post, onUploadStart } = setup(undefined);
|
||||
const secret = 'private-document SQL bearer-test-token';
|
||||
post.mockRejectedValueOnce(Object.assign(new Error(secret), details));
|
||||
const error = await upload(params).catch((error: Error) => error);
|
||||
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
|
||||
expect(error).not.toHaveProperty('cause');
|
||||
expect(String(error)).not.toContain(secret);
|
||||
expect(isDefinitiveFileStorageFailure(error)).toBe(false);
|
||||
expect(onUploadStart.mock.calls).toEqual([[]]);
|
||||
});
|
||||
|
||||
it('does not trust a no-side-effect body outside a successful HTTP response', async () => {
|
||||
const { upload, params } = setup(receipt, 500);
|
||||
await expect(upload(params)).rejects.toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
|
||||
});
|
||||
|
||||
it.each([undefined, NaN, '200', 199, 300, 400, 422, 500])(
|
||||
'rejects a non-success or malformed HTTP status without releasing the claim: %j',
|
||||
async (status) => {
|
||||
const { upload, params, post } = setup(receipt);
|
||||
post.mockResolvedValueOnce({ status, data: receipt });
|
||||
await expect(upload(params)).rejects.toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
|
||||
},
|
||||
);
|
||||
|
||||
it('sanitizes credential setup failures before dispatch', async () => {
|
||||
const { upload, params, generateToken, post, onUploadStart } = setup(receipt);
|
||||
generateToken.mockImplementationOnce(() => {
|
||||
throw new Error('private-signing-key');
|
||||
});
|
||||
const error = await upload(params).catch((error: Error) => error);
|
||||
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
|
||||
expect(String(error)).not.toContain('private-signing-key');
|
||||
expect(error).not.toHaveProperty('cause');
|
||||
expect(post).not.toHaveBeenCalled();
|
||||
expect(onUploadStart).not.toHaveBeenCalled();
|
||||
});
|
||||
117
packages/api/src/files/rag/upload.ts
Normal file
117
packages/api/src/files/rag/upload.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import FormData from 'form-data';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { FileSources } from 'librechat-data-provider';
|
||||
import type { AxiosInstance } from 'axios';
|
||||
import type { ReadStream } from 'node:fs';
|
||||
import type { FileStorageUploadObserver } from '../operations';
|
||||
import { dispatchFileStorageOperation } from '../dispatch';
|
||||
import { getSafeErrorMetadata } from '~/utils/errors';
|
||||
import { RagEmbeddingError } from './errors';
|
||||
|
||||
type VectorUpload = {
|
||||
req: { user: { id: string } };
|
||||
file: { path: string; originalname: string; mimetype?: string; size?: number };
|
||||
file_id: string;
|
||||
entity_id?: string;
|
||||
storageMetadata?: Record<string, unknown>;
|
||||
signal?: AbortSignal;
|
||||
onUploadStart?: FileStorageUploadObserver;
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value != null && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** Legacy status/known_type flags do not prove rollback completed or that inserts never began. */
|
||||
function confirmEmbedding(data: unknown, fileId: string): void {
|
||||
if (isRecord(data)) {
|
||||
if (
|
||||
data.status === false &&
|
||||
data.file_id === fileId &&
|
||||
isRecord(data.error) &&
|
||||
data.error.code === 'RAG_EMBEDDING_REFUSED' &&
|
||||
data.error.side_effects === 'none'
|
||||
) {
|
||||
throw new RagEmbeddingError('RAG_EMBEDDING_REFUSED');
|
||||
}
|
||||
if (
|
||||
data.status === true &&
|
||||
data.known_type === true &&
|
||||
!('error' in data) &&
|
||||
(data.file_id === undefined || data.file_id === fileId)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
|
||||
}
|
||||
|
||||
/** HTTP clients and credentials are supplied by the host, not application singletons. */
|
||||
export function createVectorFileUploader({
|
||||
baseUrl,
|
||||
httpClient,
|
||||
generateToken,
|
||||
}: {
|
||||
baseUrl?: string;
|
||||
httpClient: Pick<AxiosInstance, 'post'>;
|
||||
generateToken: (userId: string) => string;
|
||||
}): (params: VectorUpload) => Promise<{
|
||||
bytes?: number;
|
||||
filename: string;
|
||||
filepath: FileSources;
|
||||
embedded: boolean;
|
||||
}> {
|
||||
return async ({ req, file, file_id, entity_id, storageMetadata, signal, onUploadStart }) => {
|
||||
let stream: ReadStream | undefined;
|
||||
try {
|
||||
if (!baseUrl) throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
|
||||
const token = generateToken(req.user.id);
|
||||
const body = new FormData();
|
||||
body.append('file_id', file_id);
|
||||
stream = createReadStream(file.path);
|
||||
body.append('file', stream);
|
||||
if (entity_id) body.append('entity_id', entity_id);
|
||||
if (storageMetadata) body.append('storage_metadata', JSON.stringify(storageMetadata));
|
||||
|
||||
return await dispatchFileStorageOperation(onUploadStart, async () => {
|
||||
try {
|
||||
const response = await httpClient.post<unknown>(`${baseUrl}/embed`, body, {
|
||||
...(signal ? { signal } : {}),
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
accept: 'application/json',
|
||||
...body.getHeaders(),
|
||||
},
|
||||
});
|
||||
if (
|
||||
!Number.isInteger(response.status) ||
|
||||
response.status < 200 ||
|
||||
response.status >= 300
|
||||
) {
|
||||
throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
|
||||
}
|
||||
confirmEmbedding(response.data, file_id);
|
||||
return {
|
||||
bytes: file.size,
|
||||
filename: file.originalname,
|
||||
filepath: FileSources.vectordb,
|
||||
embedded: true,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('[uploadVectors] Embedding was not confirmed', getSafeErrorMetadata(error));
|
||||
// Upstream errors can contain document text, credentials or SQL, including in causes.
|
||||
throw error instanceof RagEmbeddingError
|
||||
? error
|
||||
: new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof RagEmbeddingError) throw error;
|
||||
logger.error('[uploadVectors] Embedding setup failed', getSafeErrorMetadata(error));
|
||||
throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
|
||||
} finally {
|
||||
stream?.destroy();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -181,7 +181,7 @@ describe('resource lease Mongo adapter', () => {
|
|||
jest.useRealTimers();
|
||||
}
|
||||
expect(await harness.read(lock.key)).toBeNull();
|
||||
expect(await methods.acquireResourceLock(lock)).toBe(true);
|
||||
expect(await methods.acquireResourceLock({ ...lock, readyWaitMs: 2_000 })).toBe(true);
|
||||
expect(build).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue