🆖 fix: Recognize Confirmed RAG Embedding Refusals (#18404)

* 🆖 fix: Recognize Confirmed RAG Embedding Refusals

* test: Separate Resource Lock Retry Readiness Budget

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-11-18 09:02:46 -05:00 • committed by GitHub
parent b2244f7b53
commit dcbccb4341
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 581 additions and 89 deletions

View file

@ -1,13 +1,8 @@
const fs = require('fs');
const axios = require('axios');
const FormData = require('form-data');
const { logger } = require('@librechat/data-schemas');
const { FileSources } = require('librechat-data-provider');
const {
logAxiosError,
generateShortLivedToken,
createVectorFileDeleter,
dispatchFileStorageOperation,
createVectorFileUploader,
} = require('@librechat/api');
const deleteVectors = (req, file) =>
@ -19,87 +14,12 @@ const deleteVectors = (req, file) =>
namespaceConcurrency: req.config?.fileConfig?.vectorDeletionConcurrency,
})(req, file);
/**
* Uploads a file to the configured Vector database
*
* @param {Object} params - The params object.
* @param {Object} params.req - The request object from Express. It should have a `user` property with an `id` representing the user
* @param {Express.Multer.File} params.file - The file object, which is part of the request. The file object should
* have a `path` property that points to the location of the uploaded file.
* @param {string} params.file_id - The file ID.
* @param {string} [params.entity_id] - The entity ID for shared resources.
* @param {Object} [params.storageMetadata] - Storage metadata for dual storage pattern.
*
* @returns {Promise<{ filepath: string, bytes: number }>}
* A promise that resolves to an object containing:
* - filepath: The path where the file is saved.
* - bytes: The size of the file in bytes.
*/
async function uploadVectors({
req,
file,
file_id,
entity_id,
storageMetadata,
signal,
onUploadStart,
}) {
if (!process.env.RAG_API_URL) {
throw new Error('RAG_API_URL not defined');
}
try {
const jwtToken = generateShortLivedToken(req.user.id);
const formData = new FormData();
formData.append('file_id', file_id);
formData.append('file', fs.createReadStream(file.path));
if (entity_id != null && entity_id) {
formData.append('entity_id', entity_id);
}
// Include storage metadata for RAG API to store with embeddings
if (storageMetadata) {
formData.append('storage_metadata', JSON.stringify(storageMetadata));
}
const formHeaders = formData.getHeaders();
const response = await dispatchFileStorageOperation(onUploadStart, () =>
axios.post(`${process.env.RAG_API_URL}/embed`, formData, {
...(signal ? { signal } : {}),
headers: {
Authorization: `Bearer ${jwtToken}`,
accept: 'application/json',
...formHeaders,
},
}),
);
const responseData = response.data;
logger.debug('Response from embedding file', responseData);
if (responseData.known_type === false) {
throw new Error(`File embedding failed. The filetype ${file.mimetype} is not supported`);
}
if (!responseData.status) {
throw new Error('File embedding failed.');
}
return {
bytes: file.size,
filename: file.originalname,
filepath: FileSources.vectordb,
embedded: Boolean(responseData.known_type),
};
} catch (error) {
logAxiosError({
error,
message: 'Error uploading vectors',
});
throw new Error(error.message || 'An error occurred during file upload.', { cause: error });
}
}
const uploadVectors = (params) =>
createVectorFileUploader({
baseUrl: process.env.RAG_API_URL,
httpClient: axios,
generateToken: generateShortLivedToken,
})(params);
module.exports = {
deleteVectors,

View file

@ -0,0 +1,62 @@
const os = require('node:os');
const path = require('node:path');
const axios = require('axios');
const { mkdtemp, writeFile, rm } = require('node:fs/promises');
const { RagEmbeddingError, isDefinitiveFileStorageFailure } = require('@librechat/api');
const { uploadVectors } = require('./crud');
let directory;
let params;
const previousUrl = process.env.RAG_API_URL;
beforeAll(async () => {
directory = await mkdtemp(path.join(os.tmpdir(), 'rag-wiring-'));
const filepath = path.join(directory, 'notes.txt');
await writeFile(filepath, 'notes');
params = {
req: { user: { id: 'user' } },
file: { path: filepath, originalname: 'notes.txt', mimetype: 'text/plain', size: 5 },
file_id: 'file',
};
process.env.RAG_API_URL = 'http://rag.test';
});
afterEach(() => jest.restoreAllMocks());
afterAll(async () => {
if (previousUrl === undefined) delete process.env.RAG_API_URL;
else process.env.RAG_API_URL = previousUrl;
await rm(directory, { recursive: true, force: true });
});
it('wires a successful embedding through the real package export', async () => {
jest.spyOn(axios, 'post').mockResolvedValueOnce({
status: 200,
data: { status: true, known_type: true, file_id: 'file' },
});
await expect(uploadVectors(params)).resolves.toMatchObject({ embedded: true, bytes: 5 });
});
it('preserves the refusal type across the CommonJS package boundary', async () => {
jest.spyOn(axios, 'post').mockResolvedValueOnce({
status: 200,
data: {
status: false,
file_id: 'file',
error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'none' },
},
});
const error = await uploadVectors(params).catch((error) => error);
expect(error).toBeInstanceOf(RagEmbeddingError);
expect(isDefinitiveFileStorageFailure(error)).toBe(true);
});
it('does not promote legacy refusal flags to proof of remote absence', async () => {
jest.spyOn(axios, 'post').mockResolvedValueOnce({
status: 200,
data: { status: false, known_type: false, message: 'private-document' },
});
const error = await uploadVectors(params).catch((error) => error);
expect(error.code).toBe('RAG_EMBEDDING_OUTCOME_UNKNOWN');
expect(isDefinitiveFileStorageFailure(error)).toBe(false);
expect(String(error)).not.toContain('private-document');
});

View file

@ -26,6 +26,8 @@ export * from './upload/fallback';
export * from './rag';
export * from './rag/context';
export * from './rag/search';
export * from './rag/errors';
export * from './rag/upload';
export * from './regexEngine';
export * from './resize';
export * from './retention';

View file

@ -1,11 +1,18 @@
import os from 'node:os';
import path from 'node:path';
import mongoose from 'mongoose';
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { FileContext, FileSources } from 'librechat-data-provider';
import { createMethods, createModels } from '@librechat/data-schemas';
import type { CodeFileCommitData, FileProvisionCommit, IMongoFile } from '@librechat/data-schemas';
import type { AxiosInstance } from 'axios';
import { createFileProvisionRecovery, withFileProvisioning, withFileDeletion } from './operations';
import { createVectorFileUploader } from './rag/upload';
let mongo: MongoMemoryServer;
let directory: string;
let uploadPath: string;
let File: mongoose.Model<IMongoFile>;
let methods: ReturnType<typeof createMethods>;
const owner = new mongoose.Types.ObjectId();
@ -19,7 +26,10 @@ const ref = {
};
beforeAll(async () => {
mongo = await MongoMemoryServer.create();
directory = await mkdtemp(path.join(os.tmpdir(), 'rag-lifecycle-'));
uploadPath = path.join(directory, 'notes.txt');
await writeFile(uploadPath, 'notes');
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
createModels(mongoose);
await mongoose.connect(mongo.getUri());
File = mongoose.models.File as mongoose.Model<IMongoFile>;
@ -29,6 +39,7 @@ beforeAll(async () => {
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
await rm(directory, { recursive: true, force: true });
});
beforeEach(async () => {
@ -311,3 +322,213 @@ it('does not overwrite fresh output bytes while deletion intent remains after pr
(await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.deletionPending,
).toBe(true);
});
const refusalReceipt = {
status: false,
file_id: file.file_id,
known_type: true,
error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'none' },
};
function ragProvision(data: unknown) {
const storage = { ...methods, recovery: createFileProvisionRecovery() };
const post = jest.fn().mockResolvedValue({ status: 200, data });
const upload = createVectorFileUploader({
baseUrl: 'http://rag.test',
httpClient: { post } as Pick<AxiosInstance, 'post'>,
generateToken: () => 'test-token',
});
const options = {
file,
userId: scope.userId,
storage,
destination: 'search:user',
operation: async (_claim: FileProvisionCommit['claim'], onUploadStart: () => void) => ({
value: await upload({
req: { user: { id: scope.userId } },
file: { path: uploadPath, originalname: 'notes.txt', size: 5 },
file_id: file.file_id,
onUploadStart,
}),
commit: { embedded: true, entityId: scope.userId },
}),
};
return { options, storage, post };
}
it('releases a confirmed RAG refusal and allows a fresh successful retry', async () => {
const { options, post } = ragProvision(refusalReceipt);
await expect(withFileProvisioning(options)).rejects.toMatchObject({
code: 'RAG_EMBEDDING_REFUSED',
});
const refused = await File.findOne({ file_id: file.file_id }).lean();
expect(refused?.storageOperations?.provisioning).toEqual([]);
expect(refused?.embedded).not.toBe(true);
expect(refused?.metadata?.embeddedEntities).toBeUndefined();
post.mockResolvedValueOnce({
status: 200,
data: { status: true, known_type: true, file_id: file.file_id },
});
await expect(withFileProvisioning(options)).resolves.toMatchObject({ embedded: true });
expect(post).toHaveBeenCalledTimes(2);
const stored = await File.findOne({ file_id: file.file_id }).lean();
expect(stored?.storageOperations?.provisioning).toEqual([]);
expect(stored?.embedded).toBe(true);
expect(stored?.metadata?.embeddedEntities).toContain(scope.userId);
});
it('allows owner-scoped deletion after refusal without publishing remote pointers', async () => {
const { options, storage } = ragProvision(refusalReceipt);
await expect(withFileProvisioning(options)).rejects.toMatchObject({
code: 'RAG_EMBEDDING_REFUSED',
});
const stored = await File.findOne({ file_id: file.file_id }).lean();
expect(
await methods.acquireFileDeletion({
...scope,
userId: new mongoose.Types.ObjectId().toString(),
token: 'foreign',
updatedAt: stored?.updatedAt,
}),
).toBeNull();
expect(
await methods.acquireFileDeletion({
...scope,
tenantId: 'other',
token: 'foreign',
updatedAt: stored?.updatedAt,
}),
).toBeNull();
const remove = jest.fn(async () => 'deleted');
await expect(
withFileDeletion({
file: { ...file, updatedAt: stored?.updatedAt },
userId: scope.userId,
storage,
operation: remove,
}),
).resolves.toBe('deleted');
expect(remove).toHaveBeenCalledTimes(1);
});
it.each([{ status: false, known_type: true }, { status: true, known_type: false }, {}, null])(
'retains the RAG claim and blocks retry and deletion for uncertain response %j',
async (data) => {
const { options, storage, post } = ragProvision(data);
await expect(withFileProvisioning(options)).rejects.toThrow('outcome is unknown');
const pending = await File.findOne({ file_id: file.file_id }).lean();
expect(pending?.storageOperations?.provisioning).toHaveLength(1);
await expect(withFileProvisioning(options)).rejects.toThrow('outcome is unknown');
const remove = jest.fn();
await expect(
withFileDeletion({ file, userId: scope.userId, storage, operation: remove }),
).rejects.toThrow('outcome is unknown');
// A restarted process cannot take over the durable claim either.
expect(
await methods.acquireFileDeletion({
...scope,
token: 'restart',
updatedAt: pending?.updatedAt,
}),
).toBeNull();
expect(remove).not.toHaveBeenCalled();
expect(post).toHaveBeenCalledTimes(1);
expect(
(await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.provisioning,
).toEqual(pending?.storageOperations?.provisioning);
},
);
it('never clears another provisioning request or its committed namespace on refusal', async () => {
const { options } = ragProvision(refusalReceipt);
const peer = { ...scope, token: 'other-request' };
expect(await methods.acquireFileProvision(peer)).toBe(true);
expect(
await methods.commitFileProvision({ claim: peer, embedded: true, entityId: 'agent-peer' }),
).toBe(true);
await expect(withFileProvisioning(options)).rejects.toMatchObject({
code: 'RAG_EMBEDDING_REFUSED',
});
const stored = await File.findOne({ file_id: file.file_id }).lean();
expect(stored?.storageOperations?.provisioning).toEqual([peer.token]);
expect(stored?.metadata?.embeddedEntities).toEqual(['agent-peer']);
expect(stored?.embedded).toBe(true);
expect(
await methods.acquireFileDeletion({ ...scope, token: 'blocked', updatedAt: stored?.updatedAt }),
).toBeNull();
expect(await methods.releaseFileProvision(peer)).toBe(true);
expect(
await methods.acquireFileDeletion({ ...scope, token: 'delete', updatedAt: stored?.updatedAt }),
).toEqual({ resumed: false });
});
it('blocks deletion while refusal is in flight and releases only after its receipt', async () => {
const { options, post } = ragProvision(refusalReceipt);
let respond!: (value: { status: number; data: unknown }) => void;
let dispatched!: () => void;
const started = new Promise<void>((resolve) => {
dispatched = resolve;
});
post.mockImplementationOnce(() => {
dispatched();
return new Promise((resolve) => {
respond = resolve;
});
});
const attempt = withFileProvisioning(options);
const rejected = attempt.catch((error: unknown) => error);
await started;
const pending = await File.findOne({ file_id: file.file_id }).lean();
expect(
await methods.acquireFileDeletion({
...scope,
token: 'blocked',
updatedAt: pending?.updatedAt,
}),
).toBeNull();
respond({ status: 200, data: refusalReceipt });
await expect(rejected).resolves.toMatchObject({ code: 'RAG_EMBEDDING_REFUSED' });
const stored = await File.findOne({ file_id: file.file_id }).lean();
expect(
await methods.acquireFileDeletion({ ...scope, token: 'delete', updatedAt: stored?.updatedAt }),
).toEqual({ resumed: false });
});
it('recovers only its refused claim when release acknowledgement is lost', async () => {
const { options, storage } = ragProvision(refusalReceipt);
const release = jest
.spyOn(storage, 'releaseFileProvision')
.mockRejectedValueOnce(new Error('database unavailable'));
await expect(withFileProvisioning(options)).rejects.toMatchObject({
code: 'RAG_EMBEDDING_REFUSED',
});
const pending = await File.findOne({ file_id: file.file_id }).lean();
expect(pending?.storageOperations?.provisioning).toHaveLength(1);
await storage.recovery.recover(scope, storage);
expect(release).toHaveBeenCalledTimes(2);
expect(release.mock.calls[0][0]).toEqual(release.mock.calls[1][0]);
expect(
(await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.provisioning,
).toEqual([]);
});
it.each(['ETIMEDOUT', 'ECONNRESET', 'ERR_CANCELED'])(
'keeps the durable claim on RAG transport uncertainty: %s',
async (code) => {
const { options, storage, post } = ragProvision(refusalReceipt);
post.mockRejectedValueOnce(Object.assign(new Error('private upstream payload'), { code }));
await expect(withFileProvisioning(options)).rejects.toThrow('outcome is unknown');
const stored = await File.findOne({ file_id: file.file_id }).lean();
expect(stored?.storageOperations?.provisioning).toHaveLength(1);
await expect(storage.recovery.recover(scope, storage)).rejects.toThrow('outcome is unknown');
expect(
await methods.acquireFileDeletion({
...scope,
token: 'delete',
updatedAt: stored?.updatedAt,
}),
).toBeNull();
expect(post).toHaveBeenCalledTimes(1);
},
);

View file

@ -7,6 +7,8 @@ import type {
FileProvisionCommit,
} from '@librechat/data-schemas';
import { RagEmbeddingError } from './rag/errors';
export type FileStorageUploadObserver = (outcome?: 'rejected') => void;
export type RawProvisionStorage = Pick<
@ -90,6 +92,9 @@ export function isDefinitiveFileStorageFailure(error: unknown): boolean {
// RAG wraps the HTTP failure; inspect at most two nested causes.
for (let depth = 0; depth < 3; depth++) {
if (typeof current !== 'object' || current === null) return false;
if (current instanceof RagEmbeddingError) {
return current.code === 'RAG_EMBEDDING_REFUSED';
}
let status: unknown;
let hasStatus = true;
if (

View file

@ -0,0 +1,16 @@
export type RagEmbeddingFailureCode = 'RAG_EMBEDDING_REFUSED' | 'RAG_EMBEDDING_OUTCOME_UNKNOWN';
/** Only a validated no-side-effect receipt establishes a definitive refusal. */
export class RagEmbeddingError extends Error {
readonly code: RagEmbeddingFailureCode;
constructor(code: RagEmbeddingFailureCode) {
super(
code === 'RAG_EMBEDDING_REFUSED'
? 'File embedding was refused. You can retry or delete the file.'
: 'File embedding was not confirmed.',
);
this.name = 'RagEmbeddingError';
this.code = code;
}
}

View file

@ -0,0 +1,149 @@
import os from 'node:os';
import path from 'node:path';
import FormData from 'form-data';
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import type { AxiosInstance } from 'axios';
import { isDefinitiveFileStorageFailure } from '../operations';
import { createVectorFileUploader } from './upload';
import { RagEmbeddingError } from './errors';
let directory: string;
let file: { path: string; originalname: string; size: number; mimetype: string };
const fileId = 'file';
const receipt = {
status: false,
file_id: fileId,
error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'none' },
};
beforeAll(async () => {
directory = await mkdtemp(path.join(os.tmpdir(), 'rag-upload-'));
file = {
path: path.join(directory, 'notes.txt'),
originalname: 'notes.txt',
size: 5,
mimetype: 'text/plain',
};
await writeFile(file.path, 'notes');
});
afterAll(async () => rm(directory, { recursive: true, force: true }));
function setup(data: unknown, status = 200) {
const post = jest.fn().mockResolvedValue({ status, data });
const generateToken = jest.fn(() => 'test-token');
const upload = createVectorFileUploader({
baseUrl: 'http://rag.test',
httpClient: { post } as Pick<AxiosInstance, 'post'>,
generateToken,
});
const onUploadStart = jest.fn();
const params = { req: { user: { id: 'user' } }, file, file_id: fileId, onUploadStart };
return { post, generateToken, upload, params, onUploadStart };
}
it('preserves successful embedding, multipart fields and cancellation', async () => {
const { upload, params, post, generateToken, onUploadStart } = setup({
status: true,
known_type: true,
file_id: fileId,
});
const signal = new AbortController().signal;
await expect(
upload({ ...params, entity_id: 'agent', storageMetadata: { source: 'local' }, signal }),
).resolves.toEqual({ bytes: 5, filename: 'notes.txt', filepath: 'vectordb', embedded: true });
expect(generateToken).toHaveBeenCalledWith('user');
expect(post).toHaveBeenCalledWith(
'http://rag.test/embed',
expect.any(FormData),
expect.objectContaining({
signal,
headers: expect.objectContaining({ Authorization: 'Bearer test-token' }),
}),
);
expect(onUploadStart.mock.calls).toEqual([[]]);
});
it('recognizes only a request-matched no-side-effect refusal receipt', async () => {
const { upload, params, onUploadStart } = setup(receipt);
const error = await upload(params).catch((error: Error) => error);
expect(error).toBeInstanceOf(RagEmbeddingError);
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_REFUSED' });
expect(isDefinitiveFileStorageFailure(error)).toBe(true);
expect(onUploadStart.mock.calls).toEqual([[], ['rejected']]);
});
it.each([
null,
undefined,
'',
[],
{},
{ status: true },
{ status: 1, known_type: true },
{ status: true, known_type: 'true' },
{ status: true, known_type: false },
{ status: false, known_type: true },
{ status: false, known_type: false },
{ status: false, message: 'RAG_EMBEDDING_REFUSED side_effects none' },
{ ...receipt, file_id: 'another-file' },
{ ...receipt, file_id: undefined },
{ ...receipt, status: true },
{ ...receipt, error: { code: 'RAG_EMBEDDING_REFUSED' } },
{ ...receipt, error: { code: 'RAG_EMBEDDING_REFUSED', side_effects: 'unknown' } },
{ ...receipt, error: { code: 'other', side_effects: 'none' } },
{ status: true, known_type: true, error: receipt.error },
{ status: true, known_type: true, file_id: 'another-file' },
])('retains uncertainty for a legacy, contradictory or malformed response: %j', async (data) => {
const { upload, params, onUploadStart } = setup(data);
const error = await upload(params).catch((error: Error) => error);
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
expect(isDefinitiveFileStorageFailure(error)).toBe(false);
expect(onUploadStart.mock.calls).toEqual([[]]);
});
it.each([
{ code: 'ETIMEDOUT' },
{ code: 'ECONNRESET' },
{ code: 'ERR_CANCELED' },
{ response: { status: 400 } },
{ response: { status: 422 } },
{ response: { status: 500 } },
{ code: 'RAG_EMBEDDING_REFUSED' },
])('never infers remote absence from a rejected request: %j', async (details) => {
const { upload, params, post, onUploadStart } = setup(undefined);
const secret = 'private-document SQL bearer-test-token';
post.mockRejectedValueOnce(Object.assign(new Error(secret), details));
const error = await upload(params).catch((error: Error) => error);
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
expect(error).not.toHaveProperty('cause');
expect(String(error)).not.toContain(secret);
expect(isDefinitiveFileStorageFailure(error)).toBe(false);
expect(onUploadStart.mock.calls).toEqual([[]]);
});
it('does not trust a no-side-effect body outside a successful HTTP response', async () => {
const { upload, params } = setup(receipt, 500);
await expect(upload(params)).rejects.toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
});
it.each([undefined, NaN, '200', 199, 300, 400, 422, 500])(
'rejects a non-success or malformed HTTP status without releasing the claim: %j',
async (status) => {
const { upload, params, post } = setup(receipt);
post.mockResolvedValueOnce({ status, data: receipt });
await expect(upload(params)).rejects.toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
},
);
it('sanitizes credential setup failures before dispatch', async () => {
const { upload, params, generateToken, post, onUploadStart } = setup(receipt);
generateToken.mockImplementationOnce(() => {
throw new Error('private-signing-key');
});
const error = await upload(params).catch((error: Error) => error);
expect(error).toMatchObject({ code: 'RAG_EMBEDDING_OUTCOME_UNKNOWN' });
expect(String(error)).not.toContain('private-signing-key');
expect(error).not.toHaveProperty('cause');
expect(post).not.toHaveBeenCalled();
expect(onUploadStart).not.toHaveBeenCalled();
});

View file

@ -0,0 +1,117 @@
import FormData from 'form-data';
import { createReadStream } from 'node:fs';
import { logger } from '@librechat/data-schemas';
import { FileSources } from 'librechat-data-provider';
import type { AxiosInstance } from 'axios';
import type { ReadStream } from 'node:fs';
import type { FileStorageUploadObserver } from '../operations';
import { dispatchFileStorageOperation } from '../dispatch';
import { getSafeErrorMetadata } from '~/utils/errors';
import { RagEmbeddingError } from './errors';
type VectorUpload = {
req: { user: { id: string } };
file: { path: string; originalname: string; mimetype?: string; size?: number };
file_id: string;
entity_id?: string;
storageMetadata?: Record<string, unknown>;
signal?: AbortSignal;
onUploadStart?: FileStorageUploadObserver;
};
function isRecord(value: unknown): value is Record<string, unknown> {
return value != null && typeof value === 'object' && !Array.isArray(value);
}
/** Legacy status/known_type flags do not prove rollback completed or that inserts never began. */
function confirmEmbedding(data: unknown, fileId: string): void {
if (isRecord(data)) {
if (
data.status === false &&
data.file_id === fileId &&
isRecord(data.error) &&
data.error.code === 'RAG_EMBEDDING_REFUSED' &&
data.error.side_effects === 'none'
) {
throw new RagEmbeddingError('RAG_EMBEDDING_REFUSED');
}
if (
data.status === true &&
data.known_type === true &&
!('error' in data) &&
(data.file_id === undefined || data.file_id === fileId)
) {
return;
}
}
throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
}
/** HTTP clients and credentials are supplied by the host, not application singletons. */
export function createVectorFileUploader({
baseUrl,
httpClient,
generateToken,
}: {
baseUrl?: string;
httpClient: Pick<AxiosInstance, 'post'>;
generateToken: (userId: string) => string;
}): (params: VectorUpload) => Promise<{
bytes?: number;
filename: string;
filepath: FileSources;
embedded: boolean;
}> {
return async ({ req, file, file_id, entity_id, storageMetadata, signal, onUploadStart }) => {
let stream: ReadStream | undefined;
try {
if (!baseUrl) throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
const token = generateToken(req.user.id);
const body = new FormData();
body.append('file_id', file_id);
stream = createReadStream(file.path);
body.append('file', stream);
if (entity_id) body.append('entity_id', entity_id);
if (storageMetadata) body.append('storage_metadata', JSON.stringify(storageMetadata));
return await dispatchFileStorageOperation(onUploadStart, async () => {
try {
const response = await httpClient.post<unknown>(`${baseUrl}/embed`, body, {
...(signal ? { signal } : {}),
headers: {
Authorization: `Bearer ${token}`,
accept: 'application/json',
...body.getHeaders(),
},
});
if (
!Number.isInteger(response.status) ||
response.status < 200 ||
response.status >= 300
) {
throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
}
confirmEmbedding(response.data, file_id);
return {
bytes: file.size,
filename: file.originalname,
filepath: FileSources.vectordb,
embedded: true,
};
} catch (error) {
logger.error('[uploadVectors] Embedding was not confirmed', getSafeErrorMetadata(error));
// Upstream errors can contain document text, credentials or SQL, including in causes.
throw error instanceof RagEmbeddingError
? error
: new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
}
});
} catch (error) {
if (error instanceof RagEmbeddingError) throw error;
logger.error('[uploadVectors] Embedding setup failed', getSafeErrorMetadata(error));
throw new RagEmbeddingError('RAG_EMBEDDING_OUTCOME_UNKNOWN');
} finally {
stream?.destroy();
}
};
}

View file

@ -181,7 +181,7 @@ describe('resource lease Mongo adapter', () => {
jest.useRealTimers();
}
expect(await harness.read(lock.key)).toBeNull();
expect(await methods.acquireResourceLock(lock)).toBe(true);
expect(await methods.acquireResourceLock({ ...lock, readyWaitMs: 2_000 })).toBe(true);
expect(build).toHaveBeenCalledTimes(3);
});