mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
* 💰 fix: Record Token Usage for Stopped Agent Turns The agents Stop route only signals the abort; the run's cleanup skipped recordCollectedUsage on an aborted signal, trusting an abort middleware that is mounted only on the assistants abort routes. Stopped and error-aborted agent turns therefore wrote no transactions and never debited the balance. The run now records its collected usage on every exit, labelling a stopped turn's transactions with context 'abort'. The owner of the run charges, so a cross-replica Stop is still billed exactly once. * 💰 fix: Make the Run the Single Charger for a Stopped Turn The legacy assistants-mounted abort route still billed a non-assistants job — provider usage via spendCollectedUsage, or a text-count fallback — on top of what the run records on exit, and its array clear ran only after its own awaited write, so the two could race even on one replica. The route now only stops and persists; the run records its usage once, labelled through resolveRunUsageContext in packages/api. * 💰 fix: Keep the Text-Count Fallback From Re-Billing Recorded Usage BaseClient falls back to text-count billing whenever the recorded usage has no positive output count, so a stopped call that reported input tokens and no output was debited by recordCollectedUsage and then charged its estimated prompt again. AgentClient.recordTokenUsage now returns early once provider usage was recorded (hasRecordedProviderUsage in packages/api); an all-zero report still falls through to the estimate. * 💰 refactor: Lift Fallback Token Billing Into packages/api recordFallbackTokenUsage owns the whole fallback operation — the recorded-usage guard, the estimate spend, the reasoning row and the error boundary — taking spendTokens as an injected dependency. AgentClient.recordTokenUsage only builds the transaction metadata from the client and calls it. * 💰 fix: Label Fallback Billing From the Run's Stop State BaseClient invokes the estimate fallback without a context, so the sole row for an initial turn stopped before any provider usage was labelled 'message'. recordTokenUsage now defaults the context from the run's own abort signal. * 💰 refactor: Default the Fallback Billing Label in packages/api recordFallbackTokenUsage takes the run's raw abort state and derives the transaction label itself; AgentClient.recordTokenUsage passes the signal through and no longer chooses the context. * 💰 fix: Guard Fallback Billing on the Collected Primary Entries The stream aggregate recordCollectedUsage returns takes its input from the first primary call only, so a later cancelled call that reported input alone was billed yet invisible to the fallback guard, which then charged the estimated prompt again. recordFallbackTokenUsage now also consults the collected entries themselves (hasRecordedPrimaryUsage), ignoring summarization, subagent and sequential usage; AgentClient passes its collected entries through.
275 lines
8.7 KiB
JavaScript
275 lines
8.7 KiB
JavaScript
/**
|
|
* Tests for abortMiddleware.
|
|
*
|
|
* The run that produced a stopped response records its own usage on exit
|
|
* (AgentClient labels it 'abort'), so this route must not bill: it only stops
|
|
* the job and persists the partial response.
|
|
*/
|
|
|
|
const mockSpendTokens = jest.fn().mockResolvedValue();
|
|
const mockSpendStructuredTokens = jest.fn().mockResolvedValue();
|
|
const mockRecordCollectedUsage = jest
|
|
.fn()
|
|
.mockResolvedValue({ input_tokens: 100, output_tokens: 50 });
|
|
|
|
const mockGetMultiplier = jest.fn().mockReturnValue(1);
|
|
const mockGetCacheMultiplier = jest.fn().mockReturnValue(null);
|
|
const mockGetTransactionsConfig = jest.fn().mockReturnValue({ enabled: false });
|
|
|
|
jest.mock('@librechat/data-schemas', () => ({
|
|
logger: {
|
|
debug: jest.fn(),
|
|
error: jest.fn(),
|
|
warn: jest.fn(),
|
|
info: jest.fn(),
|
|
},
|
|
}));
|
|
|
|
jest.mock('@librechat/api', () => ({
|
|
/** Real implementation: these tests exist to verify abort classification
|
|
* itself, so mocking it would assert the mock rather than the behavior. */
|
|
isAbortError: jest.requireActual('@librechat/api').isAbortError,
|
|
countTokens: jest.fn().mockResolvedValue(100),
|
|
isEnabled: jest.fn().mockReturnValue(false),
|
|
sendEvent: jest.fn(),
|
|
GenerationJobManager: {
|
|
abortJob: jest.fn(),
|
|
},
|
|
recordCollectedUsage: mockRecordCollectedUsage,
|
|
getTransactionsConfig: (...args) => mockGetTransactionsConfig(...args),
|
|
sanitizeMessageForTransmit: jest.fn((msg) => msg),
|
|
buildAbortedResponseMetadata: jest.fn().mockReturnValue(null),
|
|
}));
|
|
|
|
jest.mock('librechat-data-provider', () => ({
|
|
/** Keep the module real: `@librechat/api` is partially un-mocked above and
|
|
* reads constants (`CacheKeys`, ...) from it at import time. */
|
|
...jest.requireActual('librechat-data-provider'),
|
|
isAssistantsEndpoint: jest.fn().mockReturnValue(false),
|
|
ErrorTypes: { INVALID_REQUEST: 'INVALID_REQUEST', NO_SYSTEM_MESSAGES: 'NO_SYSTEM_MESSAGES' },
|
|
}));
|
|
|
|
jest.mock('~/app/clients/prompts', () => ({
|
|
truncateText: jest.fn((text) => text),
|
|
smartTruncateText: jest.fn((text) => text),
|
|
}));
|
|
|
|
jest.mock('~/cache/clearPendingReq', () => jest.fn().mockResolvedValue());
|
|
|
|
jest.mock('~/server/middleware/error', () => ({
|
|
sendError: jest.fn(),
|
|
}));
|
|
|
|
const mockUpdateBalance = jest.fn().mockResolvedValue({});
|
|
const mockBulkInsertTransactions = jest.fn().mockResolvedValue(undefined);
|
|
jest.mock('~/models', () => ({
|
|
saveMessage: jest.fn().mockResolvedValue(),
|
|
getConvo: jest.fn().mockResolvedValue({ title: 'Test Chat' }),
|
|
updateBalance: mockUpdateBalance,
|
|
bulkInsertTransactions: mockBulkInsertTransactions,
|
|
spendTokens: (...args) => mockSpendTokens(...args),
|
|
spendStructuredTokens: (...args) => mockSpendStructuredTokens(...args),
|
|
getMultiplier: mockGetMultiplier,
|
|
getCacheMultiplier: mockGetCacheMultiplier,
|
|
}));
|
|
|
|
jest.mock('./abortRun', () => ({
|
|
abortRun: jest.fn(),
|
|
}));
|
|
|
|
const { logger } = require('@librechat/data-schemas');
|
|
const { sendError } = require('~/server/middleware/error');
|
|
const { GenerationJobManager } = require('@librechat/api');
|
|
const db = require('~/models');
|
|
const { handleAbort, handleAbortError } = require('./abortMiddleware');
|
|
|
|
const buildAbortRequest = () => ({
|
|
body: {
|
|
model: 'gpt-4',
|
|
},
|
|
user: {
|
|
id: 'user-123',
|
|
},
|
|
});
|
|
|
|
describe('abortMiddleware - handleAbortError', () => {
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
});
|
|
|
|
it.each([
|
|
[
|
|
'native DOMException AbortError',
|
|
new DOMException('The operation was aborted', 'AbortError'),
|
|
'AbortError',
|
|
],
|
|
[
|
|
'wrapped AbortError message',
|
|
new Error('SSE stream disconnected: AbortError: The operation was aborted'),
|
|
'Error',
|
|
],
|
|
[
|
|
'cause-nested AbortError',
|
|
new Error('Request failed', {
|
|
cause: new DOMException('The operation was aborted', 'AbortError'),
|
|
}),
|
|
'Error',
|
|
],
|
|
])('logs a %s as a debug event instead of an error', async (_label, error, name) => {
|
|
await handleAbortError({}, buildAbortRequest(), error, {
|
|
sender: 'AI',
|
|
conversationId: 'convo-123',
|
|
messageId: 'message-123',
|
|
parentMessageId: 'parent-123',
|
|
userMessageId: 'user-message-123',
|
|
});
|
|
|
|
expect(logger.error).not.toHaveBeenCalled();
|
|
expect(logger.debug).toHaveBeenCalledWith('[handleAbortError] AI response aborted by user', {
|
|
conversationId: 'convo-123',
|
|
code: error.code,
|
|
name,
|
|
message: error.message,
|
|
});
|
|
expect(sendError).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('keeps unexpected generation errors classified as errors', async () => {
|
|
const error = new Error('Provider failed');
|
|
|
|
await handleAbortError({}, buildAbortRequest(), error, {
|
|
sender: 'AI',
|
|
conversationId: 'convo-123',
|
|
messageId: 'message-123',
|
|
parentMessageId: 'parent-123',
|
|
userMessageId: 'user-message-123',
|
|
});
|
|
|
|
expect(logger.error).toHaveBeenCalledWith(
|
|
'[handleAbortError] AI response error; aborting request:',
|
|
error,
|
|
);
|
|
expect(logger.debug).not.toHaveBeenCalled();
|
|
expect(sendError).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
/**
|
|
* The transactions config is resolved from the request's app config and must reach
|
|
* every write path in this file. `createTransaction` reads `transactions` from the
|
|
* caller-supplied data, so an omitted value is indistinguishable from enabled and
|
|
* the write proceeds even when `transactions.enabled` is false.
|
|
*/
|
|
describe('abortMiddleware - handleAbort billing', () => {
|
|
const buildJobData = () => ({
|
|
model: 'gpt-4',
|
|
responseMessageId: 'msg-123',
|
|
conversationId: 'convo-123',
|
|
endpoint: 'agents',
|
|
sender: 'AI',
|
|
promptTokens: 25,
|
|
userMessage: {
|
|
messageId: 'user-msg-123',
|
|
parentMessageId: 'parent-123',
|
|
conversationId: 'convo-123',
|
|
text: 'hello',
|
|
},
|
|
});
|
|
|
|
const buildReq = () => ({
|
|
body: { abortKey: 'convo-123:1', endpoint: 'agents' },
|
|
user: { id: 'user-123', email: 'user@example.com' },
|
|
config: { transactions: { enabled: false } },
|
|
});
|
|
|
|
const buildRes = () => ({
|
|
headersSent: false,
|
|
setHeader: jest.fn(),
|
|
send: jest.fn(),
|
|
});
|
|
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
mockGetTransactionsConfig.mockReturnValue({ enabled: false });
|
|
mockRecordCollectedUsage.mockResolvedValue({ input_tokens: 100, output_tokens: 50 });
|
|
db.getConvo.mockResolvedValue({ title: 'Test Chat' });
|
|
});
|
|
|
|
it('leaves billing to the run even when the stopped job collected usage', async () => {
|
|
const collectedUsage = [{ input_tokens: 100, output_tokens: 50, model: 'gpt-4' }];
|
|
GenerationJobManager.abortJob.mockResolvedValue({
|
|
success: true,
|
|
jobData: buildJobData(),
|
|
content: [],
|
|
text: 'partial',
|
|
collectedUsage,
|
|
});
|
|
|
|
await handleAbort()(buildReq(), buildRes());
|
|
|
|
expect(logger.error).not.toHaveBeenCalled();
|
|
expect(mockRecordCollectedUsage).not.toHaveBeenCalled();
|
|
expect(mockSpendTokens).not.toHaveBeenCalled();
|
|
expect(mockSpendStructuredTokens).not.toHaveBeenCalled();
|
|
expect(collectedUsage).toHaveLength(1);
|
|
expect(db.saveMessage).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('carries the context meta the run published onto the job into the stopped response', async () => {
|
|
const contextMeta = {
|
|
calibrationRatio: 1.2,
|
|
encoding: 'claude',
|
|
fading: { v: 1, budgetTokens: 50_000, masked: true },
|
|
};
|
|
GenerationJobManager.abortJob.mockResolvedValue({
|
|
success: true,
|
|
jobData: { ...buildJobData(), contextMeta },
|
|
content: [],
|
|
text: 'partial',
|
|
collectedUsage: [],
|
|
});
|
|
const res = buildRes();
|
|
|
|
await handleAbort()(buildReq(), res);
|
|
|
|
expect(db.saveMessage).toHaveBeenCalledWith(
|
|
expect.any(Object),
|
|
expect.objectContaining({ messageId: 'msg-123', contextMeta }),
|
|
expect.any(Object),
|
|
);
|
|
const finalEvent = JSON.parse(res.send.mock.calls[0][0]);
|
|
expect(finalEvent.responseMessage.contextMeta).toEqual(contextMeta);
|
|
});
|
|
|
|
it('unsets context meta on a stopped response when the job carries none', async () => {
|
|
GenerationJobManager.abortJob.mockResolvedValue({
|
|
success: true,
|
|
jobData: buildJobData(),
|
|
content: [],
|
|
text: 'partial',
|
|
collectedUsage: [],
|
|
});
|
|
|
|
await handleAbort()(buildReq(), buildRes());
|
|
|
|
const [, savedMessage] = db.saveMessage.mock.calls[0];
|
|
expect(savedMessage.contextMeta).toBeNull();
|
|
});
|
|
|
|
it('does not bill a stopped response by token count either', async () => {
|
|
GenerationJobManager.abortJob.mockResolvedValue({
|
|
success: true,
|
|
jobData: buildJobData(),
|
|
content: [],
|
|
text: 'partial',
|
|
collectedUsage: [],
|
|
});
|
|
|
|
await handleAbort()(buildReq(), buildRes());
|
|
|
|
expect(logger.error).not.toHaveBeenCalled();
|
|
expect(mockRecordCollectedUsage).not.toHaveBeenCalled();
|
|
expect(mockSpendTokens).not.toHaveBeenCalled();
|
|
expect(db.saveMessage).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|