mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-17 21:43:06 +00:00
🔀 feat: Show the Conversation Pull Request in the Sidebar List (#16815)
* feat: Show the Conversation Pull Request in the Sidebar List Add a batch lookup route and a sidebar row mark: the state icon with the CI dot, opening the same card beside the row on hover. The row keeps the conversation's own title. * fix: Address Review Findings on the Sidebar Pull Request Batch and Row Mark Cap the whole batch with a configurable deadline, make the row mark a keyboard-reachable button that opens the card on focus, refresh rows when the window regains focus through the shared batch, and show a retryable failure state instead of treating a failed first lookup as no pull request. * fix: Send Sidebar Pull Request Batches One at a Time Queue every batch request behind the one before it, so an oversized list or an overlapping flush cannot multiply the server's configured lookup concurrency, and bound how long one request may hold the queue so a request that never answers cannot stop the rest. * fix: Keep the Batch Request Timeout Above the Server Deadline Default the client's request timeout to just past the longest batchTimeoutSeconds a deployment may configure, so a queued request never starts while the previous request's server-side lookups are still running. * fix: Keep Batch Lookups Inside the Concurrency Limit and Per-Entry Failures Per Entry Share one lookup limiter across every batch request so lookups that outlive a request's deadline keep their slot, answer a missing token only for the conversations that needed it, and point the row at its pull request description only while that text is in the page. * fix: Bound Batch Reads, Scope Concurrency Per Credential, and Gate the Sidebar on the Batch Version Start the batch deadline when the request arrives and bound the config and lane reads, count lookup slots per credential, advertise a batch route version and ask only a server that has it, take no row space for an absent mark, show a failed refresh over a cached empty answer, keep a malformed response from stalling the request queue, and repair the lane recording tests that still used a mock removed on dev. * fix: Move Capability Resolution Into the Package and Harden the Batch Route and Queue Resolve the pull request startup capabilities in packages/api so the route stays wiring only, bound the config and lane reads by the configured deadline, merge identical lanes before taking limiter slots, fall back to the single route when a replica has no batch route, and dispose the batch queue when the signed-in user changes. * fix: Abort the Batch Fallback With Its Request So the Queue Never Overlaps It Give the batch fetcher the request's abort signal and pass it to every single-route call, abort a request that times out or is disposed, and stop the fallback workers from starting more calls once it has. * fix: Forward the Abort Signal to the Batch POST Pass the request's abort signal through the batch fetcher, the data-provider call and Axios, so a batch request that times out or is disposed is cancelled instead of continuing beside the next one, and settle a request promptly when it is aborted. * fix: Close the Pull Request Lookup Follow-Ups and Cancel Abandoned Sidebar Rows
This commit is contained in:
parent
cdf2e94dcf
commit
caeb786e97
47 changed files with 4260 additions and 128 deletions
758
client/src/data-provider/PullRequest/batch.test.ts
Normal file
758
client/src/data-provider/PullRequest/batch.test.ts
Normal file
|
|
@ -0,0 +1,758 @@
|
|||
import type { TConversationPullRequest } from 'librechat-data-provider';
|
||||
import { PullRequestBatchError, createBatchFetcher, createPullRequestBatcher } from './batch';
|
||||
|
||||
const pr: TConversationPullRequest = {
|
||||
number: 1,
|
||||
title: 't',
|
||||
url: 'https://github.com/o/r/pull/1',
|
||||
additions: 1,
|
||||
deletions: 0,
|
||||
state: 'open',
|
||||
isDraft: false,
|
||||
mergeable: 'clean',
|
||||
checks: 'passing',
|
||||
};
|
||||
|
||||
describe('createPullRequestBatcher', () => {
|
||||
beforeEach(() => jest.useFakeTimers());
|
||||
afterEach(() => jest.useRealTimers());
|
||||
|
||||
it('asks once for every conversation requested in the same moment', async () => {
|
||||
const fetchMany = jest.fn().mockResolvedValue({
|
||||
results: [
|
||||
{ conversationId: 'a', pullRequest: pr },
|
||||
{ conversationId: 'b', pullRequest: null },
|
||||
],
|
||||
});
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const first = load('a');
|
||||
const second = load('b');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMany).toHaveBeenCalledWith(['a', 'b'], expect.any(AbortSignal));
|
||||
await expect(first).resolves.toEqual({ pullRequest: pr });
|
||||
await expect(second).resolves.toEqual({ pullRequest: null });
|
||||
});
|
||||
|
||||
it('shares one entry between callers asking for the same conversation', async () => {
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockResolvedValue({ results: [{ conversationId: 'a', pullRequest: pr }] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const calls = [load('a'), load('a')];
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledWith(['a'], expect.any(AbortSignal));
|
||||
await expect(Promise.all(calls)).resolves.toEqual([{ pullRequest: pr }, { pullRequest: pr }]);
|
||||
});
|
||||
|
||||
it('answers no pull request for a conversation the server left out', async () => {
|
||||
const fetchMany = jest.fn().mockResolvedValue({ results: [] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const answer = load('a');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(answer).resolves.toEqual({ pullRequest: null });
|
||||
});
|
||||
|
||||
it('splits a crowd into requests the server accepts', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => ({
|
||||
results: ids.map((conversationId) => ({ conversationId, pullRequest: null })),
|
||||
}));
|
||||
const { load } = createPullRequestBatcher({ fetchMany, maxBatch: 2 });
|
||||
const all = ['a', 'b', 'c', 'd', 'e'].map((id) => load(id));
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await Promise.all(all);
|
||||
expect(fetchMany.mock.calls.map(([ids]) => ids)).toEqual([['a', 'b'], ['c', 'd'], ['e']]);
|
||||
});
|
||||
|
||||
describe('requests in flight', () => {
|
||||
/** A server that answers only when told to, so overlap is visible. */
|
||||
const slowServer = () => {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
const fetchMany = jest.fn(
|
||||
(ids: string[]) =>
|
||||
new Promise<{ results: Array<{ conversationId: string; pullRequest: null }> }>(
|
||||
(resolve) => {
|
||||
active += 1;
|
||||
peak = Math.max(peak, active);
|
||||
releases.push(() => {
|
||||
active -= 1;
|
||||
resolve({
|
||||
results: ids.map((conversationId) => ({ conversationId, pullRequest: null })),
|
||||
});
|
||||
});
|
||||
},
|
||||
),
|
||||
);
|
||||
return { fetchMany, releases, peak: () => peak };
|
||||
};
|
||||
|
||||
it('sends the chunks of a crowd one at a time, so the server limit is not multiplied', async () => {
|
||||
const { fetchMany, releases, peak } = slowServer();
|
||||
const { load } = createPullRequestBatcher({ fetchMany, maxBatch: 2 });
|
||||
const all = ['a', 'b', 'c', 'd', 'e'].map((id) => load(id));
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
releases.shift()?.();
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(2);
|
||||
releases.shift()?.();
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(3);
|
||||
releases.shift()?.();
|
||||
await Promise.all(all);
|
||||
expect(peak()).toBe(1);
|
||||
expect(fetchMany.mock.calls.map(([ids]) => ids)).toEqual([['a', 'b'], ['c', 'd'], ['e']]);
|
||||
});
|
||||
|
||||
it('does not start a later flush while an earlier request is still out', async () => {
|
||||
const { fetchMany, releases, peak } = slowServer();
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const first = load('a');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
const second = load('b');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
releases.shift()?.();
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(2);
|
||||
releases.shift()?.();
|
||||
await Promise.all([first, second]);
|
||||
expect(peak()).toBe(1);
|
||||
});
|
||||
|
||||
it('moves on when a request never answers, failing only the callers it carried', async () => {
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockImplementationOnce(() => new Promise(() => undefined))
|
||||
.mockResolvedValue({ results: [{ conversationId: 'b', pullRequest: null }] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany, requestTimeoutMs: 1000 });
|
||||
const hung = load('a').catch((error) => error);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
const later = load('b');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
const error = await hung;
|
||||
expect(error).toBeInstanceOf(PullRequestBatchError);
|
||||
expect(error.code).toBe('TIMEOUT');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(later).resolves.toEqual({ pullRequest: null });
|
||||
});
|
||||
|
||||
it('waits longer than the longest server deadline a deployment can configure by default', async () => {
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
setTimeout(
|
||||
() => resolve({ results: [{ conversationId: 'a', pullRequest: null }] }),
|
||||
125_000,
|
||||
),
|
||||
),
|
||||
)
|
||||
.mockResolvedValue({ results: [{ conversationId: 'b', pullRequest: null }] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const slow = load('a');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
const queued = load('b');
|
||||
await jest.advanceTimersByTimeAsync(124_000);
|
||||
/** A 120 s server batch has not finished at 124 s of waiting, and the next must not have started. */
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(2_000);
|
||||
await expect(slow).resolves.toEqual({ pullRequest: null });
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(queued).resolves.toEqual({ pullRequest: null });
|
||||
expect(fetchMany).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not time out a request that answers in time', async () => {
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockResolvedValue({ results: [{ conversationId: 'a', pullRequest: null }] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany, requestTimeoutMs: 1000 });
|
||||
const answer = load('a');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(answer).resolves.toEqual({ pullRequest: null });
|
||||
await jest.advanceTimersByTimeAsync(5000);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no body', undefined],
|
||||
['a body without results', {}],
|
||||
['results that are not a list', { results: 'nope' }],
|
||||
['a result that is not an entry', { results: [null] }],
|
||||
])(
|
||||
'fails its callers, and not the queue, when the server answers with %s',
|
||||
async (_label, body) => {
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(body)
|
||||
.mockResolvedValue({ results: [{ conversationId: 'b', pullRequest: null }] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany, maxBatch: 1 });
|
||||
const bad = load('a').then(
|
||||
() => 'resolved',
|
||||
() => 'rejected',
|
||||
);
|
||||
const good = load('b');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(bad).resolves.toBe('rejected');
|
||||
await expect(good).resolves.toEqual({ pullRequest: null });
|
||||
expect(fetchMany).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps sending after a request fails', async () => {
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('503'))
|
||||
.mockResolvedValue({ results: [{ conversationId: 'c', pullRequest: null }] });
|
||||
const { load } = createPullRequestBatcher({ fetchMany, maxBatch: 1 });
|
||||
const failed = load('a').catch((error) => error.message);
|
||||
const fine = load('c');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(failed).resolves.toBe('503');
|
||||
await expect(fine).resolves.toEqual({ pullRequest: null });
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects only the conversation whose entry failed, carrying its code', async () => {
|
||||
const fetchMany = jest.fn().mockResolvedValue({
|
||||
results: [
|
||||
{ conversationId: 'a', error: { code: 'RATE_LIMITED' } },
|
||||
{ conversationId: 'b', pullRequest: pr },
|
||||
],
|
||||
});
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const failed = load('a');
|
||||
const fine = load('b');
|
||||
const outcome = failed.catch((error) => error);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
const error = await outcome;
|
||||
expect(error).toBeInstanceOf(PullRequestBatchError);
|
||||
expect(error.code).toBe('RATE_LIMITED');
|
||||
await expect(fine).resolves.toEqual({ pullRequest: pr });
|
||||
});
|
||||
|
||||
it('rejects every caller of a request that fails outright', async () => {
|
||||
const fetchMany = jest.fn().mockRejectedValue(new Error('503'));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const outcomes = [load('a'), load('b')].map((promise) =>
|
||||
promise.catch((error) => error.message),
|
||||
);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(Promise.all(outcomes)).resolves.toEqual(['503', '503']);
|
||||
});
|
||||
|
||||
it('starts a new request for a conversation asked for after the last went out', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => ({
|
||||
results: ids.map((conversationId) => ({ conversationId, pullRequest: null })),
|
||||
}));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const first = load('a');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await first;
|
||||
const second = load('b');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await second;
|
||||
expect(fetchMany.mock.calls.map(([ids]) => ids)).toEqual([['a'], ['b']]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createPullRequestBatcher dispose', () => {
|
||||
beforeEach(() => jest.useFakeTimers());
|
||||
afterEach(() => jest.useRealTimers());
|
||||
|
||||
it('rejects ids that have not gone out, and sends nothing for them', async () => {
|
||||
const fetchMany = jest.fn();
|
||||
const batcher = createPullRequestBatcher({ fetchMany });
|
||||
const waiting = batcher.load('a').catch((error) => error);
|
||||
batcher.dispose();
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
const error = await waiting;
|
||||
expect(error).toBeInstanceOf(PullRequestBatchError);
|
||||
expect(error.code).toBe('DISPOSED');
|
||||
expect(fetchMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not send chunks that were still queued behind a request in flight', async () => {
|
||||
const releases: Array<() => void> = [];
|
||||
const fetchMany = jest.fn(
|
||||
(ids: string[]) =>
|
||||
new Promise<{ results: Array<{ conversationId: string; pullRequest: null }> }>((resolve) =>
|
||||
releases.push(() =>
|
||||
resolve({
|
||||
results: ids.map((conversationId) => ({ conversationId, pullRequest: null })),
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
const batcher = createPullRequestBatcher({ fetchMany, maxBatch: 1 });
|
||||
const outcomes = ['a', 'b', 'c'].map((id) =>
|
||||
batcher.load(id).then(
|
||||
() => 'sent',
|
||||
(error) => error.code,
|
||||
),
|
||||
);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
batcher.dispose();
|
||||
releases.shift()?.();
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
/** The request on the wire is aborted with the batcher, and the queued chunks never go out. */
|
||||
expect(await Promise.all(outcomes)).toEqual(['ABORTED', 'DISPOSED', 'DISPOSED']);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('refuses new work once disposed', async () => {
|
||||
const batcher = createPullRequestBatcher({ fetchMany: jest.fn() });
|
||||
batcher.dispose();
|
||||
await expect(batcher.load('a')).rejects.toMatchObject({ code: 'DISPOSED' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('createBatchFetcher', () => {
|
||||
const entry = (conversationId: string) => ({ conversationId, pullRequest: null });
|
||||
const notFound = () => Object.assign(new Error('Not Found'), { response: { status: 404 } });
|
||||
|
||||
it('uses the batch route when the server has it', async () => {
|
||||
const fetchMany = jest.fn().mockResolvedValue({ results: [entry('a')] });
|
||||
const fetchOne = jest.fn();
|
||||
await expect(
|
||||
createBatchFetcher({ fetchMany, fetchOne })(['a'], new AbortController().signal),
|
||||
).resolves.toEqual({
|
||||
results: [entry('a')],
|
||||
});
|
||||
expect(fetchOne).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('asks the single route for each conversation when the replica has no batch route', async () => {
|
||||
const fetchMany = jest.fn().mockRejectedValue(notFound());
|
||||
const fetchOne = jest.fn(async (id: string) => ({ pullRequest: id === 'a' ? pr : null }));
|
||||
const results = await createBatchFetcher({ fetchMany, fetchOne })(
|
||||
['a', 'b'],
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(results).toEqual({
|
||||
results: [
|
||||
{ conversationId: 'a', pullRequest: pr },
|
||||
{ conversationId: 'b', pullRequest: null },
|
||||
],
|
||||
});
|
||||
expect(fetchOne.mock.calls.map(([id]) => id)).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it("keeps one conversation's failure to itself and reports its code", async () => {
|
||||
const fetchMany = jest.fn().mockRejectedValue(notFound());
|
||||
const fetchOne = jest.fn(async (id: string) => {
|
||||
if (id === 'a') {
|
||||
throw Object.assign(new Error('x'), {
|
||||
response: { status: 503, data: { code: 'RATE_LIMITED' } },
|
||||
});
|
||||
}
|
||||
return { pullRequest: pr };
|
||||
});
|
||||
const { results } = await createBatchFetcher({ fetchMany, fetchOne })(
|
||||
['a', 'b'],
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(results).toEqual([
|
||||
{ conversationId: 'a', error: { code: 'RATE_LIMITED' } },
|
||||
{ conversationId: 'b', pullRequest: pr },
|
||||
]);
|
||||
});
|
||||
|
||||
it('answers a failure with no code as an upstream error, never with its text', async () => {
|
||||
const fetchMany = jest.fn().mockRejectedValue(notFound());
|
||||
const fetchOne = jest.fn().mockRejectedValue(new Error('secret host name'));
|
||||
const { results } = await createBatchFetcher({ fetchMany, fetchOne })(
|
||||
['a'],
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(results).toEqual([{ conversationId: 'a', error: { code: 'UPSTREAM_ERROR' } }]);
|
||||
expect(JSON.stringify(results)).not.toContain('secret');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a server error', { response: { status: 503 } }],
|
||||
['a rate limit', { response: { status: 429 } }],
|
||||
['an expired session', { response: { status: 401 } }],
|
||||
['a network failure', new Error('Network Error')],
|
||||
])(
|
||||
'does not fall back on %s, since the route exists and really failed',
|
||||
async (_label, failure) => {
|
||||
const error = failure instanceof Error ? failure : Object.assign(new Error('x'), failure);
|
||||
const fetchMany = jest.fn().mockRejectedValue(error);
|
||||
const fetchOne = jest.fn();
|
||||
await expect(
|
||||
createBatchFetcher({ fetchMany, fetchOne })(['a'], new AbortController().signal),
|
||||
).rejects.toBe(error);
|
||||
expect(fetchOne).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
const measuredFallback = async (concurrency?: () => number | undefined) => {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const fetchMany = jest.fn().mockRejectedValue(notFound());
|
||||
const fetchOne = jest.fn(async () => {
|
||||
active += 1;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
active -= 1;
|
||||
return { pullRequest: null };
|
||||
});
|
||||
jest.useRealTimers();
|
||||
const ids = Array.from({ length: 12 }, (_, i) => `c${i}`);
|
||||
const { results } = await createBatchFetcher({ fetchMany, fetchOne, concurrency })(
|
||||
ids,
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(results.map((r) => r.conversationId)).toEqual(ids);
|
||||
return peak;
|
||||
};
|
||||
|
||||
it('runs the single-route calls one at a time when the server never said how many it allows', async () => {
|
||||
expect(await measuredFallback()).toBe(1);
|
||||
expect(await measuredFallback(() => undefined)).toBe(1);
|
||||
});
|
||||
|
||||
it('runs the single-route calls at the limit the server advertised, no more', async () => {
|
||||
expect(await measuredFallback(() => 3)).toBe(3);
|
||||
expect(await measuredFallback(() => 1)).toBe(1);
|
||||
});
|
||||
|
||||
it.each([0, -2, 1.5, Number.NaN])(
|
||||
'treats an unusable advertised limit of %s as one',
|
||||
async (limit) => {
|
||||
expect(await measuredFallback(() => limit)).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
it('reads the advertised limit when the fallback runs, not when the fetcher was made', async () => {
|
||||
let limit = 1;
|
||||
const fetchMany = jest.fn().mockRejectedValue(notFound());
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const fetchOne = jest.fn(async () => {
|
||||
active += 1;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
active -= 1;
|
||||
return { pullRequest: null };
|
||||
});
|
||||
const fetcher = createBatchFetcher({ fetchMany, fetchOne, concurrency: () => limit });
|
||||
/** The startup config arrives after the fetcher exists, as it does for the module-level one. */
|
||||
limit = 3;
|
||||
jest.useRealTimers();
|
||||
const ids = Array.from({ length: 9 }, (_, i) => `c${i}`);
|
||||
await fetcher(ids, new AbortController().signal);
|
||||
expect(peak).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the single-route fallback and the queue', () => {
|
||||
const notFound = () => Object.assign(new Error('Not Found'), { response: { status: 404 } });
|
||||
|
||||
/** A single-route call that ends only when released or aborted, like a stalled GET. */
|
||||
const stalledSingleRoute = () => {
|
||||
const calls: string[] = [];
|
||||
const aborted: string[] = [];
|
||||
const fetchOne = jest.fn(
|
||||
(id: string, signal: AbortSignal) =>
|
||||
new Promise<{ pullRequest: null }>((_, reject) => {
|
||||
calls.push(id);
|
||||
signal.addEventListener('abort', () => {
|
||||
aborted.push(id);
|
||||
reject(new Error('aborted'));
|
||||
});
|
||||
}),
|
||||
);
|
||||
return { fetchOne, calls, aborted };
|
||||
};
|
||||
|
||||
it('stops its calls when the signal aborts, and starts no more', async () => {
|
||||
const { fetchOne, calls, aborted } = stalledSingleRoute();
|
||||
const controller = new AbortController();
|
||||
const ids = Array.from({ length: 12 }, (_, i) => `c${i}`);
|
||||
const fetcher = createBatchFetcher({
|
||||
fetchMany: jest.fn().mockRejectedValue(notFound()),
|
||||
fetchOne,
|
||||
concurrency: () => 4,
|
||||
});
|
||||
const settled = fetcher(ids, controller.signal).catch((error) => error);
|
||||
await Promise.resolve();
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
expect(calls).toHaveLength(4);
|
||||
controller.abort();
|
||||
const error = await settled;
|
||||
expect(error).toBeInstanceOf(PullRequestBatchError);
|
||||
expect(error.code).toBe('ABORTED');
|
||||
expect(aborted).toHaveLength(4);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
expect(calls).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('starts nothing when the signal is already aborted', async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const fetchOne = jest.fn();
|
||||
const fetcher = createBatchFetcher({
|
||||
fetchMany: jest.fn().mockRejectedValue(notFound()),
|
||||
fetchOne,
|
||||
});
|
||||
await expect(fetcher(['a', 'b'], controller.signal)).rejects.toMatchObject({ code: 'ABORTED' });
|
||||
expect(fetchOne).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not report an aborted call as one conversation's failure", async () => {
|
||||
const controller = new AbortController();
|
||||
const fetchOne = jest.fn(
|
||||
(_id: string, signal: AbortSignal) =>
|
||||
new Promise<never>((_, reject) =>
|
||||
signal.addEventListener('abort', () => reject(new Error('x'))),
|
||||
),
|
||||
);
|
||||
const fetcher = createBatchFetcher({
|
||||
fetchMany: jest.fn().mockRejectedValue(notFound()),
|
||||
fetchOne,
|
||||
});
|
||||
const settled = fetcher(['a'], controller.signal).catch((error) => error);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
controller.abort();
|
||||
expect(await settled).toMatchObject({ code: 'ABORTED' });
|
||||
});
|
||||
|
||||
it('aborts a request that runs past its timeout, so its fallback stops before the next request starts', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { fetchOne, calls, aborted } = stalledSingleRoute();
|
||||
const fetchMany = jest
|
||||
.fn()
|
||||
.mockRejectedValueOnce(notFound())
|
||||
.mockResolvedValue({ results: [{ conversationId: 'z', pullRequest: null }] });
|
||||
const fetcher = createBatchFetcher({ fetchMany, fetchOne });
|
||||
const batcher = createPullRequestBatcher({
|
||||
fetchMany: fetcher,
|
||||
maxBatch: 1,
|
||||
requestTimeoutMs: 1000,
|
||||
});
|
||||
const first = batcher.load('a').catch((error) => error.code);
|
||||
const second = batcher.load('z');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(calls).toEqual(['a']);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
expect(await first).toBe('TIMEOUT');
|
||||
/** The stalled call was aborted before the queue handed the next request its turn. */
|
||||
expect(aborted).toEqual(['a']);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(second).resolves.toEqual({ pullRequest: null });
|
||||
expect(fetchMany).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('aborts the request on the wire when the batcher is disposed', async () => {
|
||||
const { fetchOne, calls, aborted } = stalledSingleRoute();
|
||||
const fetcher = createBatchFetcher({
|
||||
fetchMany: jest.fn().mockRejectedValue(notFound()),
|
||||
fetchOne,
|
||||
});
|
||||
const batcher = createPullRequestBatcher({ fetchMany: fetcher, delayMs: 1 });
|
||||
const outcome = batcher.load('a').catch((error) => error.code);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
expect(calls).toEqual(['a']);
|
||||
batcher.dispose();
|
||||
expect(aborted).toEqual(['a']);
|
||||
expect(await outcome).toBe('ABORTED');
|
||||
});
|
||||
|
||||
it("hands the batcher's own signal to the fetcher, live while the request runs", async () => {
|
||||
let seen: AbortSignal | undefined;
|
||||
const fetchMany = jest.fn(async (_ids: string[], signal: AbortSignal) => {
|
||||
seen = signal;
|
||||
return { results: [{ conversationId: 'a', pullRequest: null }] };
|
||||
});
|
||||
const batcher = createPullRequestBatcher({ fetchMany, delayMs: 1 });
|
||||
await batcher.load('a');
|
||||
expect(seen).toBeDefined();
|
||||
expect(seen?.aborted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the batch POST receives the request signal', () => {
|
||||
it("hands the batcher's signal to the primary batch call, and aborts it on timeout", async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const seen: AbortSignal[] = [];
|
||||
const fetchMany = jest.fn(
|
||||
(_ids: string[], signal: AbortSignal) =>
|
||||
new Promise<never>(() => {
|
||||
seen.push(signal);
|
||||
}),
|
||||
);
|
||||
const fetcher = createBatchFetcher({ fetchMany, fetchOne: jest.fn() });
|
||||
const batcher = createPullRequestBatcher({ fetchMany: fetcher, requestTimeoutMs: 1000 });
|
||||
const outcome = batcher.load('a').catch((error) => error.code);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(seen[0].aborted).toBe(false);
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
expect(await outcome).toBe('TIMEOUT');
|
||||
expect(seen[0].aborted).toBe(true);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('aborts the batch call on the wire when the batcher is disposed', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
let seen: AbortSignal | undefined;
|
||||
const fetchMany = jest.fn(
|
||||
(_ids: string[], signal: AbortSignal) =>
|
||||
new Promise<never>(() => {
|
||||
seen = signal;
|
||||
}),
|
||||
);
|
||||
const batcher = createPullRequestBatcher({
|
||||
fetchMany: createBatchFetcher({ fetchMany, fetchOne: jest.fn() }),
|
||||
});
|
||||
const outcome = batcher.load('a').catch(() => 'rejected');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
batcher.dispose();
|
||||
expect(seen?.aborted).toBe(true);
|
||||
expect(await outcome).toBe('rejected');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('withdrawing a caller that stopped waiting', () => {
|
||||
beforeEach(() => jest.useFakeTimers());
|
||||
afterEach(() => jest.useRealTimers());
|
||||
|
||||
const answer = (ids: string[]) => ({
|
||||
results: ids.map((conversationId) => ({ conversationId, pullRequest: null })),
|
||||
});
|
||||
|
||||
it('rejects the caller at once', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => answer(ids));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
const waiting = load('a', controller.signal).catch((error) => error);
|
||||
controller.abort();
|
||||
const error = await waiting;
|
||||
expect(error).toBeInstanceOf(PullRequestBatchError);
|
||||
expect(error.code).toBe('ABORTED');
|
||||
});
|
||||
|
||||
it('never asks for an id whose only caller left before the request went out', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => answer(ids));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
void load('gone', controller.signal).catch(() => undefined);
|
||||
const kept = load('kept');
|
||||
controller.abort();
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await kept;
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMany.mock.calls[0][0]).toEqual(['kept']);
|
||||
});
|
||||
|
||||
it('sends nothing at all when every caller left', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => answer(ids));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
void load('a', controller.signal).catch(() => undefined);
|
||||
void load('b', controller.signal).catch(() => undefined);
|
||||
controller.abort();
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
expect(fetchMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps an id that another caller still waits for', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => answer(ids));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
void load('shared', controller.signal).catch(() => undefined);
|
||||
const stayed = load('shared');
|
||||
controller.abort();
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
await expect(stayed).resolves.toEqual({ pullRequest: null });
|
||||
expect(fetchMany.mock.calls[0][0]).toEqual(['shared']);
|
||||
});
|
||||
|
||||
it('skips a queued request nobody waits for any more, so the next one is not held behind it', async () => {
|
||||
const releases: Array<() => void> = [];
|
||||
const fetchMany = jest.fn(
|
||||
(ids: string[]) =>
|
||||
new Promise<ReturnType<typeof answer>>((resolve) => {
|
||||
releases.push(() => resolve(answer(ids)));
|
||||
}),
|
||||
);
|
||||
const { load } = createPullRequestBatcher({ fetchMany, maxBatch: 1 });
|
||||
const first = load('first');
|
||||
const controller = new AbortController();
|
||||
void load('queued', controller.signal).catch(() => undefined);
|
||||
const last = load('last');
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(fetchMany).toHaveBeenCalledTimes(1);
|
||||
controller.abort();
|
||||
releases.shift()?.();
|
||||
await first;
|
||||
await jest.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMany.mock.calls.map(([ids]) => ids)).toEqual([['first'], ['last']]);
|
||||
releases.shift()?.();
|
||||
await last;
|
||||
});
|
||||
|
||||
it('aborts the request on the wire once nobody waits for it', async () => {
|
||||
let seen: AbortSignal | undefined;
|
||||
const fetchMany = jest.fn(
|
||||
(_ids: string[], signal: AbortSignal) =>
|
||||
new Promise<never>(() => {
|
||||
seen = signal;
|
||||
}),
|
||||
);
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
void load('a', controller.signal).catch(() => undefined);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
expect(seen?.aborted).toBe(false);
|
||||
controller.abort();
|
||||
expect(seen?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it('leaves the request on the wire running while another caller still waits for it', async () => {
|
||||
let seen: AbortSignal | undefined;
|
||||
const fetchMany = jest.fn(
|
||||
(_ids: string[], signal: AbortSignal) =>
|
||||
new Promise<never>(() => {
|
||||
seen = signal;
|
||||
}),
|
||||
);
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
void load('a', controller.signal).catch(() => undefined);
|
||||
void load('b').catch(() => undefined);
|
||||
await jest.advanceTimersByTimeAsync(50);
|
||||
controller.abort();
|
||||
expect(seen?.aborted).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects a caller whose signal had already aborted, without queueing it', async () => {
|
||||
const fetchMany = jest.fn(async (ids: string[]) => answer(ids));
|
||||
const { load } = createPullRequestBatcher({ fetchMany });
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(load('a', controller.signal)).rejects.toMatchObject({ code: 'ABORTED' });
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
expect(fetchMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
300
client/src/data-provider/PullRequest/batch.ts
Normal file
300
client/src/data-provider/PullRequest/batch.ts
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
import {
|
||||
PULL_REQUEST_BATCH_MAX,
|
||||
PULL_REQUEST_BATCH_TIMEOUT_MAX_SECONDS,
|
||||
} from 'librechat-data-provider';
|
||||
import type {
|
||||
TConversationPullRequestResponse,
|
||||
TConversationPullRequestsResponse,
|
||||
} from 'librechat-data-provider';
|
||||
|
||||
/** A failed entry keeps its stable server code, so a caller can tell a rate limit from an outage. */
|
||||
export class PullRequestBatchError extends Error {
|
||||
constructor(public readonly code: string) {
|
||||
super(`Pull request lookup failed: ${code}`);
|
||||
this.name = 'PullRequestBatchError';
|
||||
}
|
||||
}
|
||||
|
||||
type Waiter = {
|
||||
resolve: (value: TConversationPullRequestResponse) => void;
|
||||
reject: (reason: unknown) => void;
|
||||
/** Set when its caller stopped waiting; nothing is sent, or kept running, only for such waiters. */
|
||||
cancelled?: boolean;
|
||||
};
|
||||
|
||||
/** A request that has been queued or sent and not yet settled. */
|
||||
type Dispatch = { waiters: Map<string, Waiter[]>; controller?: AbortController };
|
||||
|
||||
const hasLiveWaiter = (list: Waiter[] | undefined): boolean =>
|
||||
list?.some((waiter) => !waiter.cancelled) === true;
|
||||
|
||||
type BatcherOptions = {
|
||||
/**
|
||||
* `signal` aborts when the request has run out of time or the batcher was disposed. A fetcher
|
||||
* that starts work of its own must stop it on abort: the queue moves on the moment the request
|
||||
* is settled, so work left running would overlap the next request.
|
||||
*/
|
||||
fetchMany: (
|
||||
conversationIds: string[],
|
||||
signal: AbortSignal,
|
||||
) => Promise<TConversationPullRequestsResponse>;
|
||||
/** How long to gather more ids before asking, so rows mounted together share one request. */
|
||||
delayMs?: number;
|
||||
/** Ids per request; the server refuses more than its own bound. */
|
||||
maxBatch?: number;
|
||||
/**
|
||||
* Longest a request may hold the queue. Requests go out one at a time, so one that never
|
||||
* answers would otherwise stop every later one. It defaults to just past the longest deadline a
|
||||
* deployment may configure for the server's own batch, so the server always answers first and
|
||||
* the next request never starts while the previous one's lookups are still running.
|
||||
*/
|
||||
requestTimeoutMs?: number;
|
||||
};
|
||||
|
||||
const DEFAULT_DELAY_MS = 30;
|
||||
const DEFAULT_REQUEST_TIMEOUT_MS = (PULL_REQUEST_BATCH_TIMEOUT_MAX_SECONDS + 10) * 1000;
|
||||
|
||||
/**
|
||||
* Collects the conversation ids asked for within a short window and fetches them in one request.
|
||||
* Each caller gets its own conversation's answer, and callers asking for the same conversation
|
||||
* share one entry. A request that fails rejects only the callers it carried.
|
||||
*/
|
||||
export function createPullRequestBatcher({
|
||||
fetchMany,
|
||||
delayMs = DEFAULT_DELAY_MS,
|
||||
maxBatch = PULL_REQUEST_BATCH_MAX,
|
||||
requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,
|
||||
}: BatcherOptions) {
|
||||
const pending = new Map<string, Waiter[]>();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let disposed = false;
|
||||
|
||||
/**
|
||||
* Requests go out one at a time. Each request lets the server run its own number of lookups at
|
||||
* once, so requests in flight together would multiply the limit the operator configured: a long
|
||||
* pinned list, or a second flush while one is still out, would otherwise put several times that
|
||||
* many lookups on GitHub at once. The tail never rejects, so one failed request cannot stall
|
||||
* the ones queued behind it.
|
||||
*/
|
||||
let tail: Promise<void> = Promise.resolve();
|
||||
/** Aborts whatever request is on the wire, so disposal does not leave its work running. */
|
||||
let active: AbortController | undefined;
|
||||
/** Requests queued or on the wire, so a cancel can tell when one has nobody left waiting. */
|
||||
const dispatches = new Set<Dispatch>();
|
||||
|
||||
const send = (dispatch: Dispatch): Promise<void> => {
|
||||
const { waiters } = dispatch;
|
||||
/** Ids nobody waits for any more (their rows unmounted while queued) are never asked for. */
|
||||
const ids = [...waiters.keys()].filter((id) => hasLiveWaiter(waiters.get(id)));
|
||||
if (!disposed && ids.length === 0) {
|
||||
dispatches.delete(dispatch);
|
||||
return Promise.resolve();
|
||||
}
|
||||
/** Work that was still waiting its turn when the batcher was disposed never goes out. */
|
||||
if (disposed) {
|
||||
for (const list of waiters.values()) {
|
||||
list.forEach((waiter) => waiter.reject(new PullRequestBatchError('DISPOSED')));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
const settle = ({ results }: TConversationPullRequestsResponse) => {
|
||||
const byId = new Map(results.map((entry) => [entry.conversationId, entry]));
|
||||
for (const [id, list] of waiters) {
|
||||
const entry = byId.get(id);
|
||||
for (const waiter of list) {
|
||||
if (entry != null && 'error' in entry) {
|
||||
waiter.reject(new PullRequestBatchError(entry.error.code));
|
||||
} else {
|
||||
waiter.resolve({ pullRequest: entry?.pullRequest ?? null });
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
const fail = (error: unknown) => {
|
||||
for (const list of waiters.values()) list.forEach((waiter) => waiter.reject(error));
|
||||
};
|
||||
const controller = new AbortController();
|
||||
dispatch.controller = controller;
|
||||
active = controller;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
/**
|
||||
* Settles the request on its timeout, and on an abort from disposal, so callers never wait on
|
||||
* a fetcher that ignores its signal. The timeout aborts first, so the code it reports stays
|
||||
* `TIMEOUT` and the abort listener only ever fires for disposal.
|
||||
*/
|
||||
const timedOut = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(new PullRequestBatchError('TIMEOUT'));
|
||||
controller.abort();
|
||||
}, requestTimeoutMs);
|
||||
controller.signal.addEventListener('abort', () =>
|
||||
reject(new PullRequestBatchError('ABORTED')),
|
||||
);
|
||||
});
|
||||
/**
|
||||
* `settle` is inside the guarded chain: a response that is not the shape it should be would
|
||||
* otherwise throw out of it, leave these callers waiting forever and, because the queue chains
|
||||
* on this promise, stop every later request. Anything that goes wrong here fails the callers
|
||||
* this request carried and nothing else.
|
||||
*/
|
||||
return Promise.race([fetchMany(ids, controller.signal), timedOut])
|
||||
.then(settle)
|
||||
.catch(fail)
|
||||
.finally(() => {
|
||||
clearTimeout(timer);
|
||||
dispatches.delete(dispatch);
|
||||
if (active === controller) active = undefined;
|
||||
});
|
||||
};
|
||||
|
||||
const flush = () => {
|
||||
timer = undefined;
|
||||
const ids = [...pending.keys()];
|
||||
for (let start = 0; start < ids.length; start += maxBatch) {
|
||||
const chunk = ids.slice(start, start + maxBatch);
|
||||
const waiters = new Map<string, Waiter[]>();
|
||||
for (const id of chunk) {
|
||||
waiters.set(id, pending.get(id) ?? []);
|
||||
pending.delete(id);
|
||||
}
|
||||
const dispatch: Dispatch = { waiters };
|
||||
dispatches.add(dispatch);
|
||||
tail = tail.then(() => send(dispatch));
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
/**
|
||||
* Ends this batcher's life, for the identity it was made for. What has not gone out yet is
|
||||
* rejected instead of being sent under whoever is signed in next, and nothing queued behind a
|
||||
* request still in flight waits on it for the next session. A request already on the wire
|
||||
* finishes on its own.
|
||||
*/
|
||||
dispose(): void {
|
||||
disposed = true;
|
||||
active?.abort();
|
||||
clearTimeout(timer);
|
||||
timer = undefined;
|
||||
for (const list of pending.values()) {
|
||||
list.forEach((waiter) => waiter.reject(new PullRequestBatchError('DISPOSED')));
|
||||
}
|
||||
pending.clear();
|
||||
},
|
||||
/**
|
||||
* `signal` withdraws this caller: it is rejected at once, its id leaves the queue if it has not
|
||||
* gone out, and a request nobody is waiting for any more is not started, or is aborted if it
|
||||
* is already on the wire, so rows that scrolled away never hold up the ones now visible.
|
||||
*/
|
||||
load(conversationId: string, signal?: AbortSignal): Promise<TConversationPullRequestResponse> {
|
||||
if (disposed) return Promise.reject(new PullRequestBatchError('DISPOSED'));
|
||||
if (signal?.aborted) return Promise.reject(new PullRequestBatchError('ABORTED'));
|
||||
return new Promise((resolve, reject) => {
|
||||
const waiter: Waiter = { resolve, reject };
|
||||
const waiters = pending.get(conversationId) ?? [];
|
||||
waiters.push(waiter);
|
||||
pending.set(conversationId, waiters);
|
||||
timer ??= setTimeout(flush, delayMs);
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
waiter.cancelled = true;
|
||||
reject(new PullRequestBatchError('ABORTED'));
|
||||
const queued = pending.get(conversationId);
|
||||
if (queued != null) {
|
||||
const rest = queued.filter((candidate) => candidate !== waiter);
|
||||
if (rest.length > 0) pending.set(conversationId, rest);
|
||||
else pending.delete(conversationId);
|
||||
if (pending.size === 0) {
|
||||
clearTimeout(timer);
|
||||
timer = undefined;
|
||||
}
|
||||
}
|
||||
for (const dispatch of dispatches) {
|
||||
const ids = [...dispatch.waiters.values()];
|
||||
if (ids.every((list) => !hasLiveWaiter(list))) dispatch.controller?.abort();
|
||||
}
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** An unknown or unusable limit is one call at a time, never more than the operator allowed. */
|
||||
const usableConcurrency = (value: number | undefined): number =>
|
||||
value != null && Number.isInteger(value) && value >= 1 ? value : 1;
|
||||
|
||||
const statusOf = (error: unknown): number | undefined => {
|
||||
if (error == null || typeof error !== 'object') return undefined;
|
||||
const candidate = error as { status?: number; response?: { status?: number } };
|
||||
return candidate.response?.status ?? candidate.status;
|
||||
};
|
||||
|
||||
const codeOf = (error: unknown): string => {
|
||||
const code = (error as { response?: { data?: { code?: unknown } } } | null)?.response?.data?.code;
|
||||
return typeof code === 'string' ? code : 'UPSTREAM_ERROR';
|
||||
};
|
||||
|
||||
/**
|
||||
* Asks the batch route, and when the server that answered does not have it (a replica that has
|
||||
* not been upgraded yet answers 404, whatever the startup config of another replica advertised),
|
||||
* asks the single route for each conversation instead, at the limit the server advertised (one at
|
||||
* a time when it did not say), since the single route has no limiter. Anything but a 404 is a
|
||||
* real failure and is passed on untouched.
|
||||
*/
|
||||
export function createBatchFetcher({
|
||||
fetchMany,
|
||||
fetchOne,
|
||||
concurrency,
|
||||
}: {
|
||||
/** The limit the server advertised, read when the fallback runs; one call at a time without it. */
|
||||
concurrency?: () => number | undefined;
|
||||
fetchMany: (
|
||||
conversationIds: string[],
|
||||
signal: AbortSignal,
|
||||
) => Promise<TConversationPullRequestsResponse>;
|
||||
fetchOne: (
|
||||
conversationId: string,
|
||||
signal: AbortSignal,
|
||||
) => Promise<TConversationPullRequestResponse>;
|
||||
}) {
|
||||
return async (
|
||||
conversationIds: string[],
|
||||
signal: AbortSignal,
|
||||
): Promise<TConversationPullRequestsResponse> => {
|
||||
try {
|
||||
return await fetchMany(conversationIds, signal);
|
||||
} catch (error) {
|
||||
if (statusOf(error) !== 404) throw error;
|
||||
}
|
||||
const results: TConversationPullRequestsResponse['results'] = new Array(conversationIds.length);
|
||||
let next = 0;
|
||||
/**
|
||||
* The fallback is many requests, and the batcher moves on the moment this one is settled, so
|
||||
* it stops itself: once the signal aborts no worker starts another call, and the call each
|
||||
* has in flight is aborted with it, so nothing carries on beside the next request.
|
||||
*/
|
||||
const worker = async (): Promise<void> => {
|
||||
for (let index = next++; index < conversationIds.length; index = next++) {
|
||||
if (signal.aborted) return;
|
||||
const conversationId = conversationIds[index];
|
||||
try {
|
||||
const { pullRequest } = await fetchOne(conversationId, signal);
|
||||
results[index] = { conversationId, pullRequest };
|
||||
} catch (error) {
|
||||
/** After an abort the whole fetch throws below, so a result written here is never read. */
|
||||
results[index] = { conversationId, error: { code: codeOf(error) } };
|
||||
}
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from(
|
||||
{ length: Math.min(usableConcurrency(concurrency?.()), conversationIds.length) },
|
||||
worker,
|
||||
),
|
||||
);
|
||||
if (signal.aborted) throw new PullRequestBatchError('ABORTED');
|
||||
return { results };
|
||||
};
|
||||
}
|
||||
|
|
@ -1 +1,2 @@
|
|||
export * from './queries';
|
||||
export * from './batch';
|
||||
|
|
|
|||
|
|
@ -1,11 +1,20 @@
|
|||
import React from 'react';
|
||||
import { renderHook, waitFor } from '@testing-library/react';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
import { act, renderHook, waitFor } from '@testing-library/react';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import type { TConversationPullRequest } from 'librechat-data-provider';
|
||||
import { pullRequestRefetchInterval, useConversationPullRequestQuery } from './queries';
|
||||
import type { TStartupConfig } from 'librechat-data-provider';
|
||||
import {
|
||||
endPullRequestSession,
|
||||
pullRequestRefetchInterval,
|
||||
useRowPullRequestQuery,
|
||||
useConversationPullRequestQuery,
|
||||
} from './queries';
|
||||
import store from '~/store';
|
||||
|
||||
const mockGet = jest.fn();
|
||||
const mockStartup: { current: { pullRequestsEnabled?: boolean } | undefined } = {
|
||||
const mockGetMany = jest.fn();
|
||||
const mockStartup: { current: Partial<TStartupConfig> | undefined } = {
|
||||
current: { pullRequestsEnabled: true },
|
||||
};
|
||||
|
||||
|
|
@ -20,6 +29,7 @@ jest.mock('librechat-data-provider', () => {
|
|||
dataService: {
|
||||
...actual.dataService,
|
||||
getConversationPullRequest: (...args: unknown[]) => mockGet(...args),
|
||||
getConversationPullRequests: (...args: unknown[]) => mockGetMany(...args),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
|
@ -116,3 +126,134 @@ describe('useConversationPullRequestQuery', () => {
|
|||
expect(mockGet).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('useRowPullRequestQuery', () => {
|
||||
const rowStartup = {
|
||||
pullRequestsEnabled: true,
|
||||
pullRequestsBatchVersion: 1,
|
||||
pullRequestsMaxConcurrentLookups: 2,
|
||||
} as const;
|
||||
const answerFor = (ids: string[]) => ({
|
||||
results: ids.map((conversationId) => ({ conversationId, pullRequest: pr })),
|
||||
});
|
||||
|
||||
const wrapRows = (userId = 'user-1') => {
|
||||
const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
const Wrapper = ({ children }: { children: React.ReactNode }) => (
|
||||
<QueryClientProvider client={client}>
|
||||
<RecoilRoot initializeState={({ set }) => set(store.user, { id: userId } as never)}>
|
||||
{children}
|
||||
</RecoilRoot>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
return { Wrapper, client };
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
endPullRequestSession();
|
||||
mockGet.mockReset().mockResolvedValue({ pullRequest: null });
|
||||
mockGetMany.mockReset().mockImplementation(async (ids: string[]) => answerFor(ids));
|
||||
mockStartup.current = { ...rowStartup };
|
||||
});
|
||||
afterEach(() => endPullRequestSession());
|
||||
|
||||
it('asks the batch route once for rows mounted together', async () => {
|
||||
const { Wrapper } = wrapRows();
|
||||
const a = renderHook(() => useRowPullRequestQuery('a'), { wrapper: Wrapper });
|
||||
const b = renderHook(() => useRowPullRequestQuery('b'), { wrapper: Wrapper });
|
||||
await waitFor(() => expect(a.result.current.data).toEqual({ pullRequest: pr }));
|
||||
await waitFor(() => expect(b.result.current.data).toEqual({ pullRequest: pr }));
|
||||
expect(mockGetMany).toHaveBeenCalledTimes(1);
|
||||
expect(mockGetMany.mock.calls[0][0]).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('does not ask for a row that unmounted before its request went out', async () => {
|
||||
const { Wrapper } = wrapRows();
|
||||
const gone = renderHook(() => useRowPullRequestQuery('gone'), { wrapper: Wrapper });
|
||||
const kept = renderHook(() => useRowPullRequestQuery('kept'), { wrapper: Wrapper });
|
||||
gone.unmount();
|
||||
await waitFor(() => expect(kept.result.current.data).toEqual({ pullRequest: pr }));
|
||||
expect(mockGetMany).toHaveBeenCalledTimes(1);
|
||||
expect(mockGetMany.mock.calls[0][0]).toEqual(['kept']);
|
||||
});
|
||||
|
||||
it('aborts a request on the wire once the last row waiting for it unmounts', async () => {
|
||||
let seen: AbortSignal | undefined;
|
||||
mockGetMany.mockReset().mockImplementation(
|
||||
(_ids: string[], options?: { signal?: AbortSignal }) =>
|
||||
new Promise(() => {
|
||||
seen = options?.signal;
|
||||
}),
|
||||
);
|
||||
const { Wrapper } = wrapRows();
|
||||
const row = renderHook(() => useRowPullRequestQuery('a'), { wrapper: Wrapper });
|
||||
await waitFor(() => expect(seen).toBeDefined());
|
||||
expect(seen?.aborted).toBe(false);
|
||||
row.unmount();
|
||||
await waitFor(() => expect(seen?.aborted).toBe(true));
|
||||
});
|
||||
|
||||
it('starts a fresh queue after the session ends, even for the same account', async () => {
|
||||
const stalled: Array<AbortSignal | undefined> = [];
|
||||
mockGetMany
|
||||
.mockReset()
|
||||
.mockImplementationOnce(
|
||||
(_ids: string[], options?: { signal?: AbortSignal }) =>
|
||||
new Promise(() => {
|
||||
stalled.push(options?.signal);
|
||||
}),
|
||||
)
|
||||
.mockImplementation(async (ids: string[]) => answerFor(ids));
|
||||
const first = wrapRows();
|
||||
renderHook(() => useRowPullRequestQuery('a'), { wrapper: first.Wrapper });
|
||||
await waitFor(() => expect(stalled).toHaveLength(1));
|
||||
act(() => endPullRequestSession());
|
||||
expect(stalled[0]?.aborted).toBe(true);
|
||||
const second = wrapRows();
|
||||
const row = renderHook(() => useRowPullRequestQuery('b'), { wrapper: second.Wrapper });
|
||||
await waitFor(() => expect(row.result.current.data).toEqual({ pullRequest: pr }));
|
||||
expect(mockGetMany.mock.calls.at(-1)?.[0]).toEqual(['b']);
|
||||
});
|
||||
|
||||
it('falls back to the single route no faster than the limit the server advertised', async () => {
|
||||
const notFound = Object.assign(new Error('Not Found'), { response: { status: 404 } });
|
||||
mockGetMany.mockReset().mockRejectedValue(notFound);
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
mockGet.mockReset().mockImplementation(async () => {
|
||||
active += 1;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
active -= 1;
|
||||
return { pullRequest: null };
|
||||
});
|
||||
const { Wrapper } = wrapRows();
|
||||
const rows = ['a', 'b', 'c', 'd', 'e', 'f'].map((id) =>
|
||||
renderHook(() => useRowPullRequestQuery(id), { wrapper: Wrapper }),
|
||||
);
|
||||
await waitFor(() => rows.forEach((row) => expect(row.result.current.isSuccess).toBe(true)));
|
||||
expect(mockGet).toHaveBeenCalledTimes(6);
|
||||
expect(peak).toBe(2);
|
||||
});
|
||||
|
||||
it('falls back one call at a time when the server advertised no limit', async () => {
|
||||
mockStartup.current = { pullRequestsEnabled: true, pullRequestsBatchVersion: 1 };
|
||||
const notFound = Object.assign(new Error('Not Found'), { response: { status: 404 } });
|
||||
mockGetMany.mockReset().mockRejectedValue(notFound);
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
mockGet.mockReset().mockImplementation(async () => {
|
||||
active += 1;
|
||||
peak = Math.max(peak, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
active -= 1;
|
||||
return { pullRequest: null };
|
||||
});
|
||||
const { Wrapper } = wrapRows();
|
||||
const rows = ['a', 'b', 'c'].map((id) =>
|
||||
renderHook(() => useRowPullRequestQuery(id), { wrapper: Wrapper }),
|
||||
);
|
||||
await waitFor(() => rows.forEach((row) => expect(row.result.current.isSuccess).toBe(true)));
|
||||
expect(peak).toBe(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,16 @@
|
|||
import { useRecoilValue } from 'recoil';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Constants, QueryKeys, dataService } from 'librechat-data-provider';
|
||||
import {
|
||||
Constants,
|
||||
QueryKeys,
|
||||
dataService,
|
||||
PULL_REQUEST_BATCH_VERSION,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TConversationPullRequestResponse } from 'librechat-data-provider';
|
||||
import type { UseQueryOptions } from '@tanstack/react-query';
|
||||
import { createBatchFetcher, createPullRequestBatcher } from './batch';
|
||||
import { useGetStartupConfig } from '../Endpoints';
|
||||
import store from '~/store';
|
||||
|
||||
const SETTLED_REFRESH_MS = 60_000;
|
||||
/** Checks and conflict state change while a pull request is being worked on. */
|
||||
|
|
@ -23,6 +31,11 @@ export const pullRequestRefetchInterval = (
|
|||
return pr.mergeable === 'unknown' ? ACTIVE_REFRESH_MS : SETTLED_REFRESH_MS;
|
||||
};
|
||||
|
||||
const isSavedConversation = (conversationId: string) =>
|
||||
conversationId !== '' &&
|
||||
conversationId !== Constants.NEW_CONVO &&
|
||||
conversationId !== Constants.PENDING_CONVO;
|
||||
|
||||
/**
|
||||
* Asks only when the deployment advertises the feature and the conversation is a saved one, so a
|
||||
* default installation never calls the endpoint. The server answers no pull request for a chat
|
||||
|
|
@ -37,11 +50,7 @@ export const useConversationPullRequestQuery = (
|
|||
[QueryKeys.conversationPullRequest, conversationId],
|
||||
() => dataService.getConversationPullRequest(conversationId),
|
||||
{
|
||||
enabled:
|
||||
startupConfig?.pullRequestsEnabled === true &&
|
||||
conversationId !== '' &&
|
||||
conversationId !== Constants.NEW_CONVO &&
|
||||
conversationId !== Constants.PENDING_CONVO,
|
||||
enabled: startupConfig?.pullRequestsEnabled === true && isSavedConversation(conversationId),
|
||||
staleTime: 15_000,
|
||||
retry: false,
|
||||
refetchOnWindowFocus: true,
|
||||
|
|
@ -51,3 +60,71 @@ export const useConversationPullRequestQuery = (
|
|||
},
|
||||
);
|
||||
};
|
||||
|
||||
/** What the server last advertised, for a fallback that has no batch limiter to lean on. */
|
||||
let advertisedConcurrency: number | undefined;
|
||||
|
||||
const fetchRows = createBatchFetcher({
|
||||
fetchMany: (conversationIds, signal) =>
|
||||
dataService.getConversationPullRequests(conversationIds, { signal }),
|
||||
fetchOne: (conversationId, signal) =>
|
||||
dataService.getConversationPullRequest(conversationId, { signal }),
|
||||
concurrency: () => advertisedConcurrency,
|
||||
});
|
||||
|
||||
type RowBatcher = ReturnType<typeof createPullRequestBatcher>;
|
||||
let rowBatcher: { userId: string; batcher: RowBatcher } | undefined;
|
||||
|
||||
/**
|
||||
* The batcher for the signed-in user: every sidebar row mounted in the same moment shares one
|
||||
* request. It belongs to one identity, so when the user changes the old one is disposed, and ids
|
||||
* it still held are neither sent under the next account's authorization nor made to wait ahead
|
||||
* of that account's own rows.
|
||||
*/
|
||||
const batcherFor = (userId: string): RowBatcher => {
|
||||
if (rowBatcher?.userId === userId) return rowBatcher.batcher;
|
||||
rowBatcher?.batcher.dispose();
|
||||
const batcher = createPullRequestBatcher({ fetchMany: fetchRows });
|
||||
rowBatcher = { userId, batcher };
|
||||
return batcher;
|
||||
};
|
||||
|
||||
/**
|
||||
* Ends the row queue with the session. Called wherever the session's client state is cleared, so
|
||||
* signing back in as the same account starts a fresh queue and neither waits behind, nor sends,
|
||||
* anything the previous session left.
|
||||
*/
|
||||
export const endPullRequestSession = (): void => {
|
||||
rowBatcher?.batcher.dispose();
|
||||
rowBatcher = undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* The pull request of a sidebar row. It shares its cache entry with the header's query, so the
|
||||
* open conversation is never fetched twice, but it never polls: a list of rows would otherwise
|
||||
* poll GitHub once per row. It refreshes when the window regains focus and a row's answer is
|
||||
* older than a minute; every mounted row asks in the same moment, so the batcher turns that
|
||||
* into one request for the whole list rather than one per row.
|
||||
*/
|
||||
export const useRowPullRequestQuery = (conversationId: string) => {
|
||||
const { data: startupConfig } = useGetStartupConfig();
|
||||
const userId = useRecoilValue(store.user)?.id ?? '';
|
||||
return useQuery<TConversationPullRequestResponse>(
|
||||
[QueryKeys.conversationPullRequest, conversationId],
|
||||
({ signal }) => {
|
||||
advertisedConcurrency = startupConfig?.pullRequestsMaxConcurrentLookups;
|
||||
return batcherFor(userId).load(conversationId, signal);
|
||||
},
|
||||
{
|
||||
/** The batch route is newer than the flag: ask only a server that says it has it. */
|
||||
enabled:
|
||||
startupConfig?.pullRequestsEnabled === true &&
|
||||
startupConfig.pullRequestsBatchVersion === PULL_REQUEST_BATCH_VERSION &&
|
||||
userId !== '' &&
|
||||
isSavedConversation(conversationId),
|
||||
staleTime: SETTLED_REFRESH_MS,
|
||||
retry: false,
|
||||
refetchOnWindowFocus: true,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue