🚜 feat: Support Native PostgreSQL Tool Favorites (#18032)

* 🚜 feat: Support Native PostgreSQL Tool Favorites

* fix: Guard PostgreSQL Reconciliation and Connection Leases

* fix: Scope Favorites Erasure Fences to Deletion Attempts

* fix: Preserve Account Erasure Ownership Until Cancellation Completes

* fix: Reject Unsupported PostgreSQL Command Flags Before Connecting

* fix: Keep Command Validation Compatible With the TypeScript Target

* fix: Keep Migration Previews Read-Only and Preserve Erasure Recovery Identity

* fix: Fence Erasure Recovery Across Tenant and Ownership Changes

* fix: Isolate Migration Dependencies and Keep Mongo Scans Outside Transactions

* fix: Inject PostgreSQL Backend Factories at Composition Roots

* ci: Shard Complete Mutation Work Without Relaxing the Aggregate Gate

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
lia-by-librechat[bot] 2026-11-11 12:07:07 -05:00 • committed by GitHub
parent dbe9f16a20
commit 42219ec8e9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
60 changed files with 6433 additions and 112 deletions

View file

@ -54,6 +54,7 @@ jobs:
outputs:
workspaces: ${{ steps.select.outputs.workspaces }}
hasChanges: ${{ steps.select.outputs.hasChanges }}
postgresChanged: ${{ steps.postgres.outputs.changed }}
steps:
- uses: actions/checkout@v5
with:
@ -69,13 +70,56 @@ jobs:
QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }}
run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE"
- name: Select PostgreSQL schema and lifecycle changes
id: postgres
run: |
if git diff --name-only "$QUALITY_BASE...$QUALITY_HEAD" | grep -Eq '^(packages/data-schemas/src/(postgres/|types/favorite|methods/(favorite|index))|packages/api/src/app/(persistence|postgres)|packages/data-provider/src/config|api/(models/index|server/(index|experimental))|config/(postgres|delete-user)|packages/data-schemas/jest.postgres|package-lock.json|.github/workflows/change-quality.yml)'; then
echo 'changed=true' >> "$GITHUB_OUTPUT"
fi
postgres:
name: PostgreSQL 16 contracts and readiness
needs: [select]
if: needs.select.outputs.postgresChanged == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_HOST_AUTH_METHOD: trust
ports: ['5432:5432']
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 5
env:
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
NODE_OPTIONS: --max-old-space-size=6144
steps:
- uses: actions/checkout@v5
with:
ref: ${{ env.QUALITY_HEAD }}
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: '24.22.1'
cache: npm
- run: npm ci
- run: npm run build:data-provider && npm run build:data-schemas && npm run build:api
- run: npm run test:postgres --workspace packages/data-schemas
- run: npx --no-install jest src/app/persistence.spec.ts src/app/postgres.spec.ts src/favorites/handlers.spec.ts src/user/erasure.spec.ts --coverage=false --runInBand
working-directory: packages/api
- run: npx --no-install jest --config config/jest.config.js --runTestsByPath config/__tests__/delete-user.recovery.spec.js --coverage=false --runInBand
reports:
name: ${{ matrix.check == 'risk' && 'Change risk' || 'Mutation testing' }} (${{ matrix.workspace }})
name: ${{ matrix.report.check == 'risk' && format('Change risk ({0})', matrix.workspace) || format('Mutation shard {0}/3 ({1})', matrix.report.shard, matrix.workspace) }}
needs: [select, verify]
if: needs.select.outputs.hasChanges == 'true'
runs-on: ubuntu-latest
# Whole edited database functions can produce thousands of mutation cases.
timeout-minutes: ${{ matrix.check == 'mutation' && 240 || 45 }}
timeout-minutes: ${{ matrix.report.check == 'mutation' && 240 || 45 }}
defaults:
run:
working-directory: source
@ -83,7 +127,11 @@ jobs:
fail-fast: false
matrix:
workspace: ${{ fromJSON(needs.select.outputs.workspaces) }}
check: [risk, mutation]
report:
- { check: risk, shard: 0 }
- { check: mutation, shard: 1 }
- { check: mutation, shard: 2 }
- { check: mutation, shard: 3 }
services:
redis:
image: redis:7-alpine
@ -93,7 +141,18 @@ jobs:
--health-interval 5s
--health-timeout 5s
--health-retries 5
postgres:
image: postgres:16-alpine
env:
POSTGRES_HOST_AUTH_METHOD: trust
ports: ['5432:5432']
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 5
env:
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
REDIS_URI: redis://127.0.0.1:6379
NODE_OPTIONS: --max-old-space-size=6144
steps:
@ -138,15 +197,59 @@ jobs:
- name: Generate advisory report
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
QUALITY_CHECK: ${{ matrix.check }}
run: >-
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD"
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK"
QUALITY_CHECK: ${{ matrix.report.check }}
QUALITY_SHARD: ${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}
run: |
args=()
if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}"
- name: Upload reports
if: always()
uses: actions/upload-artifact@v6
with:
name: change-${{ matrix.check }}-${{ matrix.workspace }}-${{ github.event.pull_request.head.sha || github.sha }}
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.check }}/
name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }}
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/
if-no-files-found: warn
retention-days: 14
mutation:
name: Mutation testing (${{ matrix.workspace }})
needs: [select, reports]
if: always() && needs.select.result == 'success' && needs.select.outputs.hasChanges == 'true'
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
workspace: ${{ fromJSON(needs.select.outputs.workspaces) }}
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: '24.22.1'
- uses: actions/download-artifact@v7
with:
pattern: change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}
path: mutation-reports
- name: Verify all mutation shards
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
run: >-
node scripts/quality/aggregate.mts --directory mutation-reports --head "$QUALITY_HEAD"
--workspace "$QUALITY_WORKSPACE" --count 3
--output "coverage/changes/$QUALITY_WORKSPACE/mutation"
- name: Require successful report jobs
env:
REPORT_RESULT: ${{ needs.reports.result }}
run: test "$REPORT_RESULT" = success
- uses: actions/upload-artifact@v6
if: always()
with:
name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}
path: coverage/changes/${{ matrix.workspace }}/mutation/
if-no-files-found: warn
retention-days: 14

View file

@ -1,10 +1,16 @@
const mongoose = require('mongoose');
const { createMethods } = require('@librechat/data-schemas');
const {
createMethods,
createToolFavoriteMethods,
createPostgresFavoriteBackend,
} = require('@librechat/data-schemas');
const {
matchModelName,
findMatchingPattern,
isDeploymentSkillId,
createMessageBudgetReader,
createFavoriteRuntime,
getPersistenceConfig,
createChannelStorageConfigReader,
} = require('@librechat/api');
const getLogStores = require('~/cache/getLogStores');
@ -12,7 +18,14 @@ const getLogStores = require('~/cache/getLogStores');
const messageBudget = createMessageBudgetReader();
const channelStorageConfig = createChannelStorageConfigReader();
const favorites = createFavoriteRuntime(
createToolFavoriteMethods(mongoose),
process.env,
createPostgresFavoriteBackend,
);
const methods = createMethods(mongoose, {
toolFavorites: favorites.methods,
matchModelName,
findMatchingPattern,
isExternalSkillId: isDeploymentSkillId,
@ -30,6 +43,8 @@ const seedDatabase = async () => {
module.exports = {
...methods,
initializeFavorites: (config) => favorites.initialize(getPersistenceConfig(config)),
closeFavorites: favorites.close,
initializeMessageBudget: messageBudget.initialize,
initializeChannelStorageConfig: channelStorageConfig.initialize,
seedDatabase,

View file

@ -1,5 +1,18 @@
jest.mock('@librechat/data-schemas', () => ({ createMethods: jest.fn(() => ({})) }));
jest.mock('@librechat/data-schemas', () => ({
createPostgresFavoriteBackend: jest.fn(),
createMethods: jest.fn((_mongoose, deps) => ({ ...deps.toolFavorites })),
createToolFavoriteMethods: jest.fn(() => ({
getToolFavorites: jest.fn().mockResolvedValue([]),
addToolFavorite: jest.fn().mockResolvedValue({ ok: true, added: true }),
removeToolFavorite: jest.fn().mockResolvedValue({ ok: true, removed: true }),
deleteUserToolFavorites: jest.fn().mockResolvedValue(2),
})),
}));
jest.mock('@librechat/api', () => ({
...jest.requireActual('../../packages/api/src/app/persistence'),
createFavoriteRuntime: jest.fn(
jest.requireActual('../../packages/api/src/app/persistence').createFavoriteRuntime,
),
matchModelName: jest.fn(),
findMatchingPattern: jest.fn(),
isDeploymentSkillId: jest.fn(),
@ -11,13 +24,25 @@ jest.mock('@librechat/api', () => ({
}));
jest.mock('~/cache/getLogStores', () => jest.fn());
const { createMethods } = require('@librechat/data-schemas');
const { createMessageBudgetReader, createChannelStorageConfigReader } = require('@librechat/api');
const { initializeMessageBudget, initializeChannelStorageConfig } = require('./index');
const {
createToolFavoriteMethods,
createMethods,
createPostgresFavoriteBackend,
} = require('@librechat/data-schemas');
const {
createMessageBudgetReader,
createFavoriteRuntime,
createChannelStorageConfigReader,
} = require('@librechat/api');
const models = require('./index');
const { initializeMessageBudget, initializeChannelStorageConfig } = models;
const reader = createMessageBudgetReader.mock.results[0].value;
const channelStorageConfigReader = createChannelStorageConfigReader.mock.results[0].value;
const dependencies = createMethods.mock.calls[0][1];
const runtimeDependencies = createFavoriteRuntime.mock.calls[0];
const { getMCPAppMessageBudget, getChannelStorageConfig: channelStorageConfigDependency } =
createMethods.mock.calls[0][1];
dependencies;
const mongoFavorites = createToolFavoriteMethods.mock.results[0].value;
describe('message App budget dependency wiring', () => {
it('passes the same reader to storage and initializer to the composition root', () => {
@ -27,6 +52,26 @@ describe('message App budget dependency wiring', () => {
});
});
describe('favorites composition root', () => {
it('passes stable runtime methods to storage and keeps captured deletion references usable', async () => {
expect(runtimeDependencies).toEqual([
mongoFavorites,
process.env,
createPostgresFavoriteBackend,
]);
const captured = models.deleteUserToolFavorites;
expect(dependencies.toolFavorites.deleteUserToolFavorites).toBe(captured);
await expect(captured('user')).rejects.toThrow('not initialized');
await models.initializeFavorites(null);
expect(createPostgresFavoriteBackend).not.toHaveBeenCalled();
expect(await captured('user')).toBe(2);
expect(mongoFavorites.deleteUserToolFavorites).toHaveBeenCalledWith('user');
expect(models.deleteUserToolFavorites).toBe(captured);
await models.closeFavorites();
await expect(captured('user')).rejects.toThrow('not initialized');
});
});
describe('channel storage configuration dependency wiring', () => {
it('injects the deployment reader without reading config during module initialization', () => {
expect(channelStorageConfigDependency).toBe(channelStorageConfigReader.getConfig);

View file

@ -17,6 +17,7 @@ const {
stopUserGenerations,
deleteAllSharedLinksWithCleanup,
eraseUserData,
createUserDeletionAttempt,
deleteUserMCPServers,
revokeUserCodeEnvironmentWorkers,
finalizeMCPAuthorizationMutation,
@ -370,6 +371,7 @@ const deleteUserController = async (req, res) => {
let triggerDeletionFence;
let scheduleSuspensionToken;
let userDeleted = false;
const deletionAttempt = createUserDeletionAttempt();
try {
const existingUser = await db.getUserById(
@ -391,7 +393,11 @@ const deleteUserController = async (req, res) => {
// Block new trigger admissions across replicas while preserving the user
// principal so a transient cleanup failure remains retryable.
triggerDeletionFence = new Date();
const fenceState = await db.beginAgentTriggerUserDeletion(user.id, triggerDeletionFence);
const fenceState = await db.beginAgentTriggerUserDeletion(
user.id,
triggerDeletionFence,
deletionAttempt.id,
);
if (fenceState === 'in_progress') {
triggerDeletionFence = undefined;
throw new Error('Agent trigger account deletion is already in progress');
@ -425,6 +431,7 @@ const deleteUserController = async (req, res) => {
await eraseUserData(
{
userId: user.id,
deletionAttemptId: deletionAttempt.id,
tenantId,
checkpointer: appConfig?.endpoints?.agents?.checkpointer,
},
@ -439,7 +446,12 @@ const deleteUserController = async (req, res) => {
onSessionsDeleted: () => clearSessionHintCookie(res),
},
);
const deleteResult = await db.deleteUserById(user.id);
const deleteResult = await deletionAttempt.commit(() =>
db.deleteUserById(user.id, {
startedAt: triggerDeletionFence,
attemptId: deletionAttempt.id,
}),
);
if (deleteResult.deletedCount !== 1) {
throw new Error('User disappeared before account deletion could commit');
}
@ -506,7 +518,14 @@ const deleteUserController = async (req, res) => {
);
}
try {
await db.cancelAgentTriggerUserDeletion(user.id, triggerDeletionFence);
await deletionAttempt.cancel(
{
userId: user.id,
startedAt: triggerDeletionFence,
deletionAttemptId: deletionAttempt.id,
},
db,
);
} catch (fenceError) {
logger.error('[deleteUserController] Failed to release trigger deletion fence', fenceError);
}

View file

@ -55,6 +55,7 @@ jest.mock('~/models', () => {
deleteAclEntries: jest.fn().mockResolvedValue(undefined),
deleteUserChatProjects: jest.fn().mockResolvedValue(0),
deleteUserToolFavorites: jest.fn().mockResolvedValue(0),
cancelUserToolFavoriteDeletion: jest.fn().mockResolvedValue(undefined),
deleteRefreshTokenBridges: jest.fn().mockResolvedValue(undefined),
deleteAllAgentQueuedTurnsForUser: jest.fn().mockResolvedValue(0),
findUserErasableMCPServers: jest.fn().mockResolvedValue([]),
@ -64,6 +65,7 @@ jest.mock('~/models', () => {
deleteUserById: jest.fn().mockResolvedValue({ deletedCount: 1 }),
deleteAllUserPluginAuths: jest.fn().mockResolvedValue(undefined),
beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true),
cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),
deleteUserPrompts: jest.fn().mockResolvedValue(undefined),
deleteUserSkills: jest.fn().mockResolvedValue(undefined),
@ -592,9 +594,20 @@ describe('deleteUserController', () => {
expect(mockRes.status).toHaveBeenCalledWith(200);
expect(mockRes.send).toHaveBeenCalledWith({ message: 'User deleted' });
expect(deleteUserById).toHaveBeenCalledWith(userIdStr, {
startedAt: beginAgentTriggerUserDeletion.mock.calls[0][1],
attemptId: beginAgentTriggerUserDeletion.mock.calls[0][2],
});
expect(deleteUserById.mock.calls[0][1].attemptId).not.toBe(
mockQuiesceUserSchedules.mock.calls[0][1],
);
expect(mockRes.clearCookie).toHaveBeenCalledWith('session_hint');
expect(deletePasskeysByUser).toHaveBeenCalledWith(userIdStr);
expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(userId.toString(), expect.any(Date));
expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
userId.toString(),
expect.any(Date),
expect.any(String),
);
expect(mockPrepareAgentTriggerUserPurge).toHaveBeenCalledWith(
userId.toString(),
expect.any(Date),
@ -929,7 +942,11 @@ describe('deleteUserController', () => {
expect(deleteMessages).not.toHaveBeenCalled();
const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
userIdString,
deletionFence,
expect.any(String),
);
expect(deleteUserById).not.toHaveBeenCalled();
// Account survives -> its suspended schedules are restored under the quiesce token.
expect(mockRestoreUserSchedules).toHaveBeenCalledWith(
@ -944,6 +961,42 @@ describe('deleteUserController', () => {
);
});
it.each([false, 'error'])(
'does not release the favorite marker without a proven Mongo owner: %s',
async (result) => {
const db = require('~/models');
db.deleteUserAgents.mockRejectedValueOnce(new Error('later cascade failure'));
if (result === 'error')
db.isAgentTriggerUserDeletionOwner.mockRejectedValueOnce(new Error('Mongo unavailable'));
else db.isAgentTriggerUserDeletionOwner.mockResolvedValueOnce(false);
const userId = new mongoose.Types.ObjectId().toString();
await deleteUserController({ user: { id: userId, tenantId: 'tenant-1' } }, mockRes);
expect(mockRes.status).toHaveBeenCalledWith(500);
expect(db.deleteUserToolFavorites).toHaveBeenCalledWith(
userId,
beginAgentTriggerUserDeletion.mock.calls[0][2],
);
expect(db.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
},
);
it('releases the cascade UUID before releasing Mongo ownership', async () => {
const db = require('~/models');
db.deleteUserAgents.mockRejectedValueOnce(new Error('later cascade failure'));
const userId = new mongoose.Types.ObjectId().toString();
await deleteUserController({ user: { id: userId, tenantId: 'tenant-1' } }, mockRes);
const token = beginAgentTriggerUserDeletion.mock.calls[0][2];
expect(mockRes.status).toHaveBeenCalledWith(500);
expect(db.deleteUserToolFavorites).toHaveBeenCalledWith(userId, token);
expect(db.cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(userId, token);
expect(db.isAgentTriggerUserDeletionOwner.mock.invocationCallOrder[0]).toBeLessThan(
db.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0],
);
expect(db.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]).toBeLessThan(
db.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],
);
});
it('fails closed before data cleanup when detached subagents do not drain', async () => {
const userId = new mongoose.Types.ObjectId();
mockCancelAndDrainSubagentThreads.mockRejectedValueOnce(new Error('child drain timed out'));
@ -963,6 +1016,7 @@ describe('deleteUserController', () => {
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
userId.toString(),
expect.any(Date),
expect.any(String),
);
expect(deleteUserById).not.toHaveBeenCalled();
});
@ -987,7 +1041,11 @@ describe('deleteUserController', () => {
expect(mockGetActiveJobIdsForUser).not.toHaveBeenCalled();
const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
userIdString,
deletionFence,
expect.any(String),
);
expect(deleteUserById).not.toHaveBeenCalled();
// Account survives -> its suspended schedules are restored under the quiesce token.
expect(mockRestoreUserSchedules).toHaveBeenCalledWith(
@ -1074,6 +1132,7 @@ describe('deleteUserController', () => {
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
userId.toString(),
expect.any(Date),
expect.any(String),
);
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(
userId.toString(),
@ -1090,7 +1149,22 @@ describe('deleteUserController', () => {
expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
String(userId),
expect.any(Date),
expect.any(String),
);
});
it('retains both erasure fences after an unknown final delete outcome', async () => {
const db = require('~/models');
const userId = new mongoose.Types.ObjectId().toString();
db.deleteUserById.mockRejectedValueOnce(new Error('delete acknowledgement lost'));
await deleteUserController({ user: { id: userId } }, mockRes);
expect(mockRes.status).toHaveBeenCalledWith(500);
expect(db.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled();
expect(db.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
expect(db.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
});
it('does not purge trigger payloads when the user deletion did not commit', async () => {
@ -1101,7 +1175,7 @@ describe('deleteUserController', () => {
expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));
expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
expect(mockRes.status).toHaveBeenCalledWith(500);
});

View file

@ -70,6 +70,10 @@ jest.mock('@librechat/api', () => ({
getNewS3URL: jest.fn(),
deleteAllSharedLinksWithCleanup: (...args) => mockDeleteAllSharedLinksWithCleanup(...args),
eraseUserData: jest.requireActual('../../../../packages/api/src/user/erasure.ts').eraseUserData,
createUserDeletionAttempt: jest.requireActual('../../../../packages/api/src/user/erasure.ts')
.createUserDeletionAttempt,
cancelUserDataDeletion: jest.requireActual('../../../../packages/api/src/user/erasure.ts')
.cancelUserDataDeletion,
revokeUserCodeEnvironmentWorkers: (...args) => mockRevokeUserCodeEnvironmentWorkers(...args),
GenerationJobManager: {
getAccountCleanupJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args),
@ -98,6 +102,7 @@ jest.mock('~/models', () => ({
deleteUserById: (...args) => mockDeleteUserById(...args),
deleteAllUserPluginAuths: (...args) => mockDeleteAllUserPluginAuths(...args),
beginAgentTriggerUserDeletion: (...args) => mockBeginAgentTriggerUserDeletion(...args),
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true),
cancelAgentTriggerUserDeletion: (...args) => mockCancelAgentTriggerUserDeletion(...args),
deleteMessages: (...args) => mockDeleteMessages(...args),
deletePresets: (...args) => mockDeletePresets(...args),
@ -429,7 +434,10 @@ describe('deleteUserController - 2FA enforcement', () => {
expect(res.status).toHaveBeenCalledWith(200);
expect(mockBeginAgentTriggerUserDeletion).toHaveBeenCalledTimes(1);
expect(mockDeleteMessages).toHaveBeenCalled();
expect(mockDeleteUserById).toHaveBeenCalledWith(USER_ID);
expect(mockDeleteUserById).toHaveBeenCalledWith(USER_ID, {
startedAt: mockBeginAgentTriggerUserDeletion.mock.calls[0][1],
attemptId: mockBeginAgentTriggerUserDeletion.mock.calls[0][2],
});
expect(mockDeleteUserCodeEnvironments).not.toHaveBeenCalled();
expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();
expect(mockCancelAgentTriggerUserDeletion).not.toHaveBeenCalled();

View file

@ -97,6 +97,8 @@ const {
getRoleByName,
updateAccessPermissions,
seedDatabase,
initializeFavorites,
closeFavorites,
recordAuditEntry,
sweepOrphanedPreviews,
} = require('~/models');
@ -104,7 +106,7 @@ const { checkMigrations } = require('./services/start/migration');
const initializeMCPs = require('./services/initializeMCPs');
const configureSocialLogins = require('./socialLogins');
const createSpaFallback = require('./utils/fallback');
const { getAppConfig } = require('./services/Config');
const { getAppConfig, loadCustomConfig } = require('./services/Config');
const staticCache = require('./utils/staticCache');
const optionalJwtAuth = require('./middleware/optionalJwtAuth');
const noIndex = require('./middleware/noIndex');
@ -524,6 +526,8 @@ if (cluster.isMaster) {
* deletion, the expired-file sweep, tool-approval hooks and the other worker-wide settings
* below are fixed for the process lifetime, so DB `__base__` overrides must not decide them.
*/
await initializeFavorites(await loadCustomConfig(false, { storageSelection: true }));
registerShutdownTask('favorites persistence', closeFavorites, { priority: -100 });
const appConfig = await getAppConfig({ baseOnly: true });
initializeFileStorage(appConfig);
initializeGitHubSkillSync(appConfig);
@ -791,7 +795,8 @@ if (cluster.isMaster) {
setupGracefulShutdown(server);
};
startServer().catch((err) => {
startServer().catch(async (err) => {
await closeFavorites().catch(() => undefined);
logger.error(`Failed to start worker ${process.pid}:`, err);
process.exit(1);
});

View file

@ -78,6 +78,8 @@ const {
sweepOrphanedPreviews,
getRoleByName,
seedDatabase,
initializeFavorites,
closeFavorites,
listAllConfigs,
recordAuditEntry,
} = require('~/models');
@ -111,7 +113,7 @@ const initializeMCPs = require('./services/initializeMCPs');
const { configureSubagentTaskRouting } = require('./services/Endpoints/agents/subagentThreadStore');
const configureSocialLogins = require('./socialLogins');
const createSpaFallback = require('./utils/fallback');
const { getAppConfig, rederiveEndpoints } = require('./services/Config');
const { getAppConfig, rederiveEndpoints, loadCustomConfig } = require('./services/Config');
const staticCache = require('./utils/staticCache');
const noIndex = require('./middleware/noIndex');
const agentEventMethods = require('~/models');
@ -257,6 +259,8 @@ const startServer = async () => {
runAsSystem(sweepOrphanedPreviews).catch((err) => {
logger.error('[sweepOrphanedPreviews] Background sweep failed:', err);
});
await initializeFavorites(await loadCustomConfig(false, { storageSelection: true }));
registerShutdownTask('favorites persistence', closeFavorites, { priority: -100 });
const appConfig = await getAppConfig({ baseOnly: true });
await configureSubagentTaskRouting(appConfig?.endpoints?.agents?.subagentActivity);
registerBackgroundTaskShutdown({
@ -618,7 +622,8 @@ const startServer = async () => {
* liveness probe eventually times out. Mirrors the pattern in
* `experimental.js`.
*/
startServer().catch((err) => {
startServer().catch(async (err) => {
await closeFavorites().catch(() => undefined);
logger.error('Failed to start server:', err);
process.exit(1);
});

View file

@ -0,0 +1,137 @@
const { readFileSync } = require('node:fs');
const { resolve, dirname } = require('node:path');
const { runInNewContext } = require('node:vm');
describe.each(['index', 'experimental'])('%s entrypoint lifecycle wiring', (entrypoint) => {
it.each([false, true])(
'retains the original startup error and cleans up (persistence failure: %s)',
async (failFavorites) => {
const file = resolve(__dirname, `${entrypoint}.js`);
const events = [];
const errors = [];
const tasks = new Map();
const bootError = new Error('boot failure');
const noop = new Proxy(
function () {
return noop;
},
{
get(_target, key) {
return key === 'then' ? undefined : noop;
},
},
);
const closeFavorites = async () => {
events.push('close');
if (failFavorites) {
throw new Error('close failure');
}
};
const api = new Proxy(
{
cacheConfig: {},
isEnabled: () => false,
waitForKeyvRedisClient: async () => {},
createMetrics: () => ({}),
registerShutdownTask: (name, fn) => tasks.set(name, fn),
},
{
get(target, key) {
return Reflect.has(target, key) ? Reflect.get(target, key) : noop;
},
},
);
const models = new Proxy(
{
seedDatabase: async () => {},
sweepOrphanedPreviews: async () => {},
initializeFavorites: async (config) => {
events.push('initialize');
expect(config.persistence.favorites).toBe('postgres');
if (failFavorites) {
throw bootError;
}
},
closeFavorites,
},
{
get(target, key) {
return Reflect.has(target, key) ? Reflect.get(target, key) : noop;
},
},
);
runInNewContext(
readFileSync(file, 'utf8'),
{
require: (id) => {
if (id === 'path') {
return { resolve, dirname };
}
if (id === 'cluster') {
return { isMaster: false };
}
if (id === 'express') {
return () => ({ locals: {}, disable() {}, set() {}, use() {} });
}
if (id === '@librechat/api') {
return api;
}
if (id === '@librechat/data-schemas') {
return {
logger: { info() {}, warn() {}, error: (...args) => errors.push(args) },
runAsSystem: async (fn) => fn(),
};
}
if (id === '~/models') {
return models;
}
if (id === '~/db') {
return { connectDb: async () => {}, indexSync: async () => {} };
}
if (id === './services/Config') {
return {
loadCustomConfig: async (print, options) => {
expect(print).toBe(false);
expect(options).toEqual({ storageSelection: true });
return { persistence: { favorites: 'postgres' } };
},
getAppConfig: async () => {
events.push('config');
throw bootError;
},
};
}
return noop;
},
__dirname: dirname(file),
module: { exports: {} },
console,
process: {
env: {},
pid: 1,
on() {},
exit: (code) => events.push(`exit:${code}`),
},
setTimeout,
clearTimeout,
setInterval,
clearInterval,
Buffer,
},
{ filename: file },
);
await new Promise((resolve) => setImmediate(resolve));
expect(events[0]).toBe('initialize');
expect(events).toContain('close');
expect(events).toContain('exit:1');
const last = errors[errors.length - 1];
expect(last[last.length - 1]).toBe(bootError);
if (failFavorites) {
expect(events).not.toContain('config');
} else {
expect(tasks.get('favorites persistence')).toBe(closeFavorites);
expect(events).toContain('config');
}
},
);
});

View file

@ -60,6 +60,7 @@ const handlers = createAdminUsersHandlers({
findUsers: db.findUsers,
countUsers: db.countUsers,
beginAgentTriggerUserDeletion: db.beginAgentTriggerUserDeletion,
isAgentTriggerUserDeletionOwner: db.isAgentTriggerUserDeletionOwner,
cancelAgentTriggerUserDeletion: db.cancelAgentTriggerUserDeletion,
drainAgentTriggerDeliveriesForUser,
prepareAgentTriggerUserPurge,

View file

@ -0,0 +1,259 @@
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const { randomUUID } = require('node:crypto');
const mongoose = require('mongoose');
const { MongoMemoryServer } = require('mongodb-memory-server');
const schemas = require('@librechat/data-schemas');
const api = require('@librechat/api');
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
realPostgres('delete-user CLI durable erasure recovery', () => {
const schema = `cli_erasure_${randomUUID().replace(/-/g, '')}`;
const config = {
persistence: { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } },
};
const options = {
connectionString: process.env.PG_TEST_URL ?? '',
max: 2,
maxWaiting: 10,
acquireTimeoutMs: 1000,
queryTimeoutMs: 5000,
idleTimeoutMs: 1000,
};
let mongo;
let admin;
let db;
const source = fs
.readFileSync(path.resolve(__dirname, '../delete-user.js'), 'utf8')
.replace(/^#!.*\n/, '');
const execute = vm.runInThisContext(
`(function(require, __dirname, process, console, Date) { ${source}\n})`,
{ filename: 'delete-user-recovery-cli.js' },
);
beforeAll(async () => {
mongo = await MongoMemoryServer.create();
schemas.createModels(mongoose);
await mongoose.connect(mongo.getUri());
admin = schemas.createPostgresDatabase(options);
await admin.query(`CREATE SCHEMA ${schema}`);
const url = new URL(options.connectionString);
url.searchParams.set('options', `-c search_path=${schema}`);
options.connectionString = url.toString();
db = schemas.createPostgresDatabase(options);
await schemas.migratePostgres(db);
await schemas.importFavorites(db, async function* () {}, {
offline: true,
verifiedEmpty: true,
batchSize: 10,
});
});
afterEach(async () => {
await db.query('DROP TRIGGER IF EXISTS reject_cancel ON tool_favorite_erasure');
await db.query('DROP FUNCTION IF EXISTS reject_cancel()');
});
afterAll(async () => {
await db.close();
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
await admin.close();
await mongoose.disconnect();
await mongo.stop();
});
async function runCli(
email,
now,
{ failCascade = true, failAdopt = false, failCancel = false } = {},
) {
const runtime = api.createFavoriteRuntime(
schemas.createToolFavoriteMethods(mongoose),
{
PG_TEST_URL: options.connectionString,
},
schemas.createPostgresFavoriteBackend,
);
const methods = schemas.createMethods(mongoose, { toolFavorites: runtime.methods });
const suspend = jest.spyOn(methods, 'suspendUserSchedulesForDeletion');
if (failAdopt)
jest
.spyOn(methods, 'getAgentTriggerUserDeletionAttemptId')
.mockRejectedValueOnce(new Error('identity read unavailable'));
const disconnect = jest.spyOn(mongoose, 'disconnect').mockResolvedValue(undefined);
class Clock extends Date {
constructor(value) {
super(value === undefined ? now : value);
}
}
const quiet = Object.assign(
Object.create(console),
Object.fromEntries(['purple', 'green', 'yellow', 'error'].map((name) => [name, jest.fn()])),
);
const code = await new Promise((resolve) => {
const modules = {
mongoose,
'module-alias': () => undefined,
'@librechat/data-schemas': schemas,
'@librechat/api': {
...api,
createStreamServices: () => ({ isRedis: false }),
waitForKeyvRedisClient: async () => undefined,
initializeFileStorage: () => undefined,
openCheckpointDeletion: async () => ({
remember: async () => undefined,
cleanup: async () => undefined,
acknowledge: async () => undefined,
}),
deleteAllSharedLinksWithCleanup: async () => {
if (failCascade) throw new Error('shared links unavailable');
},
revokeUserCodeEnvironmentWorkers: async () => 0,
},
'~/models': {
...methods,
initializeFavorites: async (value) => {
await runtime.initialize(value.persistence);
if (failCancel) await installFault();
},
closeFavorites: async () => {
if (failCancel) await removeFault();
await runtime.close();
},
},
'~/config': {},
'~/cache/getLogStores': () => undefined,
'~/server/services/Config': { getAppConfig: async () => ({}) },
'~/server/services/Config/loadCustomConfig': async () => config,
'~/server/services/Files/process': {
processDeleteRequest: async () => ({ deletedFileIds: [], failedFileIds: [] }),
},
'~/server/services/Config/getCachedTools': { invalidateCachedTools: async () => undefined },
'./helpers': { askQuestion: async () => 'y', silentExit: resolve },
'./connect': async () => undefined,
};
execute(
(name) => (Object.hasOwn(modules, name) ? modules[name] : require(name)),
path.resolve(__dirname, '..'),
{ argv: ['node', 'delete-user.js', email] },
quiet,
Clock,
);
});
disconnect.mockRestore();
return { code, suspend };
}
async function owned(id) {
return mongoose.models.User.findById(id)
.select('+agentTriggerDeletionStartedAt +agentTriggerDeletionAttemptId')
.lean();
}
async function installFault() {
await db.query(
`CREATE FUNCTION reject_cancel() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'cancel unavailable'; END $$`,
);
await db.query(
'CREATE TRIGGER reject_cancel BEFORE DELETE ON tool_favorite_erasure FOR EACH ROW EXECUTE FUNCTION reject_cancel()',
);
}
async function removeFault() {
await db.query('DROP TRIGGER reject_cancel ON tool_favorite_erasure');
await db.query('DROP FUNCTION reject_cancel()');
}
it.each([
{ outcome: 'cancel', origin: 'system' },
{ outcome: 'commit', origin: 'system' },
{ outcome: 'cancel', origin: 'tenant' },
{ outcome: 'commit', origin: 'tenant' },
])(
'adopts the canonical UUID from $origin through real offline CLI recovery and then $outcome',
async ({ outcome, origin }) => {
const id = new mongoose.Types.ObjectId();
const email = `${id}@example.com`;
await mongoose.models.User.create({
_id: id,
email,
provider: 'local',
...(origin === 'tenant' ? { tenantId: 'tenant-a' } : {}),
});
const time = Date.now();
const stale = 15 * 60_000 + 1;
const priorToken = randomUUID();
if (origin === 'tenant') {
const priorMethods = schemas.createMethods(mongoose);
await schemas.runAsSystem(() =>
priorMethods.beginAgentTriggerUserDeletion(
id.toString(),
new Date(time - stale),
priorToken,
),
);
await schemas.tenantStorage.run({ tenantId: 'tenant-a' }, () =>
schemas
.createPostgresFavoriteMethods(db)
.deleteUserToolFavorites(id.toString(), priorToken),
);
}
const first = await runCli(email, time, { failCancel: true });
expect(first.code).toBe(1);
const original = await owned(id);
const token = original.agentTriggerDeletionAttemptId;
expect(token).toMatch(/^[0-9a-f-]{36}$/);
if (origin === 'tenant') expect(token).toBe(priorToken);
// A takeover that loses its identity read must retain the adopted identity and owner.
if (outcome === 'cancel') {
expect((await runCli(email, time + stale, { failAdopt: true })).code).toBe(1);
expect((await owned(id)).agentTriggerDeletionAttemptId).toBe(token);
expect((await runCli(email, time + 2 * stale, { failCancel: true })).code).toBe(1);
expect((await owned(id)).agentTriggerDeletionAttemptId).toBe(token);
}
const result = await runCli(email, time + 3 * stale, { failCascade: outcome === 'cancel' });
expect(result.code).toBe(outcome === 'cancel' ? 1 : 0);
expect(result.suspend.mock.calls[0][1]).not.toBe(first.suspend.mock.calls[0][1]);
expect(result.suspend.mock.calls[0][1]).not.toBe(token);
const methods = schemas.createMethods(mongoose);
expect(
await methods.cancelAgentTriggerUserDeletion(
id.toString(),
original.agentTriggerDeletionStartedAt,
token,
),
).toBe(false);
const restarted = api.createFavoriteRuntime(
methods,
{
PG_TEST_URL: options.connectionString,
},
schemas.createPostgresFavoriteBackend,
);
try {
await restarted.initialize(config.persistence);
const addFavorite = () =>
restarted.methods.addToolFavorite({
userId: id.toString(),
itemType: 'tool',
itemId: 'after-recovery',
});
const add =
origin === 'tenant'
? schemas.tenantStorage.run({ tenantId: 'tenant-a' }, addFavorite)
: addFavorite();
if (outcome === 'cancel') {
expect((await owned(id)).agentTriggerDeletionStartedAt).toBeUndefined();
expect((await owned(id)).agentTriggerDeletionAttemptId).toBeUndefined();
await expect(add).resolves.toEqual({ ok: true, added: true });
expect(
(
await db.query(
'SELECT scope FROM tool_favorite_erasure WHERE user_id=$1 AND attempt_id=$2',
[id.toString(), token],
)
).rows,
).toEqual([]);
} else {
expect(await owned(id)).toBeNull();
await expect(add).rejects.toThrow('erasure');
}
} finally {
await restarted.close();
}
},
);
});

View file

@ -25,6 +25,7 @@ const mockMethods = {
deleteUserById: jest.fn(),
deleteAgentTriggerDeliveriesByUser: jest.fn(),
cancelAgentTriggerUserPurge: jest.fn(),
isAgentTriggerUserDeletionOwner: jest.fn(),
cancelAgentTriggerUserDeletion: jest.fn(),
deleteUserPrompts: jest.fn(),
};
@ -34,6 +35,7 @@ const mockDestroy = jest.fn();
const mockSilentExit = jest.fn();
const mockAskQuestion = jest.fn();
const mockGetAppConfig = jest.fn();
const mockLoadCustomConfig = jest.fn();
const mockRevokeUserCodeEnvironmentWorkers = jest.fn();
const mockEraseUserData = jest.fn();
const mockProcessDeleteRequest = jest.fn();
@ -41,16 +43,25 @@ const mockInvalidateCachedTools = jest.fn();
const mockInitializeFileStorage = jest.fn();
const mockOpenCheckpointDeletion = jest.fn();
const mockDeleteAllSharedLinksWithCleanup = jest.fn();
const mockMongoFavorites = {
getToolFavorites: jest.fn(),
addToolFavorite: jest.fn(),
removeToolFavorite: jest.fn(),
deleteUserToolFavorites: jest.fn(),
cancelUserToolFavoriteDeletion: jest.fn(),
};
jest.mock('../connect', () => jest.fn(async () => undefined));
jest.mock('mongoose', () => ({ disconnect: jest.fn(async () => undefined) }));
jest.mock('@librechat/data-schemas', () => ({
createModels: () => new Proxy({}, { get: (_target, prop) => mockModelFor(prop) }),
createMethods: () => mockMethods,
createMethods: (_mongoose, deps) => ({ ...mockMethods, ...deps.toolFavorites }),
createToolFavoriteMethods: () => mockMongoFavorites,
runAsSystem: (operation) => operation(),
tenantStorage: mockTenantStorage,
}));
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
waitForKeyvRedisClient: jest.fn(async () => undefined),
createStreamServices: jest.fn(() => ({ isRedis: true })),
GenerationJobManager: {
@ -67,6 +78,7 @@ jest.mock('@librechat/api', () => ({
deleteAllSharedLinksWithCleanup: mockDeleteAllSharedLinksWithCleanup,
}));
jest.mock('~/cache/getLogStores', () => jest.fn());
jest.mock('~/server/services/Config/loadCustomConfig', () => mockLoadCustomConfig);
jest.mock('~/server/services/Config', () => ({
getAppConfig: (...args) => mockGetAppConfig(...args),
}));
@ -129,12 +141,16 @@ describe('Delete user CLI', () => {
mockMethods.deleteUserById.mockReset().mockResolvedValue({ deletedCount: 1 });
mockMethods.deleteAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(undefined);
mockMethods.cancelAgentTriggerUserPurge.mockReset().mockResolvedValue(true);
mockMethods.isAgentTriggerUserDeletionOwner.mockReset().mockResolvedValue(true);
mockMethods.cancelAgentTriggerUserDeletion.mockReset().mockResolvedValue(true);
mockMethods.deleteUserPrompts.mockReset().mockResolvedValue(undefined);
mockGetCleanupBlockingJobIdsForUser.mockReset().mockResolvedValue([]);
mockAbortJob.mockReset().mockResolvedValue({ success: true });
mockDestroy.mockReset().mockResolvedValue(undefined);
mockGetAppConfig.mockReset().mockResolvedValue({});
mockLoadCustomConfig.mockReset().mockResolvedValue({});
mockMongoFavorites.deleteUserToolFavorites.mockReset();
mockMongoFavorites.cancelUserToolFavoriteDeletion.mockReset().mockResolvedValue(undefined);
mockRevokeUserCodeEnvironmentWorkers.mockReset().mockResolvedValue(0);
mockEraseUserData.mockReset().mockResolvedValue(undefined);
mockProcessDeleteRequest.mockReset().mockResolvedValue({ deletedFileIds: [] });
@ -149,12 +165,38 @@ describe('Delete user CLI', () => {
jest.restoreAllMocks();
});
it('initializes the injected favorites methods and closes captured references on exit', async () => {
let deleteFavorites;
mockMongoFavorites.deleteUserToolFavorites.mockResolvedValueOnce(3);
mockEraseUserData.mockImplementationOnce(async (_params, deps) => {
deleteFavorites = deps.deleteUserToolFavorites;
expect(await deleteFavorites(USER_ID)).toBe(3);
});
expect(await runCli()).toBe(0);
expect(mockLoadCustomConfig).toHaveBeenCalledWith(false, { storageSelection: true });
expect(mockMongoFavorites.deleteUserToolFavorites).toHaveBeenCalledWith(USER_ID);
await expect(deleteFavorites(USER_ID)).rejects.toThrow('not initialized');
});
it('deletes nothing when favorites persistence configuration is invalid', async () => {
mockLoadCustomConfig.mockResolvedValueOnce({ persistence: { favorites: 'invalid' } });
expect(await runCli()).toBe(1);
expect(mockMethods.beginAgentTriggerUserDeletion).not.toHaveBeenCalled();
expect(mockEraseUserData).not.toHaveBeenCalled();
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
});
it('fences triggers and schedules before inspecting active work', async () => {
expect(await runCli()).toBe(0);
expect(mockMethods.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
USER_ID,
expect.any(Date),
expect.any(String),
);
expect(mockMethods.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
USER_ID,
@ -206,7 +248,11 @@ describe('Delete user CLI', () => {
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
USER_ID,
deletionFence,
expect.any(String),
);
});
/**
@ -229,6 +275,34 @@ describe('Delete user CLI', () => {
);
});
it.each([false, 'error'])(
'keeps favorites fenced when Mongo ownership is not proven: %s',
async (result) => {
mockEraseUserData.mockRejectedValueOnce(new Error('shared link failure'));
if (result === 'error')
mockMethods.isAgentTriggerUserDeletionOwner.mockRejectedValueOnce(
new Error('Mongo unavailable'),
);
else mockMethods.isAgentTriggerUserDeletionOwner.mockResolvedValueOnce(false);
expect(await runCli()).toBe(1);
expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
},
);
it('releases only the cascade UUID before releasing Mongo ownership', async () => {
mockEraseUserData.mockRejectedValueOnce(new Error('shared link failure'));
expect(await runCli()).toBe(1);
const token = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2];
expect(mockEraseUserData.mock.calls[0][0].deletionAttemptId).toBe(token);
expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(USER_ID, token);
expect(mockMethods.isAgentTriggerUserDeletionOwner.mock.invocationCallOrder[0]).toBeLessThan(
mockMongoFavorites.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0],
);
expect(
mockMongoFavorites.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0],
).toBeLessThan(mockMethods.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0]);
});
it('never restores schedules after a committed deletion', async () => {
expect(await runCli()).toBe(0);
@ -240,6 +314,13 @@ describe('Delete user CLI', () => {
expect(await runCli()).toBe(0);
expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true });
expect(mockMethods.deleteUserById).toHaveBeenCalledWith(USER_ID, {
startedAt: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1],
attemptId: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2],
});
expect(mockMethods.deleteUserById.mock.calls[0][1].attemptId).not.toBe(
mockMethods.suspendUserSchedulesForDeletion.mock.calls[0][1],
);
expect(mockRevokeUserCodeEnvironmentWorkers).toHaveBeenCalledWith(
expect.objectContaining({ userId: USER_ID }),
);
@ -279,6 +360,7 @@ describe('Delete user CLI', () => {
tenantId: 'tenant-1',
checkpointer: { ttl: 60 },
keepTransactions: true,
deletionAttemptId: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2],
});
expect(deps).toEqual(
expect.objectContaining({
@ -337,6 +419,8 @@ describe('Delete user CLI', () => {
});
it('supplies shared-user MCP OAuth cleanup to the cascade', async () => {
mockMethods.findMCPServerByObjectId = jest.fn(async () => ({ config: { type: 'sse' } }));
mockMethods.getUserById = jest.fn(async () => ({ role: 'VIEWER' }));
expect(await runCli()).toBe(0);
const [, deps] = mockEraseUserData.mock.calls[0];
@ -347,10 +431,8 @@ describe('Delete user CLI', () => {
invalidateUserTools: expect.any(Function),
uninstallOAuthMCP: expect.any(Function),
});
mockMethods.findMCPServerByObjectId = jest.fn(async () => ({ config: { type: 'sse' } }));
expect(await deps.mcpOAuth.getServerConfig('server-1', 'server-id-1')).toEqual({ type: 'sse' });
expect(mockMethods.findMCPServerByObjectId).toHaveBeenCalledWith('server-id-1');
mockMethods.getUserById = jest.fn(async () => ({ role: 'VIEWER' }));
mockGetAppConfig.mockImplementation(async (options) =>
options?.userId === 'viewer-1' ? { mcpSettings: { allowedDomains: ['viewer.example'] } } : {},
);
@ -380,6 +462,19 @@ describe('Delete user CLI', () => {
);
});
it.each(['unknown', 'absent'])(
'keeps both erasure fences after final delete starts: %s',
async (outcome) => {
if (outcome === 'unknown')
mockMethods.deleteUserById.mockRejectedValueOnce(new Error('delete acknowledgement lost'));
else mockMethods.deleteUserById.mockResolvedValueOnce({ deletedCount: 0 });
expect(await runCli()).toBe(1);
expect(mockMethods.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled();
expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
expect(mockMethods.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
},
);
it('keeps the user and releases both fences when the cascade fails', async () => {
mockEraseUserData.mockRejectedValueOnce(new Error('storage unavailable'));
@ -388,7 +483,11 @@ describe('Delete user CLI', () => {
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
expect(mockMethods.restoreUserSchedulesFromDeletion).toHaveBeenCalled();
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
USER_ID,
deletionFence,
expect.any(String),
);
});
it('passes deleteUserPrompts through to the shared cascade, which keeps a synced group with a synthetic author untouched', async () => {
@ -409,6 +508,10 @@ describe('Delete user CLI', () => {
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
USER_ID,
deletionFence,
expect.any(String),
);
});
});

View file

@ -1,5 +1,6 @@
jest.mock('../connect', () => jest.fn().mockResolvedValue(true));
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
ensureRequiredCollectionsExist: jest.fn().mockResolvedValue(undefined),
createChannelStorageConfigReader: jest.fn(() => ({
initialize: jest.fn(),

View file

@ -4,16 +4,12 @@
const path = require('path');
const mongoose = require('mongoose');
const { randomUUID } = require('node:crypto');
const {
createModels,
createMethods,
runAsSystem,
tenantStorage,
} = require('@librechat/data-schemas');
const { createModels, runAsSystem, tenantStorage } = require('@librechat/data-schemas');
const { User } = createModels(mongoose);
require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });
const {
eraseUserData,
createUserDeletionAttempt,
GenerationJobManager,
createStreamServices,
initializeFileStorage,
@ -36,13 +32,16 @@ const connect = require('./connect');
const TRIGGER_DRAIN_TIMEOUT_MS = 35_000;
const TRIGGER_DRAIN_POLL_MS = 100;
const methods = createMethods(mongoose, { getCache: getLogStores });
const { initializeFavorites, closeFavorites, ...methods } = require('~/models');
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function gracefulExit(code = 0) {
try {
await mongoose.disconnect();
const cleanup = await Promise.allSettled([closeFavorites(), mongoose.disconnect()]);
if (cleanup.some((result) => result.status === 'rejected')) {
throw new Error('Persistence cleanup failed');
}
} catch (err) {
console.error('Error disconnecting from MongoDB:', err);
}
@ -51,6 +50,9 @@ async function gracefulExit(code = 0) {
(async () => {
await connect();
await initializeFavorites(
await require('~/server/services/Config/loadCustomConfig')(false, { storageSelection: true }),
);
console.purple('---------------');
console.purple('Deleting a user and all related data');
@ -115,11 +117,12 @@ async function gracefulExit(code = 0) {
let deletionFence;
let scheduleSuspensionToken;
let userDeleted = false;
const deletionAttempt = createUserDeletionAttempt();
try {
deletionFence = new Date();
let fenceState = await runAsSystem(() =>
methods.beginAgentTriggerUserDeletion(uid, deletionFence),
methods.beginAgentTriggerUserDeletion(uid, deletionFence, deletionAttempt.id),
);
if (fenceState === 'in_progress') {
deletionFence = undefined;
@ -135,7 +138,7 @@ async function gracefulExit(code = 0) {
const recoveredAt = new Date();
fenceState = await runAsSystem(() =>
methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt),
methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt, deletionAttempt.id),
);
if (fenceState !== 'acquired') {
throw new Error(
@ -145,6 +148,7 @@ async function gracefulExit(code = 0) {
);
}
deletionFence = recoveredAt;
await runAsSystem(() => deletionAttempt.adopt(uid, recoveredAt, methods));
}
if (fenceState === 'missing') {
deletionFence = undefined;
@ -214,6 +218,7 @@ async function gracefulExit(code = 0) {
eraseUserData(
{
userId: uid,
deletionAttemptId: deletionAttempt.id,
tenantId: user.tenantId,
checkpointer: userAppConfig?.endpoints?.agents?.checkpointer,
keepTransactions: !deleteTx,
@ -283,7 +288,14 @@ async function gracefulExit(code = 0) {
);
// 6) Finally delete the user document itself
const deletedUser = await runAsSystem(() => methods.deleteUserById(uid));
const deletedUser = await deletionAttempt.commit(() =>
runAsSystem(() =>
methods.deleteUserById(uid, {
startedAt: deletionFence,
attemptId: deletionAttempt.id,
}),
),
);
if (deletedUser.deletedCount !== 1) {
throw new Error('User disappeared before account deletion could commit');
}
@ -326,9 +338,12 @@ async function gracefulExit(code = 0) {
await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch(
(error) => console.error('Failed to disarm trigger purge recovery:', error),
);
await runAsSystem(() => methods.cancelAgentTriggerUserDeletion(uid, deletionFence)).catch(
(error) => console.error('Failed to release account-deletion fence:', error),
);
await runAsSystem(() =>
deletionAttempt.cancel(
{ userId: uid, startedAt: deletionFence, deletionAttemptId: deletionAttempt.id },
methods,
),
).catch((error) => console.error('Failed to release account-deletion fence:', error));
}
if (hasSharedGenerationStore) {
await GenerationJobManager.destroy().catch((error) =>
@ -343,11 +358,7 @@ async function gracefulExit(code = 0) {
}
return gracefulExit(0);
})().catch(async (err) => {
if (!err.message.includes('fetch failed')) {
console.error('There was an uncaught error:');
console.error(err);
await mongoose.disconnect();
process.exit(1);
}
})().catch(async () => {
console.error('Account deletion failed; retry before considering the account erased.');
await gracefulExit(1);
});

21
config/postgres.js Normal file
View file

@ -0,0 +1,21 @@
#!/usr/bin/env node
require('dotenv').config();
require('module-alias/register');
const path = require('node:path');
require('module-alias').addAlias('~', path.resolve(__dirname, '../api'));
const { runPostgresCommand } = require('@librechat/api');
const mongoose = require('mongoose');
const { createMongoFavoriteStore, createPostgresDatabase } = require('@librechat/data-schemas');
const loadCustomConfig = require('~/server/services/Config/loadCustomConfig');
runPostgresCommand({
favorites: createMongoFavoriteStore(mongoose),
createDatabase: createPostgresDatabase,
connectDb: () => mongoose.connect(process.env.MONGO_URI, { autoCreate: false, autoIndex: false }),
disconnectDb: () => mongoose.disconnect(),
loadCustomConfig,
args: process.argv.slice(2),
env: process.env,
}).catch(() => {
console.error('PostgreSQL operation failed. No automatic storage cutover was performed.');
process.exitCode = 1;
});

154
package-lock.json generated
View file

@ -19958,6 +19958,18 @@
"@types/passport": "*"
}
},
"node_modules/@types/pg": {
"version": "8.23.1",
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz",
"integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"pg-protocol": "*",
"pg-types": "^2.2.0"
}
},
"node_modules/@types/prop-types": {
"version": "15.7.11",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
@ -35414,6 +35426,95 @@
"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
"license": "MIT"
},
"node_modules/pg": {
"version": "8.23.1",
"resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz",
"integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==",
"license": "MIT",
"dependencies": {
"pg-connection-string": "^2.14.1",
"pg-pool": "^3.14.0",
"pg-protocol": "^1.16.1",
"pg-types": "2.2.0",
"pgpass": "1.0.5"
},
"engines": {
"node": ">= 16.0.0"
},
"optionalDependencies": {
"pg-cloudflare": "^1.4.1"
},
"peerDependencies": {
"pg-native": ">=3.0.1"
},
"peerDependenciesMeta": {
"pg-native": {
"optional": true
}
}
},
"node_modules/pg-cloudflare": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz",
"integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==",
"license": "MIT",
"optional": true
},
"node_modules/pg-connection-string": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz",
"integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==",
"license": "MIT"
},
"node_modules/pg-int8": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
"license": "ISC",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/pg-pool": {
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
"license": "MIT",
"peerDependencies": {
"pg": ">=8.0"
}
},
"node_modules/pg-protocol": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz",
"integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==",
"license": "MIT"
},
"node_modules/pg-types": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
"license": "MIT",
"dependencies": {
"pg-int8": "1.0.1",
"postgres-array": "~2.0.0",
"postgres-bytea": "~1.0.0",
"postgres-date": "~1.0.4",
"postgres-interval": "^1.1.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/pgpass": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
"license": "MIT",
"dependencies": {
"split2": "^4.1.0"
}
},
"node_modules/picocolors": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
@ -35639,6 +35740,45 @@
"node": "^10 || ^12 || >=14"
}
},
"node_modules/postgres-array": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/postgres-bytea": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-date": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/postgres-interval": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
"license": "MIT",
"dependencies": {
"xtend": "^4.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/powershell-utils": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.2.1.tgz",
@ -38744,6 +38884,15 @@
"node": ">=8.0"
}
},
"node_modules/split2": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
"license": "ISC",
"engines": {
"node": ">= 10.x"
}
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@ -41977,7 +42126,6 @@
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.4"
@ -45298,13 +45446,15 @@
"micromark-extension-directive": "^3.0.1",
"micromark-extension-gfm": "^3.0.0",
"micromark-extension-llm-math": "^3.1.0",
"micromark-util-decode-string": "^2.0.0"
"micromark-util-decode-string": "^2.0.0",
"pg": "^8.23.1"
},
"devDependencies": {
"@stryker-mutator/core": "10.1.1",
"@stryker-mutator/jest-runner": "10.1.1",
"@types/jest": "^29.5.2",
"@types/node": "^24.12.4",
"@types/pg": "^8.23.1",
"jest": "^30.2.0",
"jest-junit": "^17.0.0",
"mongodb-memory-server": "^11.3.0",

View file

@ -166,7 +166,8 @@
"migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50",
"lighthouse": "npm run e2e:prepare && npm run lighthouse:run",
"lighthouse:run": "playwright test --config=e2e/playwright.config.lighthouse.ts",
"lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run"
"lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run",
"postgres": "node config/postgres.js"
},
"repository": {
"type": "git",

View file

@ -9,6 +9,7 @@ import { createAdminUsersHandlers } from './users';
import { eraseUserData } from '~/user/erasure';
jest.mock('~/user/erasure', () => ({
...jest.requireActual('~/user/erasure'),
eraseUserData: jest.fn().mockResolvedValue(undefined),
}));
@ -60,6 +61,7 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {
findUsers: jest.fn().mockResolvedValue([]),
countUsers: jest.fn().mockResolvedValue(0),
beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true),
cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),
drainAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined),
prepareAgentTriggerUserPurge: jest.fn().mockResolvedValue(undefined),
@ -392,6 +394,7 @@ describe('createAdminUsersHandlers', () => {
expect(deps.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
validUserId,
expect.any(Date),
expect.any(String),
);
expect(deps.drainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId);
expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
@ -461,7 +464,10 @@ describe('createAdminUsersHandlers', () => {
await handlers.deleteUser(req, res);
expect(status).toHaveBeenCalledWith(200);
expect(deps.deleteUserById).toHaveBeenCalledWith(targetId);
expect(deps.deleteUserById).toHaveBeenCalledWith(targetId, {
startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1],
attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2],
});
});
it('does not check admin count when target is a regular user', async () => {
@ -559,7 +565,12 @@ describe('createAdminUsersHandlers', () => {
}),
);
expect(eraseUserData).toHaveBeenCalledWith(
{ userId: validUserId, tenantId: 'tenant-1', checkpointer },
{
userId: validUserId,
tenantId: 'tenant-1',
checkpointer,
deletionAttemptId: expect.any(String),
},
erasureDeps,
);
expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan(
@ -582,6 +593,34 @@ describe('createAdminUsersHandlers', () => {
expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled();
});
it.each([true, false])(
'releases the resolved favorite UUID only after a proven Mongo owner: %s',
async (owned) => {
const cancelUserToolFavoriteDeletion = jest.fn().mockResolvedValue(undefined);
jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('later cascade failure'));
const deps = createDeps({
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(owned),
resolveErasure: jest.fn().mockResolvedValue({ deps: { cancelUserToolFavoriteDeletion } }),
});
const { req, res } = createReqRes({ params: { id: validUserId } });
await createAdminUsersHandlers(deps).deleteUser(req, res);
const token = jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2];
expect(eraseUserData).toHaveBeenCalledWith(
expect.objectContaining({ deletionAttemptId: token }),
expect.anything(),
);
if (owned) {
expect(cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(validUserId, token);
expect(
jest.mocked(deps.isAgentTriggerUserDeletionOwner).mock.invocationCallOrder[0],
).toBeLessThan(cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]);
expect(cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]).toBeLessThan(
jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0],
);
} else expect(cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
},
);
it('suspends schedules before erasing and restores them when the deletion fails', async () => {
jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));
const deps = createDeps();
@ -677,7 +716,10 @@ describe('createAdminUsersHandlers', () => {
await handlers.deleteUser(req, res);
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId);
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId, {
startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1],
attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2],
});
expect(deps.deleteUserCodeEnvironments).not.toHaveBeenCalled();
});
@ -694,10 +736,7 @@ describe('createAdminUsersHandlers', () => {
expect(deps.deleteUserCodeEnvironments).not.toHaveBeenCalled();
expect(deps.deleteAclEntries).not.toHaveBeenCalled();
expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
validUserId,
expect.any(Date),
);
expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));
});
@ -735,10 +774,8 @@ describe('createAdminUsersHandlers', () => {
expect(status).toHaveBeenCalledWith(500);
expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
validUserId,
expect.any(Date),
);
expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
expect(deps.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled();
expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));
expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
});
@ -815,7 +852,10 @@ describe('createAdminUsersHandlers', () => {
expect(status).toHaveBeenCalledWith(500);
expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId);
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId, {
startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1],
attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2],
});
expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalled();
expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled();
expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();

View file

@ -7,7 +7,7 @@ import type {
ConfigRecord,
AdminUserListItem,
AdminUserSearchResult,
UserDeleteResult,
AllMethods,
} from '@librechat/data-schemas';
import type { TCheckpointerConfig } from 'librechat-data-provider';
import type { FilterQuery } from 'mongoose';
@ -15,10 +15,10 @@ import type { Response } from 'express';
import type { UserGenerationControl } from '~/user/generations';
import type { UserErasureDeps } from '~/user/erasure';
import type { ServerRequest } from '~/types/http';
import { eraseUserData, createUserDeletionAttempt } from '~/user/erasure';
import { stopUserGenerations } from '~/user/generations';
import { invalidatingConfigStore } from './cascade';
import { parsePagination } from './pagination';
import { eraseUserData } from '~/user/erasure';
const MAX_SEARCH_LENGTH = 200;
@ -75,8 +75,18 @@ export interface AdminUsersDeps {
beginAgentTriggerUserDeletion: (
userId: string,
startedAt: Date,
attemptId?: string,
) => Promise<'acquired' | 'in_progress' | 'missing'>;
cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;
isAgentTriggerUserDeletionOwner: (
userId: string,
startedAt: Date,
attemptId?: string,
) => Promise<boolean>;
cancelAgentTriggerUserDeletion: (
userId: string,
startedAt: Date,
attemptId?: string,
) => Promise<boolean>;
drainAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;
prepareAgentTriggerUserPurge: (
userId: string,
@ -102,7 +112,7 @@ export interface AdminUsersDeps {
*/
resolveErasure: (target: AdminErasureTarget) => Promise<AdminErasureContext>;
/** Thin data-layer delete: removes the User document after the erasure cascade has run. */
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
deleteUserById: AllMethods['deleteUserById'];
deleteUserCodeEnvironments: (userId: string | Types.ObjectId) => Promise<number>;
invalidateCodeEnvironmentConfigCache: (tenantId?: string) => Promise<void>;
deleteConfig: (
@ -132,6 +142,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
findUsers,
countUsers,
beginAgentTriggerUserDeletion,
isAgentTriggerUserDeletionOwner,
cancelAgentTriggerUserDeletion,
drainAgentTriggerDeliveriesForUser,
prepareAgentTriggerUserPurge,
@ -232,6 +243,8 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
let triggerDeletionFence: Date | undefined;
let scheduleSuspensionToken: string | undefined;
let userDeleted = false;
const deletionAttempt = createUserDeletionAttempt();
let erasureDeps: UserErasureDeps | undefined;
try {
const { id } = req.params as { id: string };
@ -255,7 +268,11 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
}
triggerDeletionFence = new Date();
const fenceState = await beginAgentTriggerUserDeletion(id, triggerDeletionFence);
const fenceState = await beginAgentTriggerUserDeletion(
id,
triggerDeletionFence,
deletionAttempt.id,
);
if (fenceState === 'in_progress') {
triggerDeletionFence = undefined;
return res.status(409).json({ error: 'User deletion is already in progress' });
@ -274,22 +291,40 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
}
await stopUserGenerations(id, targetUser?.tenantId, generationJobs);
const { deps: erasureDeps, checkpointer } = await resolveErasure({
const erasure = await resolveErasure({
id,
role: targetUser?.role,
tenantId: targetUser?.tenantId,
idOnTheSource: targetUser?.idOnTheSource ?? null,
profile: targetUser,
});
erasureDeps = erasure.deps;
await eraseUserData(
{ userId: id, tenantId: targetUser?.tenantId, checkpointer },
{
userId: id,
tenantId: targetUser?.tenantId,
checkpointer: erasure.checkpointer,
deletionAttemptId: deletionAttempt.id,
},
erasureDeps,
);
const result = await deleteUserById(id);
const deletionFence = { startedAt: triggerDeletionFence, attemptId: deletionAttempt.id };
const result = await deletionAttempt.commit(() => deleteUserById(id, deletionFence));
if (result.deletedCount === 0) {
await cancelAgentTriggerUserPurge(id, triggerDeletionFence);
await cancelAgentTriggerUserDeletion(id, triggerDeletionFence);
await deletionAttempt.cancel(
{
userId: id,
startedAt: triggerDeletionFence,
deletionAttemptId: deletionAttempt.id,
},
{
isAgentTriggerUserDeletionOwner,
cancelAgentTriggerUserDeletion,
cancelUserToolFavoriteDeletion: erasureDeps?.cancelUserToolFavoriteDeletion,
},
);
triggerDeletionFence = undefined;
return res.status(404).json({ error: 'User not found' });
}
@ -353,7 +388,18 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
logger.error('[adminUsers] failed to disarm trigger purge recovery:', purgeFenceError);
}
try {
await cancelAgentTriggerUserDeletion(targetUserId, triggerDeletionFence);
await deletionAttempt.cancel(
{
userId: targetUserId,
startedAt: triggerDeletionFence,
deletionAttemptId: deletionAttempt.id,
},
{
isAgentTriggerUserDeletionOwner,
cancelAgentTriggerUserDeletion,
cancelUserToolFavoriteDeletion: erasureDeps?.cancelUserToolFavoriteDeletion,
},
);
} catch (fenceError) {
logger.error('[adminUsers] failed to release trigger deletion fence:', fenceError);
}

View file

@ -22,3 +22,7 @@ export { resolveBuildInfo } from './build';
export type { BuildInfo } from './build';
export { buildPreLoginInterface, withSafePolicyLinks } from './interface';
export * from './migrations';
export * from './persistence';
export * from './postgres';

View file

@ -4,6 +4,7 @@ import { createHash } from 'node:crypto';
import { logger } from '@librechat/data-schemas';
import {
configSchema,
persistenceConfigSchema,
EModelEndpoint,
EImageOutputType,
getMaxSubagents,
@ -21,6 +22,8 @@ export type CustomConfigLoadMode = 'startup' | 'reload';
export interface CustomConfigLoadOptions {
mode?: CustomConfigLoadMode;
/** Storage initialization must distinguish an absent default from an unreadable source. */
storageSelection?: boolean;
}
export interface LoadedCustomConfig {
@ -192,6 +195,17 @@ export function createCustomConfigLoader({
cause?: unknown,
startupLogLevel: 'error' | 'info' = 'error',
): null => {
if (
options.storageSelection &&
!(
cause instanceof Error &&
'code' in cause &&
cause.code === 'ENOENT' &&
!process.env.CONFIG_PATH
)
) {
throw new ConfigReloadError('Deployment config unavailable for storage selection');
}
if (mode === 'reload') {
if (cause == null) {
logger.error(message);
@ -261,6 +275,16 @@ export function createCustomConfigLoader({
}
/** The checks below return copies, but `modelSpecs` is assigned onto this object later. */
if (
typeof loadedConfig === 'object' &&
loadedConfig != null &&
'persistence' in loadedConfig
) {
const persistence = persistenceConfigSchema.safeParse(loadedConfig.persistence);
if (!persistence.success) {
throw new ConfigReloadError('Invalid persistence configuration');
}
}
const parsedConfig = loadedConfig;
const themeCheck = checkConfigTheme(loadedConfig);
loadedConfig = themeCheck.config;
@ -302,6 +326,13 @@ export function createCustomConfigLoader({
);
}
if (
typeof loadedConfig === 'object' &&
loadedConfig != null &&
'persistence' in loadedConfig
) {
throw new ConfigReloadError('Invalid configuration containing persistence selection');
}
if (process.env.CONFIG_BYPASS_VALIDATION === 'true') {
logger.warn(
'CONFIG_BYPASS_VALIDATION is enabled. Continuing with default configuration despite validation errors.',

View file

@ -0,0 +1,319 @@
import pg from 'pg';
import { randomBytes } from 'node:crypto';
import {
createPostgresFavoriteBackend,
createPostgresDatabase,
migratePostgres,
importFavorites,
} from '@librechat/data-schemas';
import type { ToolFavoriteMethods } from '@librechat/data-schemas';
import type { Response } from 'express';
import type { ServerRequest } from '~/types';
import { createFavoriteRuntime, getPersistenceConfig } from './persistence';
import { createToolFavoritesHandlers } from '~/favorites/handlers';
import { createCustomConfigLoader } from './loader';
const mongo: ToolFavoriteMethods = {
getToolFavorites: jest.fn().mockResolvedValue([]),
addToolFavorite: jest.fn().mockResolvedValue({ ok: true, added: true }),
removeToolFavorite: jest.fn().mockResolvedValue({ ok: true, removed: true }),
deleteUserToolFavorites: jest.fn().mockResolvedValue(2),
};
describe('favorites process lifecycle', () => {
it('retains a captured method across initialization and erases through the selected domain', async () => {
const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend);
const captured = runtime.methods.deleteUserToolFavorites;
await expect(captured('user')).rejects.toThrow('not initialized');
await runtime.initialize(undefined);
expect(await captured('user')).toBe(2);
await runtime.close();
await expect(captured('user')).rejects.toThrow('not initialized');
await expect(runtime.initialize(undefined)).rejects.toThrow('closed');
});
it('keeps the optional Mongo cancellation hook a no-op without changing captured references', async () => {
const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend);
const cancel = runtime.methods.cancelUserToolFavoriteDeletion!;
await expect(cancel('user', 'attempt')).rejects.toThrow('not initialized');
await runtime.initialize(undefined);
await expect(cancel('user', 'attempt')).resolves.toBeUndefined();
await runtime.close();
await expect(cancel('user', 'attempt')).rejects.toThrow('not initialized');
});
it('fails closed on invalid explicit storage config and missing DSN', async () => {
await expect(
createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend).initialize({
favorites: 'invalid',
}),
).rejects.toThrow('Invalid');
await expect(
createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend).initialize({
favorites: 'postgres',
postgres: { connectionStringEnv: 'PG_DSN' },
}),
).rejects.toThrow('missing');
});
it('selects once for a process lifetime, never a request or config reload', async () => {
const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend);
await runtime.initialize(undefined);
await runtime.initialize({ favorites: 'postgres' });
expect(await runtime.methods.getToolFavorites('user')).toEqual([]);
await runtime.close();
});
it('constructs only the selected injected backend and retains captured method references', async () => {
const backend = {
methods: {
...mongo,
getToolFavorites: jest.fn(async () => [{ itemType: 'tool' as const, itemId: 'injected' }]),
},
initialize: jest.fn(async () => undefined),
close: jest.fn(async () => undefined),
};
const factory = jest.fn(() => backend);
const runtime = createFavoriteRuntime(mongo, { INJECTED_PG: 'owned-connection' }, factory);
const captured = runtime.methods.getToolFavorites;
expect(factory).not.toHaveBeenCalled();
await expect(captured('user')).rejects.toThrow('not initialized');
await runtime.initialize({
favorites: 'postgres',
postgres: { connectionStringEnv: 'INJECTED_PG' },
});
await runtime.initialize(undefined);
expect(factory).toHaveBeenCalledTimes(1);
expect(factory).toHaveBeenCalledWith(
expect.objectContaining({
connectionString: 'owned-connection',
max: 10,
queryTimeoutMs: 15000,
}),
);
expect(backend.initialize).toHaveBeenCalledTimes(1);
expect(runtime.methods.getToolFavorites).toBe(captured);
expect(await captured('user')).toEqual([{ itemType: 'tool', itemId: 'injected' }]);
await runtime.close();
await runtime.close();
expect(backend.close).toHaveBeenCalledTimes(1);
await expect(captured('user')).rejects.toThrow('not initialized');
const unused = jest.fn(() => backend);
const defaultRuntime = createFavoriteRuntime(mongo, {}, unused);
await defaultRuntime.initialize(undefined);
expect(unused).not.toHaveBeenCalled();
await defaultRuntime.close();
});
it.each(['construct', 'initialize'] as const)(
'sanitizes an injected %s failure without publishing methods or falling back',
async (stage) => {
const privateError = new Error('private backend detail');
const backend = {
methods: mongo,
initialize: jest.fn().mockRejectedValue(privateError),
close: jest.fn(async () => undefined),
};
const factory = jest.fn(() => {
if (stage === 'construct') throw privateError;
return backend;
});
const runtime = createFavoriteRuntime(mongo, { INJECTED_PG: 'owned-connection' }, factory);
await expect(
runtime.initialize({
favorites: 'postgres',
postgres: { connectionStringEnv: 'INJECTED_PG' },
}),
).rejects.toThrow('PostgreSQL favorites initialization failed');
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
await runtime.close();
expect(factory).toHaveBeenCalledTimes(1);
expect(backend.close).toHaveBeenCalledTimes(stage === 'initialize' ? 1 : 0);
},
);
it('waits for the injected initializer during shutdown and never publishes its methods', async () => {
let ready!: () => void;
const gate = new Promise<void>((resolve) => {
ready = resolve;
});
const backend = {
methods: mongo,
initialize: jest.fn(() => gate),
close: jest.fn(async () => undefined),
};
const runtime = createFavoriteRuntime(
mongo,
{ INJECTED_PG: 'owned-connection' },
() => backend,
);
const initialization = runtime.initialize({
favorites: 'postgres',
postgres: { connectionStringEnv: 'INJECTED_PG' },
});
const result = initialization.catch((error: Error) => error);
const closing = runtime.close();
expect(backend.close).not.toHaveBeenCalled();
ready();
expect(await result).toEqual(new Error('PostgreSQL favorites initialization failed'));
await closing;
expect(backend.close).toHaveBeenCalledTimes(1);
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
});
it('extracts only deployment persistence and rejects malformed deployment data', () => {
expect(getPersistenceConfig(null)).toBeUndefined();
expect(getPersistenceConfig({ persistence: { favorites: 'mongo' } })).toEqual({
favorites: 'mongo',
});
expect(() => getPersistenceConfig('invalid')).toThrow();
});
});
describe('storage selection source handling', () => {
const create = (loadLocal: () => unknown) =>
createCustomConfigLoader({
defaultConfigPath: 'librechat.yaml',
loadLocal,
redactConfig: (config) => config,
});
it('permits an absent default file, but never malformed YAML or an explicit missing source', async () => {
const original = process.env.CONFIG_PATH;
delete process.env.CONFIG_PATH;
const missing = Object.assign(new Error('missing'), { code: 'ENOENT' });
try {
await expect(create(() => missing)(false, { storageSelection: true })).resolves.toBeNull();
await expect(
create(() => new Error('malformed'))(false, { storageSelection: true }),
).rejects.toThrow('unavailable');
await expect(
create(() => Object.assign(new Error('unreadable'), { code: 'EACCES' }))(false, {
storageSelection: true,
}),
).rejects.toThrow('unavailable');
process.env.CONFIG_PATH = 'explicit.yaml';
await expect(create(() => missing)(false, { storageSelection: true })).rejects.toThrow(
'unavailable',
);
} finally {
if (original == null) {
delete process.env.CONFIG_PATH;
} else {
process.env.CONFIG_PATH = original;
}
}
});
it('refuses invalid persistence even with CONFIG_BYPASS_VALIDATION', async () => {
const original = process.env.CONFIG_BYPASS_VALIDATION;
process.env.CONFIG_BYPASS_VALIDATION = 'true';
try {
await expect(
create(() => ({ version: '1.3.1', persistence: { favorites: 'postgress' } }))(false),
).rejects.toThrow('Invalid persistence');
} finally {
if (original == null) {
delete process.env.CONFIG_BYPASS_VALIDATION;
} else {
process.env.CONFIG_BYPASS_VALIDATION = original;
}
}
});
});
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
realPostgres('selected PostgreSQL lifecycle and HTTP boundary', () => {
const env = { PG_TEST_URL: process.env.PG_TEST_URL };
const config = {
favorites: 'postgres',
postgres: { connectionStringEnv: 'PG_TEST_URL', acquireTimeoutMs: 1000, queryTimeoutMs: 5000 },
};
const options = {
connectionString: process.env.PG_TEST_URL ?? '',
max: 2,
maxWaiting: 10,
acquireTimeoutMs: 1000,
queryTimeoutMs: 5000,
idleTimeoutMs: 1000,
};
const schema = `favorites_lifecycle_${randomBytes(6).toString('hex')}`;
const admin = createPostgresDatabase(options);
beforeAll(async () => {
await admin.query(`CREATE SCHEMA ${schema}`);
const url = new URL(options.connectionString);
url.searchParams.set('options', `-c search_path=${schema}`);
options.connectionString = url.toString();
env.PG_TEST_URL = options.connectionString;
});
afterAll(async () => {
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
await admin.close();
});
it('refuses schema-only data, releases failed pools, and serves captured HTTP handlers after verification', async () => {
const db = createPostgresDatabase(options);
try {
await migratePostgres(db);
await db.query('TRUNCATE tool_favorites, tool_favorite_erasure');
await db.query(
'UPDATE tool_favorite_readiness SET verified=false, activated=false, checkpoint=NULL, snapshot=NULL',
);
const end = jest.spyOn(pg.Pool.prototype, 'end');
const failed = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend);
await expect(failed.initialize(config)).rejects.toThrow('initialization failed');
expect(end).toHaveBeenCalled();
await failed.close();
await importFavorites(db, async function* () {}, {
offline: true,
batchSize: 10,
verifiedEmpty: true,
});
const runtime = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend);
const handlers = createToolFavoritesHandlers(runtime.methods);
await runtime.initialize(config);
const uid = '0123456789abcdef01234567';
const req = {
user: { id: uid },
params: { itemType: 'tool', itemId: 'native' },
} as ServerRequest;
const res = {
status: jest.fn().mockReturnThis(),
json: jest.fn().mockReturnThis(),
} as unknown as Response;
try {
await handlers.addToolFavorite(req, res);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith({ itemType: 'tool', itemId: 'native' });
expect(await runtime.methods.deleteUserToolFavorites(uid)).toBe(1);
expect(await runtime.methods.getToolFavorites(uid)).toEqual([]);
await db.query('ALTER TABLE tool_favorites RENAME TO unavailable_favorites');
await handlers.listToolFavorites(req, res);
expect(res.status).toHaveBeenLastCalledWith(500);
expect(res.json).toHaveBeenLastCalledWith({ message: 'Failed to retrieve favorites' });
await db.query('ALTER TABLE unavailable_favorites RENAME TO tool_favorites');
await expect(
runtime.methods.addToolFavorite({ userId: uid, itemType: 'tool', itemId: 'late' }),
).rejects.toThrow('erasure');
} finally {
await runtime.close();
await expect(runtime.methods.getToolFavorites(uid)).rejects.toThrow('not initialized');
}
} finally {
await db.close();
}
});
it('fails startup on a selected PostgreSQL outage without using Mongo', async () => {
const end = jest.spyOn(pg.Pool.prototype, 'end');
const runtime = createFavoriteRuntime(
mongo,
{
PG_TEST_URL: 'postgresql://postgres@127.0.0.1:1/postgres',
},
createPostgresFavoriteBackend,
);
await expect(runtime.initialize(config)).rejects.toThrow('initialization failed');
expect(end).toHaveBeenCalled();
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
await runtime.close();
});
it('does not publish methods when shutdown races initialization', async () => {
const runtime = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend);
const initializing = runtime.initialize(config);
const stopping = runtime.close();
await expect(initializing).rejects.toThrow('initialization failed');
await stopping;
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
expect(runtime.close()).toBe(stopping);
});
});

View file

@ -0,0 +1,100 @@
import { persistenceConfigSchema } from 'librechat-data-provider';
import type {
ToolFavoriteMethods,
PostgresOptions,
PostgresFavoriteBackend,
} from '@librechat/data-schemas';
export interface FavoriteRuntime {
methods: ToolFavoriteMethods;
initialize(config: unknown): Promise<void>;
close(): Promise<void>;
}
/** Captured route references never change, and no request chooses its storage engine. */
export function createFavoriteRuntime(
mongo: ToolFavoriteMethods,
env: NodeJS.ProcessEnv,
createPostgres: (options: PostgresOptions) => PostgresFavoriteBackend,
): FavoriteRuntime {
let selected: ToolFavoriteMethods | undefined;
let backend: PostgresFavoriteBackend | undefined;
let backendShutdown: Promise<void> | undefined;
const closeBackend = () => (backendShutdown ??= backend?.close() ?? Promise.resolve());
let initialization: Promise<void> | undefined;
let closed = false;
let shutdown: Promise<void> | undefined;
function methods(): ToolFavoriteMethods {
if (!selected || closed) {
throw new Error('Favorites persistence is not initialized');
}
return selected;
}
return {
methods: {
getToolFavorites: async (userId) => methods().getToolFavorites(userId),
addToolFavorite: async (params) => methods().addToolFavorite(params),
removeToolFavorite: async (params) => methods().removeToolFavorite(params),
deleteUserToolFavorites: async (userId, deletionAttemptId) =>
deletionAttemptId === undefined
? methods().deleteUserToolFavorites(userId)
: methods().deleteUserToolFavorites(userId, deletionAttemptId),
cancelUserToolFavoriteDeletion: async (userId, deletionAttemptId) =>
methods().cancelUserToolFavoriteDeletion?.(userId, deletionAttemptId),
},
initialize(config) {
if (closed) {
return Promise.reject(new Error('Favorites persistence is closed'));
}
initialization ??= (async () => {
if (closed) {
throw new Error('Favorites persistence is closed');
}
const parsed = persistenceConfigSchema.safeParse(config);
if (!parsed.success) {
throw new Error('Invalid persistence configuration');
}
if (parsed.data.favorites === 'mongo') {
selected = mongo;
return;
}
const options = parsed.data.postgres;
const connectionString = options && env[options.connectionStringEnv];
if (!options || !connectionString) {
throw new Error('PostgreSQL connection environment variable is missing');
}
try {
backend = createPostgres({ ...options, connectionString });
await backend.initialize();
if (closed) {
throw new Error('Favorites persistence is closed');
}
selected = backend.methods;
} catch {
await closeBackend().catch(() => undefined);
throw new Error('PostgreSQL favorites initialization failed');
}
})();
return initialization;
},
close() {
closed = true;
shutdown ??= (async () => {
await initialization?.catch(() => undefined);
await closeBackend();
})();
return shutdown;
},
};
}
/** Storage selection is deployment-only, never a tenant config override. */
export function getPersistenceConfig(config: unknown): unknown {
if (config == null) {
return undefined;
}
if (typeof config !== 'object' || Array.isArray(config)) {
throw new Error('Invalid deployment configuration');
}
return 'persistence' in config ? config.persistence : undefined;
}

View file

@ -0,0 +1,648 @@
import pg from 'pg';
import mongoose from 'mongoose';
import { resolve } from 'node:path';
import { promisify } from 'node:util';
import { randomBytes } from 'node:crypto';
import { execFile, spawnSync } from 'node:child_process';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs';
import {
createModels,
createMongoFavoriteStore,
createPostgresDatabase,
fingerprintFavorites,
readPostgresFavorites,
} from '@librechat/data-schemas';
import type { MongoFavoriteStore } from '@librechat/data-schemas';
import { runPostgresCommand } from './postgres';
describe('PostgreSQL command arguments', () => {
const favorites: MongoFavoriteStore = {
source: async function* () {},
stage: jest.fn(),
};
const disconnectDb = jest.fn(async () => undefined);
const commands = ['migrate', 'import', 'verify', 'reconcile-mongo'];
const flags = ['--offline', '--dry-run', '--verified-empty'];
const combinations = commands.flatMap((command) =>
Array.from({ length: 8 }, (_, mask) => ({
args: [command, ...flags.filter((_flag, bit) => (mask & (1 << bit)) !== 0)],
})),
);
it.each(combinations)('validates $args before configuration or connections', async ({ args }) => {
const [command, ...flags] = args;
const unsupported =
(command === 'migrate' && flags.length > 0) ||
(command === 'verify' && flags.some((flag) => flag !== '--offline')) ||
(command === 'reconcile-mongo' && flags.includes('--verified-empty'));
const missingOffline = command !== 'migrate' && !flags.includes('--offline');
const sentinel = new Error('configuration not loaded');
const loadCustomConfig = jest.fn().mockRejectedValue(sentinel);
const connectDb = jest.fn();
const end = jest.spyOn(pg.Pool.prototype, 'end');
const operation = runPostgresCommand({
createDatabase: createPostgresDatabase,
favorites,
disconnectDb,
connectDb,
loadCustomConfig,
args,
env: {},
});
if (unsupported) await expect(operation).rejects.toThrow('Usage:');
else if (missingOffline) await expect(operation).rejects.toThrow('pass --offline');
else await expect(operation).rejects.toBe(sentinel);
expect(loadCustomConfig).toHaveBeenCalledTimes(unsupported || missingOffline ? 0 : 1);
expect(connectDb).not.toHaveBeenCalled();
expect(end).not.toHaveBeenCalled();
});
it.each([
{ args: [] },
{ args: ['unknown'] },
{ args: ['__proto__'] },
{ args: ['constructor'] },
...commands.map((command) => ({ args: [command, '--unknown'] })),
...commands.map((command) => ({ args: [command, '--offline', '--offline'] })),
])('rejects unknown commands, flags, and duplicates: $args', async ({ args }) => {
const loadCustomConfig = jest.fn();
const connectDb = jest.fn();
await expect(
runPostgresCommand({
createDatabase: createPostgresDatabase,
favorites,
disconnectDb,
connectDb,
loadCustomConfig,
args,
env: {},
}),
).rejects.toThrow('Usage:');
expect(loadCustomConfig).not.toHaveBeenCalled();
expect(connectDb).not.toHaveBeenCalled();
});
});
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
realPostgres('PostgreSQL CLI with real Mongo and PostgreSQL', () => {
const schema = `favorites_cli_${randomBytes(6).toString('hex')}`;
const baseUrl = process.env.PG_TEST_URL ?? '';
const url = new URL(baseUrl || 'postgresql://postgres@localhost/postgres');
url.searchParams.set('options', `-c search_path=${schema}`);
const env = { PG_TEST_URL: url.toString() };
const config = {
persistence: { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } },
};
const options = {
connectionString: env.PG_TEST_URL,
max: 2,
maxWaiting: 10,
acquireTimeoutMs: 1000,
queryTimeoutMs: 5000,
idleTimeoutMs: 1000,
};
const admin = createPostgresDatabase({ ...options, connectionString: baseUrl });
const db = createPostgresDatabase(options);
let mongo: MongoMemoryServer;
const connectDb = jest.fn(async () =>
mongoose.connect(mongo.getUri(), { autoCreate: false, autoIndex: false }),
);
const disconnectDb = () => mongoose.disconnect();
const favorites = createMongoFavoriteStore(mongoose);
const loadCustomConfig = jest.fn(async () => config);
const run = (args: string[]) =>
runPostgresCommand({
createDatabase: createPostgresDatabase,
favorites,
connectDb,
disconnectDb,
loadCustomConfig,
args,
env,
});
const collection = () => createModels(mongoose).ToolFavorite.collection;
const reconnect = () => mongoose.connect(mongo.getUri());
const uid = new mongoose.Types.ObjectId('0123456789abcdef01234567');
const row = (index: number) => ({
_id: new mongoose.Types.ObjectId(index.toString(16).padStart(24, '0')),
user: uid,
tenantId: index % 2 ? 'tenant-one' : null,
itemType: 'tool',
itemId: `tool-${index}`,
createdAt: new Date('2025-01-01T00:00:00.000Z'),
updatedAt: new Date('2025-01-02T00:00:00.000Z'),
});
beforeAll(async () => {
await admin.query(`CREATE SCHEMA ${schema}`);
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
}, 90_000);
beforeEach(async () => {
jest.clearAllMocks();
await run(['migrate']);
await db.query('TRUNCATE tool_favorites, tool_favorite_erasure');
await db.query(
'UPDATE tool_favorite_readiness SET verified=false, activated=false, checkpoint=NULL, snapshot=NULL, imported=0',
);
await reconnect();
await collection().deleteMany({});
await collection().createIndex({ user: 1, itemType: 1, itemId: 1 }, { unique: true });
await collection().createIndex({ tenantId: 1 });
await mongoose.disconnect();
connectDb.mockClear();
});
afterAll(async () => {
await mongoose.disconnect();
await mongo?.stop();
await db.close();
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
await admin.close();
});
it('imports and reconciles through plain snapshot, staging, and lifecycle interfaces', async () => {
const snapshot = {
id: row(1)._id.toHexString(),
userId: uid.toHexString(),
tenantId: 'tenant-one',
itemType: 'tool',
itemId: 'portable-store',
createdAt: row(1).createdAt,
updatedAt: row(1).updatedAt,
} as const;
const source = async function* () {
yield snapshot;
};
const insert = jest.fn(async () => undefined);
const replace = jest.fn(async () => undefined);
const discard = jest.fn(async () => undefined);
const store: MongoFavoriteStore = {
source,
stage: jest.fn(async () => ({ source, insert, replace, discard })),
};
const connect = jest.fn(async () => undefined);
const disconnect = jest.fn(async () => undefined);
const createDatabase = jest.fn(createPostgresDatabase);
for (const command of ['import', 'verify', 'reconcile-mongo']) {
await runPostgresCommand({
createDatabase,
favorites: store,
connectDb: connect,
disconnectDb: disconnect,
loadCustomConfig,
args: [command, '--offline'],
env,
});
}
expect(createDatabase).toHaveBeenCalledTimes(3);
expect(createDatabase).toHaveBeenCalledWith(
expect.objectContaining({ connectionString: env.PG_TEST_URL }),
);
expect(connect.mock.calls).toEqual([[], [], []]);
expect(disconnect).toHaveBeenCalledTimes(3);
expect(store.stage).toHaveBeenCalledTimes(1);
expect(insert).toHaveBeenCalledWith([snapshot]);
expect(replace).toHaveBeenCalledTimes(1);
expect(discard).not.toHaveBeenCalled();
expect(connectDb).not.toHaveBeenCalled();
});
it('runs the actual migrate entrypoint with MONGO_URI unset', () => {
const scratchRoot = resolve(__dirname, '../../../../node_modules/.cache/pg-lifecycle');
mkdirSync(scratchRoot, { recursive: true });
const scratch = mkdtempSync(resolve(scratchRoot, 'cli-'));
try {
const configPath = resolve(scratch, 'config.yaml');
writeFileSync(
configPath,
'version: 1.3.1\npersistence:\n favorites: postgres\n postgres:\n connectionStringEnv: PG_TEST_URL\n',
);
const script = resolve(__dirname, '../../../../config/postgres.js');
const child = spawnSync(
process.execPath,
[
'-e',
`delete process.env.MONGO_URI; process.argv = ['node', ${JSON.stringify(script)}, 'migrate']; require(${JSON.stringify(script)});`,
],
{
cwd: scratch,
env: { ...process.env, ...env, CONFIG_PATH: configPath },
encoding: 'utf8',
timeout: 15000,
killSignal: 'SIGKILL',
},
);
expect({ status: child.status, error: child.error?.message, stderr: child.stderr }).toEqual({
status: 0,
error: undefined,
stderr: '',
});
expect(child.stdout).toContain('PostgreSQL schema migrated');
} finally {
rmSync(scratch, { recursive: true, force: true });
}
});
it.each([
{ args: ['import', '--offline', '--dry-run'], seeded: true },
{ args: ['import', '--offline', '--dry-run', '--verified-empty'], seeded: false },
{ args: ['reconcile-mongo', '--offline', '--dry-run'], seeded: true },
{ args: ['reconcile-mongo', '--offline', '--dry-run'], seeded: false },
{ args: ['import', '--offline'], seeded: true },
{ args: ['verify', '--offline'], seeded: true },
{ args: ['reconcile-mongo', '--offline'], seeded: true },
])(
'isolates actual CLI $args from Mongo schema initialization (seeded=$seeded)',
async ({ args, seeded }) => {
const native = await mongoose
.createConnection(mongo.getUri(`cli_${randomBytes(6).toString('hex')}`), {
autoCreate: false,
autoIndex: false,
})
.asPromise();
const scratchRoot = resolve(__dirname, '../../../../node_modules/.cache/pg-lifecycle');
mkdirSync(scratchRoot, { recursive: true });
const scratch = mkdtempSync(resolve(scratchRoot, 'cli-'));
const favorites = native.collection(collection().name);
const mongoSnapshot = async () => {
const collections = await native.db!.listCollections().toArray();
return Promise.all(
collections
.sort((a, b) => a.name.localeCompare(b.name))
.map(async (metadata) => ({
metadata,
indexes: await native.collection(metadata.name).indexes(),
rows: await native.collection(metadata.name).find().sort({ _id: 1 }).toArray(),
})),
);
};
const postgresSnapshot = async () => {
const tables = await db.query<{ tablename: string }>(
'SELECT tablename FROM pg_tables WHERE schemaname=current_schema() ORDER BY tablename',
);
return {
columns: (
await db.query(
'SELECT * FROM information_schema.columns WHERE table_schema=current_schema() ORDER BY table_name,ordinal_position',
)
).rows,
indexes: (
await db.query(
'SELECT * FROM pg_indexes WHERE schemaname=current_schema() ORDER BY tablename,indexname',
)
).rows,
tables: await Promise.all(
tables.rows.map(async ({ tablename }) => ({
tablename,
rows: (await db.query(`SELECT * FROM "${tablename}" ORDER BY 1`)).rows,
})),
),
};
};
try {
if (seeded) await favorites.insertOne(row(1));
if (args[0] !== 'import') {
await db.query(
'INSERT INTO tool_favorites(id,user_id,tenant_id,item_type,item_id,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7)',
[
row(1)._id.toHexString(),
uid.toHexString(),
row(1).tenantId,
'tool',
row(1).itemId,
row(1).createdAt,
row(1).updatedAt,
],
);
const snapshot = await fingerprintFavorites(readPostgresFavorites(db, 250));
await db.query(
'UPDATE tool_favorite_readiness SET verified=true,activated=true,snapshot=$1',
[snapshot.digest],
);
}
const beforeMongo = await mongoSnapshot();
const beforePostgres = await postgresSnapshot();
const configPath = resolve(scratch, 'config.yaml');
writeFileSync(
configPath,
'version: 1.3.1\npersistence:\n favorites: postgres\n postgres:\n connectionStringEnv: PG_TEST_URL\n',
);
const guardPath = resolve(scratch, 'guard.cjs');
writeFileSync(
guardPath,
`
const assert = require('node:assert/strict');
const mongoose = require(${JSON.stringify(require.resolve('mongoose'))});
const connect = mongoose.connect.bind(mongoose);
mongoose.connect = (uri, options) => {
assert.equal(options.autoCreate, false);
assert.equal(options.autoIndex, false);
return connect(uri, options);
};
mongoose.model = () => { throw new Error('CLI registered a connected model'); };
process.on('beforeExit', () => {
assert.deepEqual(mongoose.modelNames(), []);
assert.equal(Object.keys(require.cache).some(path => path.endsWith('/api/db/index.js')), false);
});
`,
);
const child = await promisify(execFile)(
process.execPath,
['--require', guardPath, resolve(__dirname, '../../../../config/postgres.js'), ...args],
{
cwd: scratch,
env: {
...process.env,
...env,
CONFIG_PATH: configPath,
MONGO_URI: mongo.getUri(native.name),
MONGO_AUTO_CREATE: 'true',
MONGO_AUTO_INDEX: 'true',
},
encoding: 'utf8',
timeout: 15000,
killSignal: 'SIGKILL',
},
);
expect(child.stderr).toBe('');
const afterMongo = await mongoSnapshot();
if (args[0] === 'reconcile-mongo' && !args.includes('--dry-run')) {
expect(afterMongo).toHaveLength(1);
const before = beforeMongo[0].metadata;
const after = afterMongo[0].metadata;
if (!('info' in before) || !before.info || !('info' in after) || !after.info) {
throw new Error('Full Mongo collection metadata is required');
}
expect(after.info.uuid).not.toEqual(before.info.uuid);
before.info.uuid = after.info.uuid;
}
expect(afterMongo).toEqual(beforeMongo);
if (args[0] !== 'import' || args.includes('--dry-run')) {
expect(await postgresSnapshot()).toEqual(beforePostgres);
} else {
expect((await db.query('SELECT count(*) FROM tool_favorites')).rows[0].count).toBe('1');
}
} finally {
await native.dropDatabase();
await native.close();
rmSync(scratch, { recursive: true, force: true });
}
},
);
it('rejects a migration dry run before creating any schema objects or opening connections', async () => {
const emptySchema = `favorites_dry_run_${randomBytes(6).toString('hex')}`;
await admin.query(`CREATE SCHEMA ${emptySchema}`);
const target = new URL(baseUrl);
target.searchParams.set('options', `-c search_path=${emptySchema}`);
const end = jest.spyOn(pg.Pool.prototype, 'end');
loadCustomConfig.mockClear();
try {
const result = await runPostgresCommand({
createDatabase: createPostgresDatabase,
favorites,
disconnectDb,
connectDb,
loadCustomConfig,
args: ['migrate', '--dry-run'],
env: { PG_TEST_URL: target.toString() },
}).then(
() => 'success',
(error: Error) => error.message,
);
const objects = await admin.query('SELECT tablename FROM pg_tables WHERE schemaname=$1', [
emptySchema,
]);
expect(objects.rows).toEqual([]);
expect(result).toContain('Usage:');
expect(loadCustomConfig).not.toHaveBeenCalled();
expect(connectDb).not.toHaveBeenCalled();
expect(end).not.toHaveBeenCalled();
} finally {
await admin.query(`DROP SCHEMA ${emptySchema} CASCADE`);
}
});
it('migrates without connecting Mongo and releases its pool', async () => {
const end = jest.spyOn(pg.Pool.prototype, 'end');
await run(['migrate']);
expect(connectDb).not.toHaveBeenCalled();
expect(end).toHaveBeenCalledTimes(1);
expect(loadCustomConfig).toHaveBeenCalledWith(false, {
mode: 'reload',
storageSelection: true,
});
});
it.each(['import', 'verify', 'reconcile-mongo'])(
'requires offline for %s before opening connections',
async (command) => {
const end = jest.spyOn(pg.Pool.prototype, 'end');
const disconnect = jest.spyOn(mongoose, 'disconnect');
await expect(run([command])).rejects.toThrow('--offline');
expect(connectDb).not.toHaveBeenCalled();
expect(end).not.toHaveBeenCalled();
expect(disconnect).toHaveBeenCalledTimes(1);
},
);
it('validates an import dry run without changing data or readiness', async () => {
await reconnect();
await collection().insertMany([row(1), row(2)]);
const original = await collection().find().sort({ _id: 1 }).toArray();
await mongoose.disconnect();
const state = await db.query('SELECT * FROM tool_favorite_readiness');
await run(['import', '--offline', '--dry-run']);
expect((await db.query('SELECT * FROM tool_favorites')).rows).toEqual([]);
expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(state.rows);
await reconnect();
expect(await collection().find().sort({ _id: 1 }).toArray()).toEqual(original);
});
it('imports and verifies identity, tenant and dates, then reconciles deletes and preserves indexes without changing authority', async () => {
await reconnect();
await collection().insertMany([row(1), row(2)]);
const indexes = await collection().indexes();
await mongoose.disconnect();
await run(['import', '--offline']);
await run(['verify', '--offline']);
const expected = await fingerprintFavorites(readPostgresFavorites(db, 250));
await db.query('DELETE FROM tool_favorites WHERE id=$1', [row(1)._id.toHexString()]);
const state = await db.query('SELECT * FROM tool_favorite_readiness');
await expect(run(['verify', '--offline'])).rejects.toThrow('differ');
await run(['reconcile-mongo', '--offline', '--dry-run']);
await reconnect();
expect(await collection().countDocuments()).toBe(2);
await mongoose.disconnect();
await run(['reconcile-mongo', '--offline']);
await run(['verify', '--offline']);
await reconnect();
expect(await collection().findOne({ _id: row(1)._id })).toBeNull();
expect(await collection().findOne({ _id: row(2)._id })).toEqual(row(2));
expect(await collection().indexes()).toEqual(indexes);
expect(
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
).toEqual([]);
expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(state.rows);
expect(expected.count).toBe(2);
});
it.each(['schema-only', 'partial', 'missing-marker'] as const)(
'refuses destructive reconciliation from %s PostgreSQL without modifying Mongo',
async (state) => {
await reconnect();
await collection().insertMany([row(1), row(2)]);
const original = await collection().find().sort({ _id: 1 }).toArray();
const indexes = await collection().indexes();
await mongoose.disconnect();
if (state !== 'schema-only') {
await run(['import', '--offline']);
await db.query('DELETE FROM tool_favorites WHERE id=$1', [row(2)._id.toHexString()]);
await db.query('UPDATE tool_favorite_readiness SET verified=false,checkpoint=$1', [
row(1)._id.toHexString(),
]);
}
if (state === 'missing-marker') await db.query('DELETE FROM tool_favorite_readiness');
const authority = await db.query('SELECT * FROM tool_favorite_readiness');
for (const flags of [[], ['--dry-run']]) {
await expect(run(['reconcile-mongo', '--offline', ...flags])).rejects.toThrow(
'not verified',
);
await reconnect();
expect(await collection().find().sort({ _id: 1 }).toArray()).toEqual(original);
expect(await collection().indexes()).toEqual(indexes);
expect(
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
).toEqual([]);
await mongoose.disconnect();
}
expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(
authority.rows,
);
if (state === 'missing-marker')
await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)');
},
);
it('allows verified activated PostgreSQL to reconcile subsequent deletions, including an empty snapshot', async () => {
await reconnect();
await collection().insertOne(row(1));
await mongoose.disconnect();
await run(['import', '--offline']);
await db.query('UPDATE tool_favorite_readiness SET activated=true');
await db.query('DELETE FROM tool_favorites');
await run(['reconcile-mongo', '--offline']);
await reconnect();
expect(await collection().countDocuments()).toBe(0);
expect((await db.query('SELECT verified,activated FROM tool_favorite_readiness')).rows).toEqual(
[{ verified: true, activated: true }],
);
});
it('refuses a same-count corrupted staging snapshot, retains Mongo, and removes staging', async () => {
await reconnect();
await collection().insertOne(row(1));
await mongoose.disconnect();
await run(['import', '--offline']);
const original = mongoose.connection.collection.bind(mongoose.connection);
const staging = jest
.spyOn(mongoose.connection, 'collection')
.mockImplementation((name, options) => {
const target = original(name, options);
if (name.includes('_reconcile_')) {
const insert = target.insertMany.bind(target);
jest.spyOn(target, 'insertMany').mockImplementation(async (docs, options) => {
const result = await insert(docs, options);
await target.updateOne({}, { $set: { itemId: 'corrupt' } });
return result;
});
}
return target;
});
await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow(
'changed during reconciliation',
);
staging.mockRestore();
await reconnect();
expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1));
expect(
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
).toEqual([]);
});
it('rechecks PostgreSQL readiness before Mongo replacement and discards untrusted staging', async () => {
await reconnect();
await collection().insertOne(row(1));
await mongoose.disconnect();
await run(['import', '--offline']);
const original = mongoose.connection.collection.bind(mongoose.connection);
jest.spyOn(mongoose.connection, 'collection').mockImplementation((name, options) => {
const target = original(name, options);
if (name.includes('_reconcile_')) {
const insert = target.insertMany.bind(target);
jest.spyOn(target, 'insertMany').mockImplementation(async (docs, options) => {
const result = await insert(docs, options);
await db.query('UPDATE tool_favorite_readiness SET verified=false');
return result;
});
}
return target;
});
await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow('not verified');
jest.restoreAllMocks();
await reconnect();
expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1));
expect(
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
).toEqual([]);
});
it('does not race a migration or import holding the same schema authority', async () => {
await reconnect();
await collection().insertOne(row(1));
await mongoose.disconnect();
await run(['import', '--offline']);
await db.session(async (session) => {
await session.query('SELECT pg_advisory_lock(17490261,hashtext(current_schema()))');
try {
await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow(
'Another favorite migration',
);
} finally {
await session.query('SELECT pg_advisory_unlock(17490261,hashtext(current_schema()))');
}
});
await reconnect();
expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1));
});
it('reports cleanup failure after a successful command instead of silently succeeding', async () => {
const original = pg.Pool.prototype.end;
jest.spyOn(pg.Pool.prototype, 'end').mockImplementationOnce(async function (this: pg.Pool) {
await original.bind(this)();
throw new Error('cleanup failed');
});
const disconnect = jest.spyOn(mongoose, 'disconnect');
await expect(run(['migrate'])).rejects.toThrow('command cleanup failed');
expect(disconnect).toHaveBeenCalledTimes(1);
});
it.each([{ user: null }, { tenantId: 42 }, { createdAt: null }, { updatedAt: 'invalid' }])(
'closes pools and Mongo after malformed source validation without exposing the record: %j',
async (invalid) => {
await reconnect();
await collection().insertOne({ ...row(1), ...invalid });
await mongoose.disconnect();
const end = jest.spyOn(pg.Pool.prototype, 'end');
const disconnect = jest.spyOn(mongoose, 'disconnect');
await expect(run(['import', '--offline'])).rejects.toThrow(
'Invalid favorite snapshot record',
);
expect(end).toHaveBeenCalledTimes(1);
expect(disconnect).toHaveBeenCalledTimes(1);
expect((await db.query('SELECT count(*) FROM tool_favorites')).rows[0].count).toBe('0');
},
);
it('keeps the operational error when pool shutdown also fails, and still disconnects Mongo', async () => {
await reconnect();
await collection().insertOne({ ...row(1), user: null });
await mongoose.disconnect();
const original = pg.Pool.prototype.end;
jest.spyOn(pg.Pool.prototype, 'end').mockImplementationOnce(async function (this: pg.Pool) {
await original.bind(this)();
throw new Error('cleanup failed');
});
const disconnect = jest.spyOn(mongoose, 'disconnect');
await expect(run(['import', '--offline'])).rejects.toThrow('Invalid favorite snapshot record');
expect(disconnect).toHaveBeenCalledTimes(1);
});
it('disconnects Mongo even when configuration loading fails before a pool exists', async () => {
loadCustomConfig.mockRejectedValueOnce(new Error('Deployment config unavailable'));
const disconnect = jest.spyOn(mongoose, 'disconnect');
await expect(run(['migrate'])).rejects.toThrow('Deployment config unavailable');
expect(disconnect).toHaveBeenCalledTimes(1);
});
});

View file

@ -0,0 +1,164 @@
import { persistenceConfigSchema } from 'librechat-data-provider';
import {
migratePostgres,
importFavorites,
fingerprintFavorites,
readPostgresFavorites,
assertPostgresDataReady,
} from '@librechat/data-schemas';
import type {
FavoriteSnapshotRow,
MongoFavoriteStore,
PostgresDatabase,
PostgresOptions,
} from '@librechat/data-schemas';
import { getPersistenceConfig } from './persistence';
export async function runPostgresCommand(deps: {
favorites: MongoFavoriteStore;
createDatabase(options: PostgresOptions): PostgresDatabase;
connectDb(): Promise<unknown>;
disconnectDb(): Promise<void>;
loadCustomConfig(
print: boolean,
options: { mode: 'reload'; storageSelection: true },
): Promise<unknown>;
args: string[];
env: NodeJS.ProcessEnv;
}): Promise<void> {
let db: PostgresDatabase | undefined;
const close = () => Promise.allSettled([db?.close(), deps.disconnectDb()]);
const execute = async () => {
const [command, ...flags] = deps.args;
const commands: Record<string, string[]> = {
migrate: [],
import: ['--offline', '--dry-run', '--verified-empty'],
verify: ['--offline'],
'reconcile-mongo': ['--offline', '--dry-run'],
};
const allowed = Object.prototype.hasOwnProperty.call(commands, command)
? commands[command]
: undefined;
if (
!allowed ||
flags.some((flag) => !allowed.includes(flag)) ||
new Set(flags).size !== flags.length
) {
throw new Error(
'Usage: postgres migrate | import --offline [--dry-run] [--verified-empty] | verify --offline | reconcile-mongo --offline [--dry-run]',
);
}
if (command !== 'migrate' && !flags.includes('--offline')) {
throw new Error('Stop all app, worker, and migration writers and pass --offline');
}
const config = persistenceConfigSchema.parse(
getPersistenceConfig(
await deps.loadCustomConfig(false, { mode: 'reload', storageSelection: true }),
),
);
const options = config.postgres;
const connectionString = options && deps.env[options.connectionStringEnv];
if (!options || !connectionString) {
throw new Error('Missing PostgreSQL connection environment variable');
}
db = deps.createDatabase({ ...options, connectionString });
if (command === 'migrate') {
await migratePostgres(db);
console.info('PostgreSQL schema migrated. Favorites data is not yet verified.');
return;
}
await deps.connectDb();
const mongo = deps.favorites;
const source = mongo.source;
if (command === 'import') {
const result = await importFavorites(db, source, {
offline: true,
batchSize: 250,
dryRun: flags.includes('--dry-run'),
verifiedEmpty: flags.includes('--verified-empty'),
});
console.info(
`Favorites snapshot ${flags.includes('--dry-run') ? 'validated' : 'imported and verified'}: ${result.count} rows`,
);
return;
}
if (command === 'verify') {
const postgres = await fingerprintFavorites(readPostgresFavorites(db, 250));
const mongo = await fingerprintFavorites(source);
if (postgres.digest !== mongo.digest || postgres.count !== mongo.count) {
throw new Error('Favorite snapshots differ');
}
console.info(`Favorites snapshots match: ${postgres.count} rows`);
return;
}
await db.session(async (session) => {
const lock = await session.query<{ acquired: boolean }>(
'SELECT pg_try_advisory_lock(17490261, hashtext(current_schema())) AS acquired',
);
if (!lock.rows[0].acquired)
throw new Error('Another favorite migration or import is running');
try {
await assertPostgresDataReady(session);
const pgSource = readPostgresFavorites(session, 250);
const postgres = await fingerprintFavorites(pgSource);
// Build and validate a replacement before an atomic rename. Deletes are reconciled too.
if (flags.includes('--dry-run')) {
console.info(
`Offline reconciliation would replace Mongo favorites with ${postgres.count} PostgreSQL rows`,
);
return;
}
const staging = await mongo.stage();
let renamed = false;
try {
let batch: FavoriteSnapshotRow[] = [];
const write = async () => {
await staging.insert(batch);
batch = [];
};
for await (const row of pgSource()) {
batch.push(row);
if (batch.length === 250) {
await write();
}
}
if (batch.length) {
await write();
}
const staged = await fingerprintFavorites(staging.source);
const current = await fingerprintFavorites(pgSource);
if (
current.digest !== postgres.digest ||
current.count !== postgres.count ||
staged.digest !== postgres.digest ||
staged.count !== postgres.count
) {
throw new Error('PostgreSQL changed during reconciliation');
}
await assertPostgresDataReady(session);
await staging.replace();
renamed = true;
console.info(
`Mongo favorites reconciled: ${postgres.count} rows. Storage selection was not changed.`,
);
} finally {
if (!renamed) {
await staging.discard();
}
}
} finally {
await session.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))');
}
});
};
try {
await execute();
} catch (error) {
await close();
throw error;
}
const cleanup = await close();
if (cleanup.some((result) => result.status === 'rejected')) {
throw new Error('PostgreSQL command cleanup failed');
}
}

View file

@ -1,23 +1,35 @@
import { Keyv } from 'keyv';
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { ResourceType, PrincipalType } from 'librechat-data-provider';
import {
createPostgresFavoriteBackend,
createPostgresDatabase,
migratePostgres,
importFavorites,
createModels,
createMethods,
tenantStorage,
createSkillDeletionPersistenceAdapter,
logger,
} from '@librechat/data-schemas';
import type { PostgresDatabase } from '@librechat/data-schemas';
import type { AppConfig } from '@librechat/data-schemas';
import type { CheckpointDeletion } from '~/agents/checkpoints/deletion';
import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure';
import type { MCPOAuthTokens } from '~/mcp/oauth/types';
import {
deleteUserMCPServers,
eraseUserData,
cancelUserDataDeletion,
createUserDeletionAttempt,
} from './erasure';
import { MCPOAuthHandler, MCPTokenStorage, getMCPOAuthLeaseId } from '~/mcp/oauth';
import { deleteAllSharedLinksWithCleanup } from '~/shared-links/service';
import { createSkillDeletionService } from '../skills/deletionService';
import { deleteUserMCPServers, eraseUserData } from './erasure';
import { cleanupMCPServerOAuth } from '~/mcp/oauth/cleanup';
import { createFavoriteRuntime } from '~/app/persistence';
import { FlowStateManager } from '~/flow/manager';
/** Stored OAuth credentials are encrypted with a key read when the module loads. */
@ -903,3 +915,332 @@ describe('removing a shared MCP server with its owner', () => {
expect(await count('MCPServer', { _id: mcpServerId })).toBe(1);
});
});
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
realPostgres('attempt-scoped PostgreSQL account erasure', () => {
const schema = `erasure_${randomUUID().replace(/-/g, '')}`;
const config = { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } };
const options = {
connectionString: process.env.PG_TEST_URL ?? '',
max: 2,
maxWaiting: 10,
acquireTimeoutMs: 1000,
queryTimeoutMs: 5000,
idleTimeoutMs: 1000,
};
let admin: PostgresDatabase;
let db: PostgresDatabase;
beforeAll(async () => {
admin = createPostgresDatabase(options);
await admin.query(`CREATE SCHEMA ${schema}`);
const url = new URL(options.connectionString);
url.searchParams.set('options', `-c search_path=${schema}`);
options.connectionString = url.toString();
db = createPostgresDatabase(options);
await migratePostgres(db);
await importFavorites(db, async function* () {}, {
offline: true,
verifiedEmpty: true,
batchSize: 10,
});
});
afterAll(async () => {
await db.close();
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
await admin.close();
});
async function setup() {
const userId = new mongoose.Types.ObjectId().toString();
const startedAt = new Date();
const deletionAttemptId = randomUUID();
await model('User').insertOne({
_id: new mongoose.Types.ObjectId(userId),
agentTriggerDeletionStartedAt: startedAt,
agentTriggerDeletionAttemptId: deletionAttemptId,
});
const runtime = createFavoriteRuntime(
methods,
{ PG_TEST_URL: options.connectionString },
createPostgresFavoriteBackend,
);
await runtime.initialize(config);
const { deps } = createDeps();
const erasure = { ...deps, ...runtime.methods };
const cancel = { ...methods, ...runtime.methods };
const params = { userId, startedAt, deletionAttemptId };
const favorite = { userId, itemType: 'tool', itemId: 'saved' } as const;
await runtime.methods.addToolFavorite(favorite);
return { runtime, erasure, cancel, params, favorite };
}
it('unfences a retained account after a later cascade failure, including after restart', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
erasure.deleteSharedLinks = jest.fn().mockRejectedValue(new Error('shared links unavailable'));
try {
await expect(eraseUserData(params, erasure)).rejects.toThrow('shared links unavailable');
expect(await count('User', { _id: new mongoose.Types.ObjectId(params.userId) })).toBe(1);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
await cancelUserDataDeletion(params, cancel);
await runtime.close();
const restarted = createFavoriteRuntime(
methods,
{ PG_TEST_URL: options.connectionString },
createPostgresFavoriteBackend,
);
try {
await restarted.initialize(config);
await expect(restarted.methods.addToolFavorite(favorite)).resolves.toEqual({
ok: true,
added: true,
});
expect(await restarted.methods.getToolFavorites(params.userId)).toEqual([
{ itemType: 'tool', itemId: 'saved' },
]);
} finally {
await restarted.close();
}
} finally {
await runtime.close();
}
});
it.each(['deleted', 'other-attempt', 'mongo-error'] as const)(
'keeps the marker when cancellation cannot prove ownership: %s',
async (reason) => {
const { runtime, erasure, cancel, params, favorite } = await setup();
try {
await eraseUserData(params, erasure);
if (reason === 'deleted') await methods.deleteUserById(params.userId);
if (reason === 'other-attempt')
await model('User').updateOne(
{ _id: new mongoose.Types.ObjectId(params.userId) },
{ $set: { agentTriggerDeletionStartedAt: new Date(params.startedAt.getTime() + 1) } },
);
if (reason === 'mongo-error')
jest
.spyOn(mongoose.models.User.collection, 'findOne')
.mockRejectedValueOnce(new Error('Mongo unavailable'));
if (reason === 'mongo-error')
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow('Mongo unavailable');
else await cancelUserDataDeletion(params, cancel);
if (reason === 'mongo-error')
expect(
await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt),
).toBe(true);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
await runtime.close();
const restarted = createFavoriteRuntime(
methods,
{ PG_TEST_URL: options.connectionString },
createPostgresFavoriteBackend,
);
try {
await restarted.initialize(config);
await expect(restarted.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
} finally {
await restarted.close();
}
} finally {
await runtime.close();
}
},
);
it('retains Mongo ownership through a PostgreSQL cancellation fault, then retries after restart', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
try {
await eraseUserData(params, erasure);
await db.query(
`CREATE FUNCTION reject_cancel() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'cancel unavailable'; END $$`,
);
await db.query(
'CREATE TRIGGER reject_cancel BEFORE DELETE ON tool_favorite_erasure FOR EACH ROW EXECUTE FUNCTION reject_cancel()',
);
try {
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow('cancel unavailable');
} finally {
await db.query('DROP TRIGGER reject_cancel ON tool_favorite_erasure');
await db.query('DROP FUNCTION reject_cancel()');
}
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
true,
);
await runtime.close();
const restarted = createFavoriteRuntime(
methods,
{ PG_TEST_URL: options.connectionString },
createPostgresFavoriteBackend,
);
try {
await restarted.initialize(config);
await cancelUserDataDeletion(params, { ...methods, ...restarted.methods });
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
false,
);
await expect(restarted.methods.addToolFavorite(favorite)).resolves.toEqual({
ok: true,
added: true,
});
} finally {
await restarted.close();
}
} finally {
await runtime.close();
}
});
it('retries Mongo release after PostgreSQL cleanup without losing the durable owner', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
try {
await eraseUserData(params, erasure);
// Fail the real release write after the ownership read and PostgreSQL cancellation.
jest
.spyOn(mongoose.models.User.collection, 'updateOne')
.mockRejectedValueOnce(new Error('Mongo release unavailable'));
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow(
'Mongo release unavailable',
);
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
true,
);
await expect(runtime.methods.addToolFavorite(favorite)).resolves.toEqual({
ok: true,
added: true,
});
await cancelUserDataDeletion(params, cancel);
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
false,
);
} finally {
await runtime.close();
}
});
it('tolerates a lost Mongo release acknowledgement after PostgreSQL cleanup', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
try {
await eraseUserData(params, erasure);
cancel.cancelAgentTriggerUserDeletion = async (userId, startedAt) => {
await methods.cancelAgentTriggerUserDeletion(userId, startedAt, params.deletionAttemptId);
throw new Error('release acknowledgement lost');
};
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow(
'release acknowledgement lost',
);
await cancelUserDataDeletion(params, { ...methods, ...runtime.methods });
await expect(runtime.methods.addToolFavorite(favorite)).resolves.toEqual({
ok: true,
added: true,
});
} finally {
await runtime.close();
}
});
it('makes concurrent cancellation idempotent without releasing a successor UUID', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
try {
await eraseUserData(params, erasure);
const successor = randomUUID();
await runtime.methods.deleteUserToolFavorites(params.userId, successor);
await Promise.all([
cancelUserDataDeletion(params, cancel),
cancelUserDataDeletion(params, cancel),
]);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
expect(
(
await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [
params.userId,
])
).rows,
).toEqual([{ attempt_id: successor }]);
} finally {
await runtime.close();
}
});
it('preserves a successor owner and UUID acquired after the ownership proof', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
const successor = randomUUID();
const nextStartedAt = new Date(params.startedAt.getTime() + 1);
try {
await eraseUserData(params, erasure);
cancel.isAgentTriggerUserDeletionOwner = async (userId, startedAt) => {
const owned = await methods.isAgentTriggerUserDeletionOwner(userId, startedAt);
await methods.cancelAgentTriggerUserDeletion(userId, startedAt, params.deletionAttemptId);
await methods.beginAgentTriggerUserDeletion(userId, nextStartedAt, successor);
await runtime.methods.deleteUserToolFavorites(userId, successor);
return owned;
};
await cancelUserDataDeletion(params, cancel);
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, nextStartedAt)).toBe(
true,
);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
expect(
(
await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [
params.userId,
])
).rows,
).toEqual([{ attempt_id: successor }]);
} finally {
await runtime.close();
}
});
it('keeps both fences when a final delete is pending or loses its acknowledgement', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
const attempt = createUserDeletionAttempt();
let releaseCommit!: () => void;
const gate = new Promise<void>((resolve) => {
releaseCommit = resolve;
});
try {
await eraseUserData(params, erasure);
const committing = attempt.commit(async () => {
await gate;
await methods.deleteUserById(params.userId, {
startedAt: params.startedAt,
attemptId: params.deletionAttemptId,
});
throw new Error('delete acknowledgement lost');
});
await attempt.cancel(params, cancel);
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
true,
);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
releaseCommit();
await expect(committing).rejects.toThrow('delete acknowledgement lost');
await attempt.cancel(params, cancel);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
} finally {
releaseCommit();
await runtime.close();
}
});
it('keeps both fences after a final delete throws without a known outcome', async () => {
const { runtime, erasure, cancel, params, favorite } = await setup();
const attempt = createUserDeletionAttempt();
try {
await eraseUserData(params, erasure);
await expect(
attempt.commit(async () => {
throw new Error('delete outcome unknown');
}),
).rejects.toThrow('delete outcome unknown');
await attempt.cancel(params, cancel);
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
true,
);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
} finally {
await runtime.close();
}
});
it('never cancels a permanent marker when no UUID was supplied', async () => {
const { runtime, cancel, params, favorite } = await setup();
try {
await runtime.methods.deleteUserToolFavorites(params.userId);
await cancelUserDataDeletion({ userId: params.userId, startedAt: params.startedAt }, cancel);
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
} finally {
await runtime.close();
}
});
});

View file

@ -1,4 +1,5 @@
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import { logger } from '@librechat/data-schemas';
import type { TCheckpointerConfig, MCPOptions } from 'librechat-data-provider';
import type { AllMethods, IMongoFile } from '@librechat/data-schemas';
@ -47,6 +48,7 @@ type ErasureMethods = Pick<
| 'deleteSchedulesByUser'
| 'deleteUserChatProjects'
| 'deleteUserToolFavorites'
| 'cancelUserToolFavoriteDeletion'
| 'deleteRefreshTokenBridges'
| 'deleteAllAgentQueuedTurnsForUser'
>;
@ -99,10 +101,76 @@ export interface UserErasureParams {
userId: string;
tenantId?: string;
checkpointer?: TCheckpointerConfig;
/** UUID owned by this deletion attempt, distinct from the Mongo ownership timestamp. */
deletionAttemptId?: string;
/** Operator option to keep the transaction ledger; account self-deletion always erases it. */
keepTransactions?: boolean;
}
type UserDeletionCancellationParams = {
userId: string;
startedAt: Date;
deletionAttemptId?: string;
};
type UserDeletionCancellationDeps = Pick<
AllMethods,
| 'isAgentTriggerUserDeletionOwner'
| 'cancelAgentTriggerUserDeletion'
| 'cancelUserToolFavoriteDeletion'
>;
export interface UserDeletionAttempt {
readonly id: string;
adopt(
userId: string,
startedAt: Date,
deps: Pick<AllMethods, 'getAgentTriggerUserDeletionAttemptId'>,
): Promise<void>;
commit<T>(operation: () => Promise<T>): Promise<T>;
cancel(params: UserDeletionCancellationParams, deps: UserDeletionCancellationDeps): Promise<void>;
}
/** A started user delete may still commit after an error or a lost acknowledgement. */
export function createUserDeletionAttempt(): UserDeletionAttempt {
let commitStarted = false;
let identityKnown = true;
let attemptId: string = randomUUID();
return {
get id() {
return attemptId;
},
async adopt(userId, startedAt, deps) {
identityKnown = false;
const stored = await deps.getAgentTriggerUserDeletionAttemptId(userId, startedAt);
if (stored === null) throw new Error('User deletion attempt identity unavailable');
attemptId = stored;
identityKnown = true;
},
async commit<T>(operation: () => Promise<T>): Promise<T> {
commitStarted = true;
return operation();
},
async cancel(params, deps) {
if (!commitStarted && identityKnown) await cancelUserDataDeletion(params, deps);
},
};
}
/** Keep Mongo ownership retryable until this attempt's favorites fence is released. */
export async function cancelUserDataDeletion(
params: UserDeletionCancellationParams,
deps: UserDeletionCancellationDeps,
): Promise<void> {
const { userId, startedAt, deletionAttemptId } = params;
if (deletionAttemptId !== undefined && deps.cancelUserToolFavoriteDeletion) {
if ((await deps.isAgentTriggerUserDeletionOwner(userId, startedAt, deletionAttemptId)) !== true)
return;
await deps.cancelUserToolFavoriteDeletion(userId, deletionAttemptId);
}
if (deletionAttemptId === undefined) await deps.cancelAgentTriggerUserDeletion(userId, startedAt);
else await deps.cancelAgentTriggerUserDeletion(userId, startedAt, deletionAttemptId);
}
export type MCPServerCleanupDeps = Pick<
UserErasureDeps,
| 'findUserErasableMCPServers'
@ -307,7 +375,9 @@ export async function eraseUserData(
deps.deleteAllUserPluginAuths(userId),
deps.deleteToolCalls(userId),
deps.deleteConversationTags({ user: userId }),
deps.deleteUserToolFavorites(userId),
params.deletionAttemptId === undefined
? deps.deleteUserToolFavorites(userId)
: deps.deleteUserToolFavorites(userId, params.deletionAttemptId),
deps.deleteAllUserMemories(userId),
]);
await deps.deleteSharedLinks(userId);

View file

@ -102,6 +102,8 @@ describe('createErasureResolver', () => {
const { deps } = createDeps({
db: {
deleteUserAgents: jest.fn(record('agents')),
deleteUserToolFavorites: jest.fn(record('favorites')),
cancelUserToolFavoriteDeletion: jest.fn(record('cancelFavorites')),
findMCPServerOAuthTokenUserIds: jest.fn(record('tokens')),
findMCPServerViewerEntries: jest.fn(record('viewers')),
findMCPServerViewerUserIds: jest.fn(record('viewerUsers')),
@ -118,6 +120,8 @@ describe('createErasureResolver', () => {
await tenantStorage.run({ tenantId: 'admin-tenant' }, async () => {
await erasure.deleteUserAgents('user-1');
await erasure.deleteUserToolFavorites('user-1', 'attempt');
await erasure.cancelUserToolFavoriteDeletion?.('user-1', 'attempt');
await erasure.findMCPServerOAuthTokenUserIds?.('srv');
await erasure.findMCPServerViewerEntries?.('srv-id');
await erasure.findMCPServerViewerUserIds?.([]);
@ -130,7 +134,7 @@ describe('createErasureResolver', () => {
await erasure.mcpOAuth?.disconnectUser?.('other', 'srv');
});
expect(Object.keys(seen)).toHaveLength(10);
expect(Object.keys(seen)).toHaveLength(12);
expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true);
});

View file

@ -106,6 +106,16 @@ export function createErasureResolver(deps: ErasureResolverDeps) {
...deps.db,
openCheckpointDeletion: deps.openCheckpointDeletion,
deleteSharedLinks: deps.deleteSharedLinks,
deleteUserToolFavorites: (userId, deletionAttemptId) =>
inTargetTenant(async () =>
deletionAttemptId === undefined
? deps.db.deleteUserToolFavorites(userId)
: deps.db.deleteUserToolFavorites(userId, deletionAttemptId),
),
cancelUserToolFavoriteDeletion: (userId, deletionAttemptId) =>
inTargetTenant(async () =>
deps.db.cancelUserToolFavoriteDeletion?.(userId, deletionAttemptId),
),
deleteUserAgents: (userId) => inTargetTenant(async () => deps.db.deleteUserAgents(userId)),
findMCPServerOAuthTokenUserIds: (serverName) =>
inTargetTenant(async () => deps.db.findMCPServerOAuthTokenUserIds(serverName)),

View file

@ -179,6 +179,7 @@ export type DeploymentOnlyConfigReason = 'startup' | 'operator' | 'unused';
* a stored override had no effect rather than a dangerous one.
*/
export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly<Record<string, DeploymentOnlyConfigReason>> = {
persistence: 'operator',
'telemetry.stackTraceLimit': 'operator',
'telemetry.rum': 'operator',
clientLimits: 'operator',
@ -4666,7 +4667,35 @@ export const toolCallPreviewsConfigSchema = z.object({
export type TToolCallPreviewsConfig = z.infer<typeof toolCallPreviewsConfigSchema>;
export const persistenceConfigSchema = z
.object({
favorites: z.enum(['mongo', 'postgres']).default('mongo'),
postgres: z
.object({
connectionStringEnv: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/),
max: z.number().int().min(1).max(100).default(10),
maxWaiting: z.number().int().min(1).max(1000).default(100),
acquireTimeoutMs: z.number().int().min(1).max(60000).default(5000),
queryTimeoutMs: z.number().int().min(1).max(300000).default(15000),
idleTimeoutMs: z.number().int().min(1).max(300000).default(30000),
})
.strict()
.optional(),
})
.strict()
.default({ favorites: 'mongo' })
.superRefine((config, context) => {
if (config.favorites === 'postgres' && !config.postgres) {
context.addIssue({
code: z.ZodIssueCode.custom,
path: ['postgres'],
message: 'PostgreSQL configuration is required',
});
}
});
export const configSchema = z.object({
persistence: persistenceConfigSchema,
version: z.string(),
telemetry: telemetrySchema.optional(),
clientLimits: clientLimitsSchema.optional(),

View file

@ -0,0 +1,33 @@
import { configSchema, persistenceConfigSchema, DEPLOYMENT_ONLY_CONFIG_PATHS } from './config';
describe('deployment persistence configuration', () => {
it('defaults existing deployments to Mongo and keeps selection deployment-only', () => {
expect(configSchema.parse({ version: '1.3.1' }).persistence).toEqual({ favorites: 'mongo' });
expect(DEPLOYMENT_ONLY_CONFIG_PATHS.persistence).toBe('operator');
});
it('accepts bounded pool settings and an environment-variable name, never a DSN', () => {
const config = persistenceConfigSchema.parse({
favorites: 'postgres',
postgres: { connectionStringEnv: 'LIBRECHAT_POSTGRES_DSN' },
});
expect(config.postgres).toMatchObject({
max: 10,
maxWaiting: 100,
acquireTimeoutMs: 5000,
queryTimeoutMs: 15000,
});
expect(
persistenceConfigSchema.safeParse({
favorites: 'postgres',
postgres: { connectionStringEnv: 'postgres://secret' },
}).success,
).toBe(false);
});
it.each([
{ favorites: 'postgres' },
{ favorites: 'invalid' },
{ favorites: 'postgres', postgres: { connectionStringEnv: 'PG', max: 0 } },
])('refuses an explicit invalid selection: %j', (config) => {
expect(persistenceConfigSchema.safeParse(config).success).toBe(false);
});
});

View file

@ -0,0 +1,9 @@
import config from './jest.config.mjs';
export default {
...config,
testMatch: ['<rootDir>/src/postgres/*.spec.ts'],
globalSetup: undefined,
testResultsProcessor: undefined,
maxWorkers: 1,
};

View file

@ -47,7 +47,8 @@
"test:quality:favorites": "jest --config jest.favorite.config.mjs --coverage && node ../../scripts/crap.mts coverage/favorites/coverage-final.json src/methods/favorite.ts",
"test:contracts:leases": "jest --config jest.lease.config.mjs --coverage=false",
"test:mutations:leases": "cd ../.. && stryker run packages/data-schemas/stryker.lease.config.mjs",
"test:quality:leases": "jest --config jest.lease.config.mjs --coverage && node ../../scripts/crap.mts coverage/leases/coverage-final.json src/methods/resourceLock.ts"
"test:quality:leases": "jest --config jest.lease.config.mjs --coverage && node ../../scripts/crap.mts coverage/leases/coverage-final.json src/methods/resourceLock.ts",
"test:postgres": "jest --config jest.postgres.config.mjs --coverage=false --runInBand"
},
"repository": {
"type": "git",
@ -67,13 +68,15 @@
"micromark-extension-directive": "^3.0.1",
"micromark-extension-gfm": "^3.0.0",
"micromark-extension-llm-math": "^3.1.0",
"micromark-util-decode-string": "^2.0.0"
"micromark-util-decode-string": "^2.0.0",
"pg": "^8.23.1"
},
"devDependencies": {
"@stryker-mutator/core": "10.1.1",
"@stryker-mutator/jest-runner": "10.1.1",
"@types/jest": "^29.5.2",
"@types/node": "^24.12.4",
"@types/pg": "^8.23.1",
"jest": "^30.2.0",
"jest-junit": "^17.0.0",
"mongodb-memory-server": "^11.3.0",

View file

@ -47,6 +47,7 @@ export type {
} from './methods/response';
export {
createMethods,
createToolFavoriteMethods,
createChannelIdentityStorage,
createChannelFeedbackStorage,
ChannelFeedbackAuthorityError,
@ -182,3 +183,10 @@ export type { SearchTokenBackfillResult } from './migrations';
export { createBrowserSetupStore, createBrowserSetupBackend } from './methods/browserSetup';
export type * from './types/browserSetup';
export type * from './types/codeEnvironmentSetup';
export * from './postgres/backend';
export * from './postgres/pool';
export * from './postgres/migrations';
export * from './postgres/favorite';
export * from './postgres/import';
export * from './postgres/mongo';

View file

@ -19,12 +19,9 @@ function capError(): Error {
return error;
}
export function createToolFavoriteMethods(mongoose: typeof import('mongoose')): {
getToolFavorites: (userId: string) => Promise<t.IToolFavoriteLean[]>;
addToolFavorite: (params: t.ToolFavoriteParams) => Promise<t.AddToolFavoriteResult>;
removeToolFavorite: (params: t.ToolFavoriteParams) => Promise<t.RemoveToolFavoriteResult>;
deleteUserToolFavorites: (userId: string) => Promise<number>;
} {
export function createToolFavoriteMethods(
mongoose: typeof import('mongoose'),
): t.ToolFavoriteMethods {
async function getToolFavorites(userId: string): Promise<t.IToolFavoriteLean[]> {
const ToolFavorite = mongoose.models.ToolFavorite;
return ToolFavorite.find({ user: userId })
@ -96,4 +93,4 @@ export function createToolFavoriteMethods(mongoose: typeof import('mongoose')):
};
}
export type ToolFavoriteMethods = ReturnType<typeof createToolFavoriteMethods>;
export type { ToolFavoriteMethods } from '~/types/favorite';

View file

@ -368,6 +368,7 @@ export type AllMethods = PermissionPrincipalStorage &
/** Dependencies injected from the api layer into createMethods */
export interface CreateMethodsDeps {
toolFavorites?: ToolFavoriteMethods;
getChannelStorageConfig?: () => Promise<Partial<ChannelStorageConfig>>;
/** Matches a model name to a canonical key. From @librechat/api. */
matchModelName?: (model: string, endpoint?: string) => string | undefined;
@ -575,7 +576,7 @@ export function createMethods(
...createFileMethods(mongoose),
...createFileStorageMethods(mongoose),
...createMemoryMethods(mongoose),
...createToolFavoriteMethods(mongoose),
...(deps.toolFavorites ?? createToolFavoriteMethods(mongoose)),
...createAgentCategoryMethods(mongoose),
...createAgentApiKeyMethods(mongoose),
...createAgentChannelMethods(mongoose, deps.getChannelStorageConfig),
@ -730,3 +731,5 @@ export type {
};
export { recordAgentEventActorReceiptMetric, setAgentEventActorReceiptMetricObserver };
export { createToolFavoriteMethods } from './favorite';

View file

@ -1,4 +1,5 @@
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import { MongoMemoryServer } from 'mongodb-memory-server';
import {
CacheKeys,
@ -1556,6 +1557,111 @@ describe('User Methods - Database Tests', () => {
expect(found).toBeNull();
});
test.each(['cancel', 'commit'] as const)(
'rejects an old queued delete after an offline handoff and %s',
async (outcome) => {
const user = await User.create({
email: `delayed-${outcome}@example.com`,
provider: 'local',
});
const id = user._id.toString();
const attemptId = randomUUID();
const startedAt = new Date('2026-01-01');
const recoveredAt = new Date(startedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
await methods.beginAgentTriggerUserDeletion(id, startedAt, attemptId);
let release!: () => void;
let entered!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const pendingWrite = new Promise<void>((resolve) => {
entered = resolve;
});
const original = User.collection.deleteOne.bind(User.collection);
const write = jest
.spyOn(User.collection, 'deleteOne')
.mockImplementationOnce(async (filter, options) => {
entered();
await gate;
return original(filter, options);
});
const pendingDelete = methods.deleteUserById(id, { startedAt, attemptId });
try {
await pendingWrite;
expect(write.mock.calls[0][0]).toMatchObject({
agentTriggerDeletionStartedAt: startedAt,
agentTriggerDeletionAttemptId: attemptId,
});
expect(
await methods.recoverStaleAgentTriggerUserDeletion(id, recoveredAt, randomUUID()),
).toBe('acquired');
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, recoveredAt)).toBe(
attemptId,
);
if (outcome === 'cancel') {
expect(await methods.cancelAgentTriggerUserDeletion(id, recoveredAt, attemptId)).toBe(
true,
);
} else {
expect(
await methods.deleteUserById(id, { startedAt: recoveredAt, attemptId }),
).toMatchObject({ deletedCount: 1 });
}
release();
expect(await pendingDelete).toMatchObject({ deletedCount: 0 });
expect(await User.exists({ _id: id })).toEqual(
outcome === 'cancel' ? { _id: user._id } : null,
);
} finally {
release();
await pendingDelete;
write.mockRestore();
}
},
);
test('rejects the wrong UUID or Date and commits only the current owner', async () => {
const user = await User.create({ email: 'fenced-delete@example.com', provider: 'local' });
const id = user._id.toString();
const attemptId = randomUUID();
const startedAt = new Date();
await methods.beginAgentTriggerUserDeletion(id, startedAt, attemptId);
expect(
await methods.deleteUserById(id, { startedAt, attemptId: randomUUID() }),
).toMatchObject({ deletedCount: 0 });
expect(
await methods.deleteUserById(id, {
startedAt: new Date(startedAt.getTime() + 1),
attemptId,
}),
).toMatchObject({ deletedCount: 0 });
expect(await methods.isAgentTriggerUserDeletionOwner(id, startedAt, attemptId)).toBe(true);
expect(await methods.deleteUserById(id, { startedAt, attemptId })).toMatchObject({
deletedCount: 1,
});
expect(
await methods.recoverStaleAgentTriggerUserDeletion(
id,
new Date(startedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1),
randomUUID(),
),
).toBe('missing');
});
test('rejects invalid commit fences before any cleanup or delete', async () => {
const id = new mongoose.Types.ObjectId().toString();
const write = jest.spyOn(User, 'deleteOne');
const grants = jest.spyOn(mongoose.models.ToolApprovalGrant, 'deleteMany');
await expect(
methods.deleteUserById(id, { startedAt: new Date(Number.NaN), attemptId: randomUUID() }),
).rejects.toThrow('startedAt must be a valid Date');
await expect(
methods.deleteUserById(id, { startedAt: new Date(), attemptId: '' }),
).rejects.toThrow('Invalid user deletion attempt ID');
expect(write).not.toHaveBeenCalled();
expect(grants).not.toHaveBeenCalled();
});
test('should return zero count for non-existent user', async () => {
const fakeId = new mongoose.Types.ObjectId();
const result = await methods.deleteUserById(fakeId.toString());
@ -1590,6 +1696,88 @@ describe('User Methods - Database Tests', () => {
await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);
});
test('proves the exact live owner from primary without releasing its fence', async () => {
const user = await User.create({ email: 'deletion-owner@example.com', provider: 'local' });
const userId = user._id.toString();
const startedAt = new Date('2026-11-10T12:00:00.000Z');
await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(false);
await methods.beginAgentTriggerUserDeletion(userId, startedAt);
const read = jest.spyOn(mongoose.Query.prototype, 'read');
await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(true);
expect(read).toHaveBeenCalledWith('primary');
await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);
await expect(
methods.isAgentTriggerUserDeletionOwner(userId, new Date(startedAt.getTime() + 1)),
).resolves.toBe(false);
await methods.deleteUserById(userId);
await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(false);
});
test('propagates ownership-read errors instead of treating them as absent ownership', async () => {
const failure = new Error('ownership read unavailable');
jest.spyOn(User.collection, 'findOne').mockRejectedValueOnce(failure);
await expect(
methods.isAgentTriggerUserDeletionOwner(
new mongoose.Types.ObjectId().toString(),
new Date(),
),
).rejects.toThrow(failure);
});
test('persists identity across repeated offline recoveries and fences stale Date and UUID owners', async () => {
const user = await User.create({ email: 'durable-attempt@example.com', provider: 'local' });
const id = user._id.toString();
const attempt = randomUUID();
const other = randomUUID();
const first = new Date('2026-01-01');
const second = new Date(first.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
const third = new Date(second.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
expect(await methods.beginAgentTriggerUserDeletion(id, first, attempt)).toBe('acquired');
expect((await User.findById(id).lean())?.agentTriggerDeletionAttemptId).toBeUndefined();
expect(await methods.recoverStaleAgentTriggerUserDeletion(id, second, other)).toBe(
'acquired',
);
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, first)).toBeNull();
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBe(attempt);
expect(await methods.cancelAgentTriggerUserDeletion(id, first, attempt)).toBe(false);
expect(await methods.cancelAgentTriggerUserDeletion(id, second, other)).toBe(false);
expect(await methods.isAgentTriggerUserDeletionOwner(id, second, other)).toBe(false);
// Recovery can itself crash before adopting the identity; the next process still finds it.
expect(await methods.recoverStaleAgentTriggerUserDeletion(id, third, randomUUID())).toBe(
'acquired',
);
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, third)).toBe(attempt);
expect(await methods.cancelAgentTriggerUserDeletion(id, third, attempt)).toBe(true);
expect(
await User.findById(id)
.select('+agentTriggerDeletionAttemptId +agentTriggerDeletionStartedAt')
.lean(),
).toEqual(expect.not.objectContaining({ agentTriggerDeletionAttemptId: expect.anything() }));
expect(await methods.beginAgentTriggerUserDeletion(id, third, other)).toBe('acquired');
expect(await methods.cancelAgentTriggerUserDeletion(id, third, attempt)).toBe(false);
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, third)).toBe(other);
});
test('initializes only a legacy missing UUID and clears stray identity on a no-token acquisition', async () => {
const user = await User.create({ email: 'legacy-attempt@example.com', provider: 'local' });
const id = user._id.toString();
const first = new Date('2026-01-01');
const second = new Date(first.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
const fallback = randomUUID();
await methods.beginAgentTriggerUserDeletion(id, first);
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, first)).toBeNull();
expect(await methods.recoverStaleAgentTriggerUserDeletion(id, second, fallback)).toBe(
'acquired',
);
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBe(fallback);
await methods.cancelAgentTriggerUserDeletion(id, second, fallback);
await User.updateOne({ _id: id }, { $set: { agentTriggerDeletionAttemptId: randomUUID() } });
await methods.beginAgentTriggerUserDeletion(id, second);
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBeNull();
await expect(methods.beginAgentTriggerUserDeletion(id, second, '')).rejects.toThrow(
'attempt',
);
});
test('reports a missing principal without creating a deletion fence', async () => {
const userId = new mongoose.Types.ObjectId().toString();

View file

@ -183,18 +183,33 @@ export function createUserMethods(
beginAgentTriggerUserDeletion: (
userId: string,
startedAt: Date,
attemptId?: string,
) => Promise<'acquired' | 'in_progress' | 'missing'>;
recoverStaleAgentTriggerUserDeletion: (
userId: string,
recoveredAt: Date,
legacyAttemptId?: string,
) => Promise<'acquired' | 'in_progress' | 'missing'>;
cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;
getAgentTriggerUserDeletionAttemptId: (userId: string, startedAt: Date) => Promise<string | null>;
isAgentTriggerUserDeletionOwner: (
userId: string,
startedAt: Date,
attemptId?: string,
) => Promise<boolean>;
cancelAgentTriggerUserDeletion: (
userId: string,
startedAt: Date,
attemptId?: string,
) => Promise<boolean>;
isAgentTriggerPrincipalActive: (userId: string) => Promise<boolean>;
fenceSubagentAdmission: (userId: string, token: string, fencedUntil: Date) => Promise<void>;
renewSubagentAdmission: (userId: string, token: string, fencedUntil: Date) => Promise<boolean>;
releaseSubagentAdmission: (userId: string, token: string) => Promise<void>;
isSubagentOwnerAdmissible: (userId: string) => Promise<boolean>;
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
deleteUserById: (
userId: string,
fence?: { startedAt: Date; attemptId: string },
) => Promise<UserDeleteResult>;
updateUserPlugins: (
userId: string,
plugins: string[] | undefined,
@ -645,7 +660,16 @@ export function createUserMethods(
/**
* Delete a user by their unique ID.
*/
async function deleteUserById(userId: string): Promise<UserDeleteResult> {
async function deleteUserById(
userId: string,
fence?: { startedAt: Date; attemptId: string },
): Promise<UserDeleteResult> {
if (fence !== undefined) {
if (!(fence?.startedAt instanceof Date) || !Number.isFinite(fence.startedAt.getTime())) {
throw new TypeError('startedAt must be a valid Date');
}
validateDeletionAttemptId(fence.attemptId);
}
try {
const User = mongoose.models.User;
const owner = await User.findById(userId).select('tenantId').lean<Pick<IUser, 'tenantId'>>();
@ -664,7 +688,15 @@ export function createUserMethods(
);
}
await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId });
const result = await User.deleteOne({ _id: userId });
const result = await User.deleteOne({
_id: userId,
...(fence === undefined
? {}
: {
agentTriggerDeletionStartedAt: fence.startedAt,
agentTriggerDeletionAttemptId: fence.attemptId,
}),
});
if (result.deletedCount === 0) {
return { deletedCount: 0, message: 'No user found with that ID.' };
}
@ -676,18 +708,32 @@ export function createUserMethods(
}
}
function validateDeletionAttemptId(attemptId: string): void {
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(attemptId)) {
throw new TypeError('Invalid user deletion attempt ID');
}
}
/** Establishes the durable admission fence used while account deletion drains triggers. */
async function beginAgentTriggerUserDeletion(
userId: string,
startedAt: Date,
attemptId?: string,
): Promise<'acquired' | 'in_progress' | 'missing'> {
if (!(startedAt instanceof Date) || !Number.isFinite(startedAt.getTime())) {
throw new TypeError('startedAt must be a valid Date');
}
if (attemptId !== undefined) validateDeletionAttemptId(attemptId);
const User = mongoose.models.User;
const result = await User.updateOne(
{ _id: userId, agentTriggerDeletionStartedAt: { $exists: false } },
{ $set: { agentTriggerDeletionStartedAt: startedAt } },
{
$set: {
agentTriggerDeletionStartedAt: startedAt,
...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }),
},
...(attemptId === undefined ? { $unset: { agentTriggerDeletionAttemptId: 1 } } : {}),
},
);
if (result.modifiedCount === 1) {
await invalidateAuthUserDocCache(userId);
@ -701,12 +747,34 @@ export function createUserMethods(
async function recoverStaleAgentTriggerUserDeletion(
userId: string,
recoveredAt: Date,
legacyAttemptId?: string,
): Promise<'acquired' | 'in_progress' | 'missing'> {
if (!(recoveredAt instanceof Date) || !Number.isFinite(recoveredAt.getTime())) {
throw new TypeError('recoveredAt must be a valid Date');
}
const User = mongoose.models.User;
if (legacyAttemptId !== undefined) validateDeletionAttemptId(legacyAttemptId);
const staleBefore = new Date(recoveredAt.getTime() - USER_DELETION_FENCE_STALE_MS);
// Two ordinary CAS updates keep DocumentDB compatibility and never overwrite a stored UUID.
if (legacyAttemptId !== undefined) {
const legacy = await User.updateOne(
{
_id: userId,
agentTriggerDeletionStartedAt: { $lte: staleBefore },
agentTriggerDeletionAttemptId: { $exists: false },
},
{
$set: {
agentTriggerDeletionStartedAt: recoveredAt,
agentTriggerDeletionAttemptId: legacyAttemptId,
},
},
);
if (legacy.modifiedCount === 1) {
await invalidateAuthUserDocCache(userId);
return 'acquired';
}
}
const result = await User.updateOne(
{ _id: userId, agentTriggerDeletionStartedAt: { $lte: staleBefore } },
{ $set: { agentTriggerDeletionStartedAt: recoveredAt } },
@ -718,12 +786,55 @@ export function createUserMethods(
return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress';
}
async function getAgentTriggerUserDeletionAttemptId(
userId: string,
startedAt: Date,
): Promise<string | null> {
const user = await mongoose.models.User.findOne({
_id: userId,
agentTriggerDeletionStartedAt: startedAt,
})
.select({ agentTriggerDeletionAttemptId: 1, _id: 0 })
.read('primary')
.lean<Pick<IUser, 'agentTriggerDeletionAttemptId'>>();
const attemptId = user?.agentTriggerDeletionAttemptId;
if (attemptId == null) return null;
validateDeletionAttemptId(attemptId);
return attemptId;
}
/** Reads current ownership without releasing the durable retry anchor. */
async function isAgentTriggerUserDeletionOwner(
userId: string,
startedAt: Date,
attemptId?: string,
): Promise<boolean> {
if (attemptId !== undefined) validateDeletionAttemptId(attemptId);
const User = mongoose.models.User;
return (
(await User.exists({
_id: userId,
agentTriggerDeletionStartedAt: startedAt,
...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }),
}).read('primary')) != null
);
}
/** Releases only the account-deletion attempt that owns this exact fence. */
async function cancelAgentTriggerUserDeletion(userId: string, startedAt: Date): Promise<boolean> {
async function cancelAgentTriggerUserDeletion(
userId: string,
startedAt: Date,
attemptId?: string,
): Promise<boolean> {
if (attemptId !== undefined) validateDeletionAttemptId(attemptId);
const User = mongoose.models.User;
const result = await User.updateOne(
{ _id: userId, agentTriggerDeletionStartedAt: startedAt },
{ $unset: { agentTriggerDeletionStartedAt: 1 } },
{
_id: userId,
agentTriggerDeletionStartedAt: startedAt,
...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }),
},
{ $unset: { agentTriggerDeletionStartedAt: 1, agentTriggerDeletionAttemptId: 1 } },
);
if (result.modifiedCount === 1) {
await invalidateAuthUserDocCache(userId);
@ -1088,6 +1199,8 @@ export function createUserMethods(
generateToken,
beginAgentTriggerUserDeletion,
recoverStaleAgentTriggerUserDeletion,
getAgentTriggerUserDeletionAttemptId,
isAgentTriggerUserDeletionOwner,
cancelAgentTriggerUserDeletion,
isAgentTriggerPrincipalActive,
fenceSubagentAdmission,

View file

@ -0,0 +1,117 @@
import mongoose from 'mongoose';
import type { PostgresDatabase, PostgresOptions } from './pool';
import type { PostgresFavoriteBackend } from './backend';
import { createPostgresTestDatabase } from './postgres.helper';
import { createPostgresFavoriteBackend } from './backend';
import { migratePostgres } from './migrations';
import { importFavorites } from './import';
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
const userId = '0123456789abcdef01234567';
postgres('PostgreSQL favorite backend composition', () => {
let db: PostgresDatabase | undefined;
let backend: PostgresFavoriteBackend | undefined;
let options: PostgresOptions;
beforeEach(async () => {
({ db, options } = await createPostgresTestDatabase());
await migratePostgres(db);
});
afterEach(async () => {
try {
await backend?.close();
} finally {
backend = undefined;
await db?.close();
db = undefined;
}
});
async function verifyEmpty(): Promise<void> {
await importFavorites(
db!,
async function* () {
/* verified empty source */
},
{ offline: true, batchSize: 1, verifiedEmpty: true },
);
}
async function readiness() {
return (await db!.query('SELECT * FROM tool_favorite_readiness')).rows;
}
it('does not activate verified data when constructed or read before initialization', async () => {
await verifyEmpty();
const before = await readiness();
expect(before).toEqual([expect.objectContaining({ verified: true, activated: false })]);
backend = createPostgresFavoriteBackend(options);
expect(await backend.methods.getToolFavorites(userId)).toEqual([]);
expect(await readiness()).toEqual(before);
});
it('initializes verified-empty data by activating readiness without touching Mongo', async () => {
await verifyEmpty();
const before = await readiness();
const connect = jest.spyOn(mongoose, 'connect');
const collection = jest.spyOn(mongoose.connection, 'collection');
const model = jest.spyOn(mongoose, 'model');
backend = createPostgresFavoriteBackend(options);
await backend.initialize();
expect(await readiness()).toEqual([{ ...before[0], activated: true }]);
expect(await backend.methods.getToolFavorites(userId)).toEqual([]);
expect(connect).not.toHaveBeenCalled();
expect(collection).not.toHaveBeenCalled();
expect(model).not.toHaveBeenCalled();
});
it('rejects unverified data without activating it', async () => {
backend = createPostgresFavoriteBackend(options);
await expect(backend.initialize()).rejects.toThrow('PostgreSQL favorites data not verified');
expect(await readiness()).toEqual([
expect.objectContaining({ verified: false, activated: false }),
]);
});
it('exposes the native favorite add, read, and removal contract', async () => {
await verifyEmpty();
backend = createPostgresFavoriteBackend(options);
await backend.initialize();
const favorite = { userId, itemType: 'tool', itemId: 'backend-tool' } as const;
await expect(backend.methods.addToolFavorite(favorite)).resolves.toEqual({
ok: true,
added: true,
});
await expect(backend.methods.getToolFavorites(userId)).resolves.toEqual([
{ itemType: 'tool', itemId: 'backend-tool' },
]);
expect((await db!.query('SELECT count(*) FROM tool_favorites')).rows).toEqual([{ count: '1' }]);
await expect(backend.methods.removeToolFavorite(favorite)).resolves.toEqual({
ok: true,
removed: true,
});
await expect(backend.methods.getToolFavorites(userId)).resolves.toEqual([]);
});
it('closes idempotently and rejects methods and initialization after close', async () => {
await verifyEmpty();
backend = createPostgresFavoriteBackend(options);
await backend.initialize();
const closing = backend.close();
expect(backend.close()).toBe(closing);
await closing;
await expect(backend.close()).resolves.toBeUndefined();
await expect(backend.methods.getToolFavorites(userId)).rejects.toThrow('admission');
await expect(backend.initialize()).rejects.toThrow('admission');
});
it('can be closed idempotently after initialization fails', async () => {
backend = createPostgresFavoriteBackend(options);
await expect(backend.initialize()).rejects.toThrow('not verified');
await expect(backend.close()).resolves.toBeUndefined();
await expect(backend.close()).resolves.toBeUndefined();
await expect(backend.methods.getToolFavorites(userId)).rejects.toThrow('admission');
});
});

View file

@ -0,0 +1,20 @@
import type { ToolFavoriteMethods } from '~/types/favorite';
import type { PostgresOptions } from './pool';
import { createPostgresFavoriteMethods } from './favorite';
import { assertPostgresReady } from './migrations';
import { createPostgresDatabase } from './pool';
export interface PostgresFavoriteBackend {
methods: ToolFavoriteMethods;
initialize(): Promise<void>;
close(): Promise<void>;
}
export function createPostgresFavoriteBackend(options: PostgresOptions): PostgresFavoriteBackend {
const db = createPostgresDatabase(options);
return {
methods: createPostgresFavoriteMethods(db),
initialize: () => assertPostgresReady(db),
close: () => db.close(),
};
}

View file

@ -0,0 +1,303 @@
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import type { FavoriteHarness } from '~/contracts/favorite';
import type { FavoriteSnapshotRow } from './import';
import type { PostgresDatabase } from './pool';
import type { PostgresOptions } from './pool';
import { importFavorites, readPostgresFavorites, fingerprintFavorites } from './import';
import { tenantStorage, SYSTEM_TENANT_ID } from '~/config/tenantContext';
import { migratePostgres, assertPostgresReady } from './migrations';
import { describeFavoriteConformance } from '~/contracts/favorite';
import { createPostgresTestDatabase } from './postgres.helper';
import { createPostgresFavoriteMethods } from './favorite';
import { resetTenantStrictCache } from '~/tenant/policy';
import { createPostgresDatabase } from './pool';
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
let db: PostgresDatabase;
const users = ['0123456789abcdef01234567', '1123456789abcdef01234567'] as const;
let options: PostgresOptions;
const harness: FavoriteHarness = {
name: 'PostgreSQL 16',
users,
async setup() {
({ db, options } = await createPostgresTestDatabase());
await migratePostgres(db);
return createPostgresFavoriteMethods(db);
},
async teardown() {
await db?.close();
},
async reset() {
jest.restoreAllMocks();
await db.query('TRUNCATE tool_favorites,tool_favorite_erasure');
await db.query(
'UPDATE tool_favorite_readiness SET checkpoint=NULL, snapshot=NULL, verified=false, activated=false, imported=0',
);
},
async seed(rows) {
for (const row of rows) {
await db.query(
`INSERT INTO tool_favorites (id,user_id,item_type,item_id,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$5)`,
[
new mongoose.Types.ObjectId().toString(),
row.userId,
row.itemType,
row.itemId,
row.createdAt ?? new Date(),
],
);
}
},
inTenant: (tenantId, operation) => tenantStorage.run({ tenantId }, operation),
failNext(operation, error) {
if (operation === 'read' || operation === 'remove') {
jest.spyOn(db, 'query').mockRejectedValueOnce(error);
} else {
jest.spyOn(db, 'transaction').mockRejectedValueOnce(error);
}
},
};
postgres('real PostgreSQL favorites', () => {
describeFavoriteConformance(harness);
describe('adapter, migrations, import, and readiness', () => {
beforeAll(() => harness.setup());
afterAll(() => harness.teardown());
beforeEach(() => harness.reset());
const source = async function* () {
yield {
id: '2123456789abcdef01234567',
userId: users[0],
tenantId: 'tenant-a',
itemType: 'mcp',
itemId: 'server',
createdAt: new Date('2026-01-01'),
updatedAt: new Date('2026-01-02'),
} satisfies FavoriteSnapshotRow;
};
it('keeps schema migration idempotent, checksum-checked, and separate from data readiness', async () => {
await migratePostgres(db);
await expect(assertPostgresReady(db)).rejects.toThrow('not verified');
await db.query("UPDATE librechat_migrations SET checksum='changed'");
await expect(migratePostgres(db)).rejects.toThrow('checksum');
// Restore through the exact original checksum, not by rerunning altered SQL.
await db.query('DELETE FROM librechat_migrations');
await db.query('DROP TABLE tool_favorites,tool_favorite_erasure,tool_favorite_readiness');
await migratePostgres(db);
});
it('requires verified-empty approval, then activates and permanently refuses import replay', async () => {
const empty = async function* () {
/* verified empty source */
};
await expect(importFavorites(db, empty, { offline: true, batchSize: 2 })).rejects.toThrow(
'verified-empty',
);
await importFavorites(db, empty, { offline: true, batchSize: 2, verifiedEmpty: true });
await assertPostgresReady(db);
await expect(
importFavorites(db, empty, { offline: true, batchSize: 2, verifiedEmpty: true }),
).rejects.toThrow('activated');
});
it('preserves source identity, tenant, timestamps, and supports idempotent checkpoint replay', async () => {
await importFavorites(db, source, { offline: true, batchSize: 1 });
await importFavorites(db, source, { offline: true, batchSize: 1 });
expect(await fingerprintFavorites(readPostgresFavorites(db, 1))).toEqual(
await fingerprintFavorites(source),
);
await assertPostgresReady(db);
});
it('does not write in dry-run and refuses a false empty-source assertion', async () => {
await importFavorites(db, source, { offline: true, batchSize: 1, dryRun: true });
expect((await db.query('SELECT * FROM tool_favorites')).rowCount).toBe(0);
await expect(
importFavorites(db, source, { offline: true, batchSize: 1, verifiedEmpty: true }),
).rejects.toThrow('not empty');
});
it('rolls back a failed transaction', async () => {
await expect(
db.transaction(async (sql) => {
await sql.query("INSERT INTO tool_favorite_erasure VALUES ($1,'*')", [users[0]]);
throw new Error('fault');
}),
).rejects.toThrow('fault');
expect((await db.query('SELECT * FROM tool_favorite_erasure')).rowCount).toBe(0);
});
it('denies missing strict context and distinguishes system all-tenants from tenantless', async () => {
const methods = createPostgresFavoriteMethods(db);
await harness.inTenant('tenant-a', () =>
methods.addToolFavorite({ userId: users[0], itemType: 'skill', itemId: 'a' }),
);
expect(
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
methods.getToolFavorites(users[0]),
),
).toHaveLength(1);
const old = process.env.TENANT_ISOLATION_STRICT;
process.env.TENANT_ISOLATION_STRICT = 'true';
resetTenantStrictCache();
try {
await expect(methods.getToolFavorites(users[0])).rejects.toThrow('without tenant context');
} finally {
resetTenantStrictCache();
if (old == null) {
delete process.env.TENANT_ISOLATION_STRICT;
} else {
process.env.TENANT_ISOLATION_STRICT = old;
}
}
});
it('accepts ObjectId input and prevents late recreation after erasure', async () => {
const methods = createPostgresFavoriteMethods(db);
const params = {
userId: new mongoose.Types.ObjectId(users[0]),
itemType: 'tool',
itemId: 'x',
} as const;
await methods.addToolFavorite(params);
await methods.deleteUserToolFavorites(users[0]);
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
});
it('cancels only its UUID and scope, never another attempt or a permanent marker', async () => {
const methods = createPostgresFavoriteMethods(db);
const first = randomUUID();
const second = randomUUID();
const params = { userId: users[0], itemType: 'tool', itemId: 'retry' } as const;
await harness.inTenant('tenant-a', async () => {
await methods.deleteUserToolFavorites(users[0], first);
await methods.deleteUserToolFavorites(users[0], second);
});
await harness.inTenant('tenant-b', () =>
methods.cancelUserToolFavoriteDeletion!(users[0], first),
);
await harness.inTenant('tenant-a', async () => {
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
await methods.cancelUserToolFavoriteDeletion!(users[0], first);
await methods.cancelUserToolFavoriteDeletion!(users[0], first);
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
await methods.cancelUserToolFavoriteDeletion!(users[0], second);
await expect(methods.addToolFavorite(params)).resolves.toEqual({ ok: true, added: true });
await methods.deleteUserToolFavorites(users[0]);
await methods.cancelUserToolFavoriteDeletion!(users[0], first);
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
});
await expect(methods.cancelUserToolFavoriteDeletion!(users[0], '')).rejects.toThrow(
'attempt',
);
await expect(methods.deleteUserToolFavorites(users[0], '')).rejects.toThrow('attempt');
});
it('lets system recovery cancel its UUID across scopes without clearing successors or permanent markers', async () => {
const methods = createPostgresFavoriteMethods(db);
const owned = randomUUID();
const successor = randomUUID();
for (const tenantId of ['tenant-a', 'tenant-b', SYSTEM_TENANT_ID]) {
await tenantStorage.run({ tenantId }, () =>
methods.deleteUserToolFavorites(users[0], owned),
);
}
await harness.inTenant('tenant-b', () =>
methods.deleteUserToolFavorites(users[0], successor),
);
await methods.deleteUserToolFavorites(users[1]);
await harness.inTenant('tenant-a', () =>
methods.cancelUserToolFavoriteDeletion!(users[0], owned),
);
expect(
(
await db.query(
'SELECT scope FROM tool_favorite_erasure WHERE user_id=$1 AND attempt_id=$2 ORDER BY scope',
[users[0], owned],
)
).rows,
).toEqual([{ scope: '*' }, { scope: 'tenant:tenant-b' }]);
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
methods.cancelUserToolFavoriteDeletion!(users[0], owned),
);
expect(
(
await db.query(
'SELECT scope,attempt_id FROM tool_favorite_erasure WHERE user_id=$1 ORDER BY scope',
[users[0]],
)
).rows,
).toEqual([{ scope: 'tenant:tenant-b', attempt_id: successor }]);
expect(
(
await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [
users[1],
])
).rows,
).toEqual([{ attempt_id: '' }]);
await harness.inTenant('tenant-b', () =>
expect(
methods.addToolFavorite({ userId: users[0], itemType: 'tool', itemId: 'still-fenced' }),
).rejects.toThrow('erasure'),
);
});
it('keeps global markers fenced against tenant cancellation and across fresh adapters', async () => {
const methods = createPostgresFavoriteMethods(db);
const attempt = randomUUID();
const params = { userId: users[0], itemType: 'tool', itemId: 'global' } as const;
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
methods.deleteUserToolFavorites(users[0], attempt),
);
await harness.inTenant('tenant-a', () =>
methods.cancelUserToolFavoriteDeletion!(users[0], attempt),
);
await harness.inTenant('tenant-b', () =>
expect(createPostgresFavoriteMethods(db).addToolFavorite(params)).rejects.toThrow(
'erasure',
),
);
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
methods.cancelUserToolFavoriteDeletion!(users[0], attempt),
);
await expect(methods.addToolFavorite(params)).resolves.toEqual({ ok: true, added: true });
});
it('bounds concurrent cap admission and enforces uniqueness across tenants', async () => {
await harness.seed(
Array.from({ length: 99 }, (_, i) => ({
userId: users[0],
itemType: 'tool',
itemId: `${i}`,
})),
);
const methods = createPostgresFavoriteMethods(db);
await Promise.allSettled(
Array.from({ length: 8 }, (_, i) =>
methods.addToolFavorite({ userId: users[0], itemType: 'tool', itemId: `race-${i}` }),
),
);
expect(await methods.getToolFavorites(users[0])).toHaveLength(100);
});
it('bounds query duration, releases failed transactions, and rejects work after close', async () => {
const bounded = createPostgresDatabase({ ...options, queryTimeoutMs: 50 });
try {
await expect(
bounded.transaction((sql) => sql.query('SELECT pg_sleep(1)')),
).rejects.toThrow();
expect((await bounded.query('SELECT 1 AS value')).rows).toEqual([{ value: 1 }]);
} finally {
await bounded.close();
}
await expect(bounded.query('SELECT 1')).rejects.toThrow('admission');
});
it('refuses concurrent import writers using the pinned session lock', async () => {
const other = createPostgresDatabase(options);
try {
await other.session(async (sql) => {
await sql.query('SELECT pg_advisory_lock(17490261, hashtext(current_schema()))');
try {
await expect(
importFavorites(db, source, { offline: true, batchSize: 1 }),
).rejects.toThrow('Another');
} finally {
await sql.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))');
}
});
} finally {
await other.close();
}
});
});
});

View file

@ -0,0 +1,121 @@
import { randomBytes } from 'node:crypto';
import type { ToolFavoriteMethods, ToolFavoriteParams } from '~/types/favorite';
import type { PostgresDatabase, SqlExecutor } from './pool';
import { resolveTenantScope, tenantFilter, stampTenantOnDocument } from '~/tenant/policy';
import { MAX_TOOL_FAVORITES } from '~/methods/favorite';
function userKey(value: ToolFavoriteParams['userId']): string {
const user = String(value).toLowerCase();
if (!/^[0-9a-f]{24}$/.test(user)) {
throw new Error('Invalid favorite user ID');
}
return user;
}
function validateAttemptId(attemptId: string): void {
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(attemptId)) {
throw new Error('Invalid favorite deletion attempt ID');
}
}
export function createPostgresFavoriteMethods(db: PostgresDatabase): ToolFavoriteMethods {
function scope(): string | null {
return tenantFilter(resolveTenantScope('Query'))?.tenantId ?? null;
}
async function lock(sql: SqlExecutor, user: string): Promise<void> {
await sql.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [user]);
}
return {
async getToolFavorites(userId) {
const result = await db.query<{ itemType: ToolFavoriteParams['itemType']; itemId: string }>(
`SELECT item_type AS "itemType", item_id AS "itemId" FROM tool_favorites
WHERE user_id = $1 AND ($2::text IS NULL OR tenant_id = $2) ORDER BY created_at, id`,
[userKey(userId), scope()],
);
return result.rows;
},
async addToolFavorite({ userId, itemType, itemId }) {
const user = userKey(userId);
const tenant = scope();
const document: { tenantId?: string } = {};
stampTenantOnDocument(resolveTenantScope('Query'), document);
if (
!['builtin', 'tool', 'mcp', 'skill'].includes(itemType) ||
typeof itemId !== 'string' ||
itemId.length < 1 ||
itemId.length > 256
) {
throw new Error('Invalid favorite item');
}
return db.transaction(async (sql) => {
await lock(sql, user);
const erased = await sql.query(
'SELECT 1 FROM tool_favorite_erasure WHERE user_id = $1 AND (scope = $2 OR scope = $3) LIMIT 1',
[user, '*', tenant == null ? '*' : `tenant:${tenant}`],
);
if (erased.rowCount) {
throw new Error('Favorite account erasure in progress');
}
const exists = await sql.query(
'SELECT 1 FROM tool_favorites WHERE user_id=$1 AND item_type=$2 AND item_id=$3 AND ($4::text IS NULL OR tenant_id=$4)',
[user, itemType, itemId, tenant],
);
if (exists.rowCount) {
return { ok: true, added: false };
}
const count = await sql.query<{ count: string }>(
'SELECT count(*) FROM tool_favorites WHERE user_id=$1 AND ($2::text IS NULL OR tenant_id=$2)',
[user, tenant],
);
if (Number(count.rows[0].count) >= MAX_TOOL_FAVORITES) {
throw Object.assign(new Error(`Maximum of ${MAX_TOOL_FAVORITES} favorites reached`), {
code: 'MAX_FAVORITES_EXCEEDED',
limit: MAX_TOOL_FAVORITES,
});
}
await sql.query(
`INSERT INTO tool_favorites (id,user_id,tenant_id,item_type,item_id,created_at,updated_at)
VALUES ($1,$2,$3,$4,$5,now(),now()) ON CONFLICT (user_id,item_type,item_id) DO NOTHING`,
[randomBytes(12).toString('hex'), user, document.tenantId ?? null, itemType, itemId],
);
return { ok: true, added: true };
});
},
async removeToolFavorite({ userId, itemType, itemId }) {
const result = await db.query(
'DELETE FROM tool_favorites WHERE user_id=$1 AND item_type=$2 AND item_id=$3 AND ($4::text IS NULL OR tenant_id=$4)',
[userKey(userId), itemType, itemId, scope()],
);
return { ok: true, removed: (result.rowCount ?? 0) > 0 };
},
async cancelUserToolFavoriteDeletion(userId, deletionAttemptId) {
validateAttemptId(deletionAttemptId);
const user = userKey(userId);
const tenant = scope();
await db.transaction(async (sql) => {
await lock(sql, user);
await sql.query(
'DELETE FROM tool_favorite_erasure WHERE user_id=$1 AND ($2::text IS NULL OR scope=$2) AND attempt_id=$3',
[user, tenant == null ? null : `tenant:${tenant}`, deletionAttemptId],
);
});
},
async deleteUserToolFavorites(userId, deletionAttemptId) {
if (deletionAttemptId !== undefined) validateAttemptId(deletionAttemptId);
const user = userKey(userId);
const tenant = scope();
return db.transaction(async (sql) => {
await lock(sql, user);
await sql.query(
'INSERT INTO tool_favorite_erasure (user_id,scope,attempt_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING',
[user, tenant == null ? '*' : `tenant:${tenant}`, deletionAttemptId ?? ''],
);
const result = await sql.query(
'DELETE FROM tool_favorites WHERE user_id=$1 AND ($2::text IS NULL OR tenant_id=$2)',
[user, tenant],
);
return result.rowCount ?? 0;
});
},
};
}

View file

@ -0,0 +1,369 @@
import type { QueryResultRow } from 'pg';
import type { FavoriteSnapshotRow, FavoriteSnapshotSource } from './import';
import type { PostgresDatabase, SqlExecutor } from './pool';
import { fingerprintFavorites, importFavorites, readPostgresFavorites } from './import';
import { createPostgresTestDatabase } from './postgres.helper';
import { migratePostgres } from './migrations';
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
const userId = '0123456789abcdef01234567';
const options = { offline: true, batchSize: 2 };
const rows: FavoriteSnapshotRow[] = Array.from({ length: 5 }, (_, i) => ({
id: (i + 1).toString(16).padStart(24, '0'),
userId,
tenantId: i % 2 ? null : 'tenant-a',
itemType: 'mcp',
itemId: `server-${i}`,
createdAt: new Date('2026-01-01T01:02:03.123Z'),
updatedAt: new Date('2026-01-02T04:05:06.789Z'),
}));
const source = (data = rows): FavoriteSnapshotSource =>
async function* () {
yield* data;
};
postgres('real PostgreSQL snapshot recovery', () => {
let db: PostgresDatabase;
beforeAll(async () => {
({ db } = await createPostgresTestDatabase());
await migratePostgres(db);
});
afterAll(async () => db?.close());
beforeEach(async () => {
await db.query('TRUNCATE tool_favorites,tool_favorite_erasure');
await db.query(
'UPDATE tool_favorite_readiness SET checkpoint=NULL,snapshot=NULL,verified=false,activated=false,imported=0',
);
});
function interruptAfter(prefix: string, occurrence: number): PostgresDatabase {
let remaining = occurrence;
return {
...db,
session: (operation) =>
db.session((session) =>
operation({
...session,
transaction: (work) =>
session.transaction((sql) =>
work({
async query<R extends QueryResultRow>(text: string, values?: unknown[]) {
const result = await sql.query<R>(text, values);
if (text.startsWith(prefix) && --remaining === 0) {
throw new Error('Injected interruption');
}
return result;
},
} satisfies SqlExecutor),
),
}),
),
};
}
async function state() {
return (
await db.query(
'SELECT checkpoint,snapshot,verified,activated,imported FROM tool_favorite_readiness',
)
).rows[0];
}
it.each([
['INSERT INTO tool_favorites', 1, 0],
['INSERT INTO tool_favorites', 2, 2],
['UPDATE tool_favorite_readiness SET checkpoint', 2, 2],
])(
'rolls back rows and checkpoint after %s interruption %i, then resumes',
async (prefix, occurrence, committed) => {
await expect(
importFavorites(interruptAfter(prefix, occurrence), source(), options),
).rejects.toThrow('Injected');
expect((await db.query('SELECT id FROM tool_favorites ORDER BY id')).rows).toEqual(
rows.slice(0, committed).map(({ id }) => ({ id })),
);
expect(await state()).toMatchObject({
checkpoint: committed ? rows[committed - 1].id : null,
verified: false,
imported: '0',
});
await importFavorites(db, source(), options);
await importFavorites(db, source(), { ...options, batchSize: 3 });
expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual(
await fingerprintFavorites(source()),
);
expect(await state()).toMatchObject({
checkpoint: rows[4].id,
verified: true,
imported: '5',
});
},
);
it('resumes when the source iterator fails after a committed batch', async () => {
let pass = 0;
const interrupted: FavoriteSnapshotSource = async function* () {
pass++;
for (const row of rows) {
yield row;
if (pass === 2 && row.id === rows[1].id) throw new Error('Source interrupted');
}
};
await expect(importFavorites(db, interrupted, options)).rejects.toThrow('Source interrupted');
expect(await state()).toMatchObject({ checkpoint: rows[1].id, verified: false });
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2);
await importFavorites(db, source(), options);
expect(await state()).toMatchObject({ verified: true, imported: '5' });
});
it('keeps verification false when the source changes during the final verification pass', async () => {
let pass = 0;
const changing: FavoriteSnapshotSource = () =>
source(++pass === 3 ? rows.map((row) => ({ ...row, tenantId: 'changed' })) : rows)();
await expect(importFavorites(db, changing, options)).rejects.toThrow('verification failed');
expect(await state()).toMatchObject({ verified: false });
expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual(
await fingerprintFavorites(source()),
);
});
it.each([false, true])(
'survives a final source scan beyond the idle transaction timeout (changed: %s)',
async (changed) => {
let pass = 0;
const slow: FavoriteSnapshotSource = async function* () {
if (++pass === 3) {
await new Promise((resolve) => setTimeout(resolve, 300));
yield* changed ? rows.map((row) => ({ ...row, tenantId: 'changed' })) : rows;
return;
}
yield* rows;
};
const shortTimeout: PostgresDatabase = {
...db,
session: (operation) =>
db.session(async (session) => {
await session.query("SET idle_in_transaction_session_timeout = '100ms'");
try {
return await operation(session);
} finally {
await session.query('RESET idle_in_transaction_session_timeout');
}
}),
};
if (changed) {
await expect(importFavorites(shortTimeout, slow, options)).rejects.toThrow(
'verification failed',
);
} else {
await expect(importFavorites(shortTimeout, slow, options)).resolves.toEqual(
await fingerprintFavorites(source()),
);
}
expect(pass).toBe(3);
expect(await state()).toMatchObject({ verified: !changed, imported: changed ? '0' : '5' });
expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual(
await fingerprintFavorites(source()),
);
},
);
it('holds the pinned migration lock without a transaction during every source pass', async () => {
let pid = 0;
let passes = 0;
const pinned: PostgresDatabase = {
...db,
session: (operation) =>
db.session(async (session) => {
pid = (await session.query<{ pid: number }>('SELECT pg_backend_pid() AS pid')).rows[0]
.pid;
return operation(session);
}),
};
const checking: FavoriteSnapshotSource = async function* () {
passes++;
expect(
(await db.query('SELECT state FROM pg_stat_activity WHERE pid=$1', [pid])).rows,
).toEqual([{ state: 'idle' }]);
await expect(importFavorites(db, source(), options)).rejects.toThrow(
'Another favorite migration or import is running',
);
yield* rows;
};
await importFavorites(pinned, checking, options);
expect(passes).toBe(3);
expect(await state()).toMatchObject({ verified: true, imported: '5' });
await importFavorites(db, source(), options);
});
it.each(['activated', 'snapshot'])(
'refuses %s authority change between batches',
async (change) => {
let pass = 0;
const changing: FavoriteSnapshotSource = async function* () {
pass++;
for (const row of rows) {
yield row;
if (pass !== 2 || row.id !== rows[1].id) continue;
if (change === 'activated')
await db.query('UPDATE tool_favorite_readiness SET activated=true');
else await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'");
}
};
await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed');
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2);
expect(await state()).toMatchObject({ checkpoint: rows[1].id, verified: false });
},
);
it('preserves canonical identity, tenant and millisecond timestamps exactly', async () => {
await importFavorites(db, source(), options);
const actual: FavoriteSnapshotRow[] = [];
for await (const row of readPostgresFavorites(db, 2)()) actual.push(row);
expect(actual).toEqual(rows);
});
it('refuses a changed fingerprint on resume without modifying committed rows', async () => {
await expect(
importFavorites(interruptAfter('INSERT INTO tool_favorites', 2), source(), options),
).rejects.toThrow('Injected');
const before = await state();
const changed = rows.map((row, i) => (i ? row : { ...row, tenantId: 'changed' }));
await expect(importFavorites(db, source(changed), options)).rejects.toThrow(
'different snapshot',
);
expect(await state()).toEqual(before);
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2);
});
it('refuses duplicate logical favorites instead of silently checkpointing them', async () => {
const duplicate = { ...rows[1], itemId: rows[0].itemId };
await expect(importFavorites(db, source([rows[0], duplicate]), options)).rejects.toThrow(
'conflicts',
);
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0);
expect(await state()).toMatchObject({ checkpoint: null, verified: false });
});
it('refuses a destination mapping collision and preserves unrelated rows', async () => {
await db.query('INSERT INTO tool_favorites VALUES ($1,$2,$3,$4,$5,$6,$7)', [
rows[0].id,
userId,
'other-tenant',
'tool',
'unrelated',
rows[0].createdAt,
rows[0].updatedAt,
]);
await expect(importFavorites(db, source(), options)).rejects.toThrow('conflicts');
expect((await db.query('SELECT tenant_id,item_id FROM tool_favorites')).rows).toEqual([
{ tenant_id: 'other-tenant', item_id: 'unrelated' },
]);
expect(await state()).toMatchObject({ checkpoint: null, verified: false });
});
it('detects a corrupted checkpoint prefix without overwriting it', async () => {
await expect(
importFavorites(interruptAfter('INSERT INTO tool_favorites', 2), source(), options),
).rejects.toThrow('Injected');
await db.query('UPDATE tool_favorites SET tenant_id=$1 WHERE id=$2', ['wrong', rows[0].id]);
await expect(importFavorites(db, source(), options)).rejects.toThrow('verification failed');
expect(await state()).toMatchObject({ verified: false });
expect(
(await db.query('SELECT tenant_id FROM tool_favorites WHERE id=$1', [rows[0].id])).rows[0]
.tenant_id,
).toBe('wrong');
});
it.each(['activated', 'snapshot', 'missing'])(
'refuses %s authority change after the last batch',
async (change) => {
let pass = 0;
const changing: FavoriteSnapshotSource = async function* () {
pass++;
yield* rows.slice(0, 2);
if (pass !== 2) return;
if (change === 'activated')
await db.query('UPDATE tool_favorite_readiness SET activated=true');
if (change === 'snapshot')
await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'");
if (change === 'missing') await db.query('DELETE FROM tool_favorite_readiness');
};
try {
await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed');
if (change !== 'missing') expect(await state()).toMatchObject({ verified: false });
} finally {
if (change === 'missing')
await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)');
}
},
);
it.each(['activated', 'snapshot', 'missing'])(
'refuses %s authority change during the final source pass',
async (change) => {
let pass = 0;
const changing: FavoriteSnapshotSource = async function* () {
if (++pass === 3) {
if (change === 'activated')
await db.query('UPDATE tool_favorite_readiness SET activated=true');
if (change === 'snapshot')
await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'");
if (change === 'missing') await db.query('DELETE FROM tool_favorite_readiness');
}
yield* rows;
};
try {
await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed');
if (change !== 'missing') expect(await state()).toMatchObject({ verified: false });
} finally {
if (change === 'missing')
await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)');
}
},
);
it('refuses a changed middle source pass even when the final source would match', async () => {
let pass = 0;
const changing: FavoriteSnapshotSource = () => source(++pass === 2 ? rows.slice(0, 2) : rows)();
await expect(importFavorites(db, changing, options)).rejects.toThrow('source changed');
expect(await state()).toMatchObject({ verified: false });
});
it('refuses unsorted rows in the write pass', async () => {
let pass = 0;
const changing: FavoriteSnapshotSource = () =>
source(++pass === 2 ? [rows[1], rows[0]] : rows)();
await expect(importFavorites(db, changing, options)).rejects.toThrow('uniquely ordered');
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0);
});
it.each([
{ userId: 'not-canonical' },
{ id: 'bad-id' },
{ itemType: 'unknown' },
{ itemId: 42 },
{ itemId: '' },
{ tenantId: 12 },
{ createdAt: new Date('invalid') },
{ updatedAt: 'not-a-date' },
])('refuses malformed input before claiming a snapshot: %p', async (invalid) => {
const malformed = { ...rows[0], ...invalid } as FavoriteSnapshotRow;
await expect(importFavorites(db, source([malformed]), options)).rejects.toThrow(
'Invalid favorite snapshot record',
);
expect(await state()).toMatchObject({ snapshot: null, checkpoint: null, verified: false });
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0);
});
it('refuses duplicate IDs before claiming a snapshot', async () => {
await expect(importFavorites(db, source([rows[0], rows[0]]), options)).rejects.toThrow(
'uniquely ordered',
);
expect(await state()).toMatchObject({ snapshot: null });
});
it('rejects a zero-sized PostgreSQL reader rather than fingerprinting a false empty source', () => {
expect(() => readPostgresFavorites(db, 0)).toThrow('batch size');
});
});

View file

@ -0,0 +1,243 @@
import { createHash } from 'node:crypto';
import type { PostgresDatabase, PostgresSession, SqlExecutor } from './pool';
import type { FavoriteItemType } from '~/types/favorite';
import { lockFavoriteMigration } from './migrations';
export interface FavoriteSnapshotRow {
id: string;
userId: string;
tenantId: string | null;
itemType: FavoriteItemType;
itemId: string;
createdAt: Date;
updatedAt: Date;
}
export type FavoriteSnapshotSource = () => AsyncIterable<FavoriteSnapshotRow>;
function encoded(row: FavoriteSnapshotRow): string {
if (
typeof row.id !== 'string' ||
!/^[0-9a-f]{24}$/.test(row.id) ||
typeof row.userId !== 'string' ||
!/^[0-9a-f]{24}$/.test(row.userId) ||
!['builtin', 'tool', 'mcp', 'skill'].includes(row.itemType) ||
typeof row.itemId !== 'string' ||
!row.itemId ||
row.itemId.length > 256 ||
(row.tenantId !== null && typeof row.tenantId !== 'string') ||
!(row.createdAt instanceof Date) ||
!Number.isFinite(row.createdAt.getTime()) ||
!(row.updatedAt instanceof Date) ||
!Number.isFinite(row.updatedAt.getTime())
) {
throw new Error('Invalid favorite snapshot record');
}
return JSON.stringify([
row.id,
row.userId,
row.tenantId,
row.itemType,
row.itemId,
row.createdAt.toISOString(),
row.updatedAt.toISOString(),
]);
}
export async function fingerprintFavorites(
source: FavoriteSnapshotSource,
): Promise<{ count: number; digest: string }> {
const hash = createHash('sha256');
let count = 0;
let previous = '';
for await (const row of source()) {
if (row.id <= previous) {
throw new Error('Favorite snapshot must be uniquely ordered by ID');
}
hash.update(encoded(row)).update('\n');
previous = row.id;
count++;
}
return { count, digest: hash.digest('hex') };
}
export function readPostgresFavorites(db: SqlExecutor, batchSize: number): FavoriteSnapshotSource {
if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1000) {
throw new Error('Favorite snapshot batch size must be between 1 and 1000');
}
return async function* () {
let after = '';
for (;;) {
const result = await db.query<FavoriteSnapshotRow>(
`SELECT id, user_id AS "userId", tenant_id AS "tenantId",
item_type AS "itemType", item_id AS "itemId", created_at AS "createdAt", updated_at AS "updatedAt"
FROM tool_favorites WHERE id > $1 ORDER BY id LIMIT $2`,
[after, batchSize],
);
if (!result.rows.length) {
return;
}
for (const row of result.rows) {
yield row;
after = row.id;
}
}
};
}
/** Source and destination writers must be stopped. Activation permanently blocks replay. */
interface ImportOptions {
offline: boolean;
batchSize: number;
dryRun?: boolean;
verifiedEmpty?: boolean;
}
export async function importFavorites(
db: PostgresDatabase,
source: FavoriteSnapshotSource,
options: ImportOptions,
): Promise<{ count: number; digest: string }> {
return db.session(async (session) => {
const lock = await session.query<{ acquired: boolean }>(
'SELECT pg_try_advisory_lock(17490261, hashtext(current_schema())) AS acquired',
);
if (!lock.rows[0].acquired) {
throw new Error('Another favorite migration or import is running');
}
try {
return await importSnapshot(session, source, options);
} finally {
await session.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))');
}
});
}
async function importSnapshot(
db: PostgresSession,
source: FavoriteSnapshotSource,
options: ImportOptions,
): Promise<{ count: number; digest: string }> {
if (
!options.offline ||
!Number.isInteger(options.batchSize) ||
options.batchSize < 1 ||
options.batchSize > 1000
) {
throw new Error('Offline assertion and batch size between 1 and 1000 are required');
}
const snapshot = await fingerprintFavorites(source);
if (snapshot.count === 0 && !options.verifiedEmpty) {
throw new Error('Empty source requires explicit verified-empty approval');
}
if (options.verifiedEmpty && snapshot.count !== 0) {
throw new Error('Verified-empty refused: Mongo favorites are not empty');
}
if (options.dryRun) {
return snapshot;
}
await db.transaction(async (sql) => {
await lockFavoriteMigration(sql);
const result = await sql.query<{ activated: boolean; snapshot: string | null }>(
'SELECT activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE',
);
if (
!result.rows[0] ||
result.rows[0].activated ||
(result.rows[0].snapshot && result.rows[0].snapshot !== snapshot.digest)
) {
throw new Error('Favorite import cannot replace an activated or different snapshot');
}
await sql.query(
'UPDATE tool_favorite_readiness SET snapshot=$1, verified=false WHERE singleton',
[snapshot.digest],
);
});
let batch: FavoriteSnapshotRow[] = [];
const write = async () => {
await db.transaction(async (sql) => {
await lockFavoriteMigration(sql);
const state = await sql.query<{
checkpoint: string | null;
activated: boolean;
snapshot: string;
}>(
'SELECT checkpoint, activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE',
);
if (!state.rows[0] || state.rows[0].activated || state.rows[0].snapshot !== snapshot.digest) {
throw new Error('Favorite import authority changed');
}
const pending = batch.filter((row) => row.id > (state.rows[0].checkpoint ?? ''));
if (pending.length) {
const inserted = await sql.query(
`INSERT INTO tool_favorites (id,user_id,tenant_id,item_type,item_id,created_at,updated_at)
SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::text[],$5::text[],$6::timestamptz[],$7::timestamptz[])
ON CONFLICT DO NOTHING`,
[
pending.map((row) => row.id),
pending.map((row) => row.userId),
pending.map((row) => row.tenantId),
pending.map((row) => row.itemType),
pending.map((row) => row.itemId),
pending.map((row) => row.createdAt),
pending.map((row) => row.updatedAt),
],
);
if (inserted.rowCount !== pending.length) {
throw new Error('Favorite snapshot conflicts with destination mapping');
}
}
await sql.query(
"UPDATE tool_favorite_readiness SET checkpoint=GREATEST(COALESCE(checkpoint,''),$1) WHERE singleton",
[batch[batch.length - 1].id],
);
});
batch = [];
};
const written = createHash('sha256');
let count = 0;
let previous = '';
for await (const row of source()) {
if (row.id <= previous) {
throw new Error('Favorite snapshot must be uniquely ordered by ID');
}
written.update(encoded(row)).update('\n');
previous = row.id;
count++;
batch.push(row);
if (batch.length === options.batchSize) {
await write();
}
}
if (batch.length) {
await write();
}
if (count !== snapshot.count || written.digest('hex') !== snapshot.digest) {
throw new Error('Favorite source changed during import');
}
const current = await fingerprintFavorites(source);
await db.transaction(async (sql) => {
await lockFavoriteMigration(sql);
const state = await sql.query<{ activated: boolean; snapshot: string | null }>(
'SELECT activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE',
);
if (!state.rows[0] || state.rows[0].activated || state.rows[0].snapshot !== snapshot.digest) {
throw new Error('Favorite import authority changed');
}
const destination = await fingerprintFavorites(readPostgresFavorites(sql, options.batchSize));
if (
current.digest !== snapshot.digest ||
destination.digest !== snapshot.digest ||
destination.count !== snapshot.count
) {
throw new Error('Favorite snapshot verification failed');
}
const verified = await sql.query(
'UPDATE tool_favorite_readiness SET verified=true, imported=$1 WHERE singleton AND NOT activated AND snapshot=$2',
[snapshot.count, snapshot.digest],
);
if (verified.rowCount !== 1) {
throw new Error('Favorite import authority changed');
}
});
return snapshot;
}

View file

@ -0,0 +1,98 @@
import { randomBytes } from 'node:crypto';
import type { PostgresDatabase } from './pool';
import { migratePostgres, assertPostgresReady } from './migrations';
import { createPostgresTestDatabase } from './postgres.helper';
import { importFavorites } from './import';
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
postgres('PostgreSQL catalog readiness', () => {
let db: PostgresDatabase;
let schema: string;
beforeAll(async () => {
({ db, schema } = await createPostgresTestDatabase());
});
afterAll(() => db?.close());
beforeEach(async () => {
await db.query(
'DROP TABLE IF EXISTS tool_favorites,tool_favorite_erasure,tool_favorite_readiness,librechat_migrations',
);
await migratePostgres(db);
await importFavorites(db, async function* () {}, {
offline: true,
batchSize: 10,
verifiedEmpty: true,
});
});
it.each([
'DROP INDEX tool_favorites_user_created',
'ALTER TABLE tool_favorites DROP CONSTRAINT tool_favorites_user_id_item_type_item_id_key',
'ALTER TABLE tool_favorites DROP CONSTRAINT tool_favorites_item_id_check',
'ALTER TABLE tool_favorites ALTER COLUMN item_id DROP NOT NULL',
'ALTER TABLE tool_favorites ENABLE ROW LEVEL SECURITY',
'DROP TABLE tool_favorite_erasure',
])('refuses schema drift before activation: %s', async (statement) => {
await db.query(statement);
await expect(assertPostgresReady(db)).rejects.toThrow('schema');
expect((await db.query('SELECT activated FROM tool_favorite_readiness')).rows).toEqual([
{ activated: false },
]);
});
it('supports concurrent idempotent migration runners', async () => {
await Promise.all([migratePostgres(db), migratePostgres(db), migratePostgres(db)]);
expect((await db.query('SELECT version FROM librechat_migrations')).rows).toEqual([
{ version: 1 },
]);
await assertPostgresReady(db);
});
it('does not turn nested work into a premature commit', async () => {
await expect(
db.session(async (session) =>
session.transaction(async (sql) => {
await sql.query("INSERT INTO tool_favorite_erasure VALUES ('nested','*')");
await session.transaction(async () => {});
}),
),
).rejects.toThrow('Nested');
expect((await db.query('SELECT * FROM tool_favorite_erasure')).rowCount).toBe(0);
});
it('validates runtime DML privileges rather than requiring migration-owner permissions', async () => {
const role = `favorites_role_${randomBytes(8).toString('hex')}`;
await db.query(`DROP ROLE IF EXISTS ${role}`);
await db.query(`CREATE ROLE ${role}`);
try {
await db.query(`GRANT USAGE ON SCHEMA ${schema} TO ${role}`);
await db.query(`GRANT SELECT ON librechat_migrations TO ${role}`);
await db.query(
`GRANT SELECT, INSERT, DELETE ON tool_favorites,tool_favorite_erasure,tool_favorite_readiness TO ${role}`,
);
await db.query(`GRANT UPDATE ON tool_favorite_readiness TO ${role}`);
const runAsRole = (operation: (database: PostgresDatabase) => Promise<void>) =>
db.session(async (session) => {
await session.query(`SET ROLE ${role}`);
try {
const selected: PostgresDatabase = {
...db,
query: session.query,
transaction: session.transaction,
};
await operation(selected);
} finally {
await session.query('RESET ROLE');
}
});
await runAsRole((database) => assertPostgresReady(database));
await db.query(`REVOKE DELETE ON tool_favorite_erasure FROM ${role}`);
await runAsRole(async (database) => {
await expect(assertPostgresReady(database)).rejects.toThrow('permissions');
});
await db.query(`GRANT DELETE ON tool_favorite_erasure TO ${role}`);
await db.query(`REVOKE INSERT ON tool_favorites FROM ${role}`);
await runAsRole(async (database) => {
await expect(assertPostgresReady(database)).rejects.toThrow('permissions');
});
} finally {
await db.query(`DROP OWNED BY ${role}`);
await db.query(`DROP ROLE ${role}`);
}
});
});

View file

@ -0,0 +1,158 @@
import { createHash } from 'node:crypto';
import type { PostgresDatabase, SqlExecutor } from './pool';
const migration = {
version: 1,
sql: `CREATE TABLE tool_favorites (
id text PRIMARY KEY,
user_id text NOT NULL CHECK (user_id ~ '^[0-9a-f]{24}$'),
tenant_id text,
item_type text NOT NULL CHECK (item_type IN ('builtin', 'tool', 'mcp', 'skill')),
item_id text NOT NULL CHECK (char_length(item_id) BETWEEN 1 AND 256),
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
UNIQUE (user_id, item_type, item_id)
);
CREATE INDEX tool_favorites_user_created ON tool_favorites (user_id, created_at, id);
CREATE INDEX tool_favorites_tenant_user ON tool_favorites (tenant_id, user_id);
CREATE TABLE tool_favorite_erasure (
user_id text NOT NULL,
scope text NOT NULL,
attempt_id text NOT NULL DEFAULT '',
PRIMARY KEY (user_id, scope, attempt_id)
);
CREATE TABLE tool_favorite_readiness (
singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
checkpoint text,
verified boolean NOT NULL DEFAULT false,
activated boolean NOT NULL DEFAULT false,
snapshot text,
imported bigint NOT NULL DEFAULT 0
);
INSERT INTO tool_favorite_readiness (singleton) VALUES (true);`,
};
const checksum = createHash('sha256').update(migration.sql).digest('hex');
export async function lockFavoriteMigration(sql: SqlExecutor): Promise<void> {
await sql.query('SELECT pg_advisory_xact_lock(17490261, hashtext(current_schema()))');
}
async function assertVersion(sql: SqlExecutor): Promise<void> {
const result = await sql.query<{ version: string }>(
"SELECT current_setting('server_version_num') AS version",
);
if (Number(result.rows[0].version) < 160000 || Number(result.rows[0].version) >= 170000) {
throw new Error('PostgreSQL 16 is required');
}
}
const tables = ['tool_favorites', 'tool_favorite_erasure', 'tool_favorite_readiness'];
/** Compare deployed columns, constraints and indexes, not only the applied-version marker. */
async function schemaChecksum(sql: SqlExecutor): Promise<string> {
const catalog = await sql.query<{ definition: unknown }>(
`SELECT jsonb_build_object(
'name', expected.name, 'kind', relation.relkind,
'rowSecurity', relation.relrowsecurity, 'forceRowSecurity', relation.relforcerowsecurity,
'columns', (SELECT jsonb_agg(jsonb_build_array(attribute.attname,
format_type(attribute.atttypid, attribute.atttypmod), attribute.attnotnull,
pg_get_expr(defaults.adbin, defaults.adrelid), attribute.attidentity, attribute.attgenerated)
ORDER BY attribute.attnum)
FROM pg_attribute attribute LEFT JOIN pg_attrdef defaults
ON defaults.adrelid=attribute.attrelid AND defaults.adnum=attribute.attnum
WHERE attribute.attrelid=relation.oid AND attribute.attnum>0 AND NOT attribute.attisdropped),
'constraints', (SELECT jsonb_agg(jsonb_build_array(constraint_name.conname,
pg_get_constraintdef(constraint_name.oid), constraint_name.convalidated)
ORDER BY constraint_name.conname)
FROM pg_constraint constraint_name WHERE constraint_name.conrelid=relation.oid),
'indexes', (SELECT jsonb_agg(jsonb_build_array(pg_get_indexdef(index_definition.indexrelid),
index_definition.indisvalid, index_definition.indisready) ORDER BY pg_get_indexdef(index_definition.indexrelid))
FROM pg_index index_definition WHERE index_definition.indrelid=relation.oid),
'triggers', (SELECT jsonb_agg(pg_get_triggerdef(trigger_definition.oid)
ORDER BY trigger_definition.tgname) FROM pg_trigger trigger_definition
WHERE trigger_definition.tgrelid=relation.oid AND NOT trigger_definition.tgisinternal),
'policies', (SELECT jsonb_agg(jsonb_build_array(policy.polname, policy.polcmd,
policy.polpermissive, policy.polroles, pg_get_expr(policy.polqual, policy.polrelid),
pg_get_expr(policy.polwithcheck, policy.polrelid)) ORDER BY policy.polname)
FROM pg_policy policy WHERE policy.polrelid=relation.oid)
) AS definition FROM unnest($1::text[]) AS expected(name)
LEFT JOIN pg_class relation ON relation.oid=to_regclass(expected.name)
ORDER BY expected.name`,
[tables],
);
return createHash('sha256').update(JSON.stringify(catalog.rows)).digest('hex');
}
async function assertSchema(sql: SqlExecutor): Promise<void> {
const applied = await sql.query<{ version: number; checksum: string; schema_checksum: string }>(
'SELECT version, checksum, schema_checksum FROM librechat_migrations ORDER BY version',
);
if (
applied.rows.length !== 1 ||
applied.rows[0].version !== migration.version ||
applied.rows[0].checksum !== checksum ||
applied.rows[0].schema_checksum !== (await schemaChecksum(sql))
) {
throw new Error('PostgreSQL schema checksum or version mismatch');
}
const privileges = await sql.query<{ allowed: boolean }>(
`SELECT has_table_privilege(current_user, 'tool_favorites', 'SELECT')
AND has_table_privilege(current_user, 'tool_favorites', 'INSERT')
AND has_table_privilege(current_user, 'tool_favorites', 'DELETE')
AND has_table_privilege(current_user, 'tool_favorite_erasure', 'SELECT')
AND has_table_privilege(current_user, 'tool_favorite_erasure', 'INSERT')
AND has_table_privilege(current_user, 'tool_favorite_erasure', 'DELETE')
AND has_table_privilege(current_user, 'tool_favorite_readiness', 'SELECT')
AND has_table_privilege(current_user, 'tool_favorite_readiness', 'UPDATE') AS allowed`,
);
if (!privileges.rows[0]?.allowed) throw new Error('PostgreSQL runtime permissions not ready');
}
export async function migratePostgres(db: PostgresDatabase): Promise<void> {
await db.transaction(async (sql) => {
await assertVersion(sql);
await lockFavoriteMigration(sql);
await sql.query(`CREATE TABLE IF NOT EXISTS librechat_migrations (
version integer PRIMARY KEY, checksum text NOT NULL, schema_checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now()
)`);
const result = await sql.query<{ version: number; checksum: string }>(
'SELECT version, checksum FROM librechat_migrations ORDER BY version',
);
if (result.rows.length) {
if (
result.rows.length !== 1 ||
result.rows[0].version !== migration.version ||
result.rows[0].checksum !== checksum
) {
throw new Error('PostgreSQL migration checksum or version mismatch');
}
await assertSchema(sql);
return;
}
await sql.query(migration.sql);
await sql.query(
'INSERT INTO librechat_migrations (version, checksum, schema_checksum) VALUES ($1, $2, $3)',
[migration.version, checksum, await schemaChecksum(sql)],
);
});
}
/** Readiness check without selecting an authority or changing its activation marker. */
export async function assertPostgresDataReady(sql: SqlExecutor): Promise<void> {
await assertVersion(sql);
await assertSchema(sql);
const ready = await sql.query<{ verified: boolean; snapshot: string | null }>(
'SELECT verified, snapshot FROM tool_favorite_readiness WHERE singleton',
);
if (ready.rows[0]?.verified !== true || !/^[a-f0-9]{64}$/.test(ready.rows[0].snapshot ?? '')) {
throw new Error('PostgreSQL favorites data not verified');
}
}
export async function assertPostgresReady(db: PostgresDatabase): Promise<void> {
await db.transaction(async (sql) => {
await lockFavoriteMigration(sql);
await assertPostgresDataReady(sql);
await sql.query('UPDATE tool_favorite_readiness SET activated = true WHERE singleton');
});
}

View file

@ -0,0 +1,102 @@
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { createToolFavoriteModel } from '~/models/favorite';
import toolFavoriteSchema from '~/schema/favorite';
import { createMongoFavoriteStore } from './mongo';
async function rows(source: AsyncIterable<unknown>): Promise<unknown[]> {
const result: unknown[] = [];
for await (const row of source) result.push(row);
return result;
}
describe('CLI Mongo favorite snapshots', () => {
let server: MongoMemoryServer;
const database = new mongoose.Mongoose();
beforeAll(async () => {
server = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
}, 90000);
beforeEach(async () => {
await database.connect(server.getUri(), { autoCreate: false, autoIndex: false });
await database.connection.dropDatabase();
});
afterEach(async () => {
await database.disconnect();
});
afterAll(async () => {
await server.stop();
});
it('reads missing collections without DDL, model registration, or inherited schema options', async () => {
const previous = {
autoCreate: toolFavoriteSchema.get('autoCreate'),
autoIndex: toolFavoriteSchema.get('autoIndex'),
autoSearchIndex: toolFavoriteSchema.get('autoSearchIndex'),
};
toolFavoriteSchema.set('autoCreate', true);
toolFavoriteSchema.set('autoIndex', true);
toolFavoriteSchema.set('autoSearchIndex', true);
try {
for (let attempt = 0; attempt < 2; attempt++) {
const store = createMongoFavoriteStore(database);
expect(await rows(store.source())).toEqual([]);
expect(database.modelNames()).toEqual([]);
expect(await database.connection.db!.listCollections().toArray()).toEqual([]);
await database.disconnect();
await database.connect(server.getUri(), { autoCreate: false, autoIndex: false });
}
} finally {
toolFavoriteSchema.set('autoCreate', previous.autoCreate);
toolFavoriteSchema.set('autoIndex', previous.autoIndex);
toolFavoriteSchema.set('autoSearchIndex', previous.autoSearchIndex);
}
});
it('uses the data-schema collection name and preserves raw documents and indexes', async () => {
const metadata = new mongoose.Mongoose();
metadata.set('autoCreate', false);
metadata.set('autoIndex', false);
const name = createToolFavoriteModel(metadata).collection.name;
const collection = database.connection.collection(name);
const row = {
_id: new mongoose.Types.ObjectId(),
user: new mongoose.Types.ObjectId(),
tenantId: 'tenant-one',
itemType: 'tool',
itemId: 'favorite',
createdAt: new Date('2025-01-01'),
updatedAt: new Date('2025-01-02'),
};
await collection.insertOne(row);
const before = await database.connection.db!.listCollections().toArray();
const indexes = await collection.indexes();
const store = createMongoFavoriteStore(database);
expect(await rows(store.source())).toEqual([
{
id: row._id.toHexString(),
userId: row.user.toHexString(),
tenantId: row.tenantId,
itemType: row.itemType,
itemId: row.itemId,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
},
]);
expect(await collection.find().toArray()).toEqual([row]);
expect(await collection.indexes()).toEqual(indexes);
expect(await database.connection.db!.listCollections().toArray()).toEqual(before);
expect(database.modelNames()).toEqual([]);
});
it('can replace a missing collection with an empty snapshot using only explicit staging DDL', async () => {
const store = createMongoFavoriteStore(database);
const staging = await store.stage();
expect(await rows(staging.source())).toEqual([]);
const staged = await database.connection.db!.listCollections().toArray();
expect(staged).toHaveLength(1);
expect(staged[0].name).toContain('_reconcile_');
await staging.replace();
const replaced = await database.connection.db!.listCollections().toArray();
expect(replaced).toHaveLength(1);
expect(replaced[0].name).toBe(staged[0].name.split('_reconcile_')[0]);
expect(await rows(store.source())).toEqual([]);
expect(database.modelNames()).toEqual([]);
});
});

View file

@ -0,0 +1,95 @@
import { randomBytes } from 'node:crypto';
import type mongoose from 'mongoose';
import type { FavoriteSnapshotRow, FavoriteSnapshotSource } from './import';
import { buildIndexWithRetry } from '~/utils/retry';
import toolFavoriteSchema from '~/schema/favorite';
export interface MongoFavoriteStaging {
source: FavoriteSnapshotSource;
insert(rows: FavoriteSnapshotRow[]): Promise<void>;
replace(): Promise<void>;
discard(): Promise<void>;
}
export interface MongoFavoriteStore {
source: FavoriteSnapshotSource;
stage(): Promise<MongoFavoriteStaging>;
}
/** Never register application models on the connected CLI instance. */
export function createMongoFavoriteStore(database: typeof mongoose): MongoFavoriteStore {
const metadata = new database.Mongoose();
const schema = toolFavoriteSchema.clone();
schema.set('autoCreate', false);
schema.set('autoIndex', false);
schema.set('autoSearchIndex', false);
// eslint-disable-next-line no-restricted-syntax -- Collection name metadata only, no raw driver operations.
const name = metadata.model('ToolFavorite', schema).collection.name;
const collection = database.connection.collection(name);
const snapshot = (target: typeof collection): FavoriteSnapshotSource =>
async function* () {
const cursor = target.find({}).sort({ _id: 1 }).batchSize(250);
try {
for await (const row of cursor) {
yield {
id: String(row._id),
userId: String(row.user),
tenantId: row.tenantId ?? null,
itemType: row.itemType,
itemId: row.itemId,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
} finally {
await cursor.close();
}
};
return {
source: snapshot(collection),
async stage() {
const stagingName = `${name}_reconcile_${randomBytes(6).toString('hex')}`;
const staging = database.connection.collection(stagingName);
const discard = async () => {
await staging.drop().catch(() => undefined);
};
try {
await database.connection.createCollection(stagingName);
const exists = await database.connection.db!.listCollections({ name }).hasNext();
for (const index of exists ? await collection.indexes() : []) {
if (index.name === '_id_') continue;
const options = Object.fromEntries(
Object.entries(index).filter(([key]) => !['key', 'v', 'ns'].includes(key)),
);
await buildIndexWithRetry(
() => staging.createIndex(index.key, options),
'favorites_reconciliation_index',
);
}
} catch (error) {
await discard();
throw error;
}
return {
source: snapshot(staging),
async insert(rows) {
await staging.insertMany(
rows.map((row) => ({
_id: new database.Types.ObjectId(row.id),
user: new database.Types.ObjectId(row.userId),
tenantId: row.tenantId,
itemType: row.itemType,
itemId: row.itemId,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
})),
);
},
async replace() {
await staging.rename(name, { dropTarget: true });
},
discard,
};
},
};
}

View file

@ -0,0 +1,237 @@
import pg from 'pg';
import path from 'node:path';
import { promisify } from 'node:util';
import { EventEmitter } from 'node:events';
import { execFile } from 'node:child_process';
import type { PostgresOptions } from './pool';
import { createPostgresDatabase } from './pool';
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
const options = (): PostgresOptions => ({
connectionString: process.env.PG_TEST_URL ?? '',
max: 1,
maxWaiting: 10,
acquireTimeoutMs: 2000,
queryTimeoutMs: 10000,
idleTimeoutMs: 1000,
});
const safeMessage = 'PostgreSQL connection unavailable';
const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
postgres('PostgreSQL borrowed connections', () => {
it.each(['session', 'transaction'] as const)(
'survives an idle %s disconnect in a disposable child process',
async (kind) => {
const source = `
import pg from 'pg';
import { createPostgresDatabase } from ${JSON.stringify(path.resolve(__dirname, 'pool.ts'))};
const db = createPostgresDatabase({ ...${JSON.stringify({ ...options(), connectionString: undefined })}, connectionString: process.env.PG_TEST_URL });
const killer = new pg.Client({ connectionString: process.env.PG_TEST_URL });
await killer.connect();
let openGate;
const gate = new Promise(resolve => { openGate = resolve; });
let session;
let entered;
const ready = new Promise(resolve => { entered = resolve; });
const operation = db[${JSON.stringify(kind)}](async sql => {
session = sql;
const { rows } = await sql.query('SELECT pg_backend_pid() AS pid');
entered(rows[0].pid);
await gate;
return 'must not succeed';
});
const outcome = operation.then(() => 'success', error => error.message);
await killer.query('SELECT pg_terminate_backend($1)', [await ready]);
const observed = await Promise.race([outcome, new Promise(resolve => setTimeout(() => resolve('not reported'), 1000))]);
if (observed !== ${JSON.stringify(safeMessage)}) throw new Error(observed);
for (const attempt of [
() => session.query('SELECT 1'),
() => session.transaction(async () => 'must not succeed'),
]) {
const message = await attempt().then(() => 'success', error => error.message);
if (message !== ${JSON.stringify(safeMessage)}) throw new Error(message);
}
let closed = false;
const closing = db.close().then(() => { closed = true; });
await new Promise(resolve => setTimeout(resolve, 30));
if (closed) throw new Error('close released an unfinished callback');
openGate();
await outcome;
await closing;
await killer.end();
if (observed !== ${JSON.stringify(safeMessage)}) throw new Error(observed);
console.log('borrowed-disconnect-safe');
`;
const { stdout } = await promisify(execFile)(
process.execPath,
['--experimental-strip-types', '--input-type=module', '--eval', source],
{ timeout: 10000 },
);
expect(stdout).toContain('borrowed-disconnect-safe');
},
);
it.each(['query', 'transaction'] as const)(
'sanitizes an active %s disconnect and replaces the failed backend',
async (kind) => {
const db = createPostgresDatabase(options());
const killer = new pg.Client({ connectionString: process.env.PG_TEST_URL });
await killer.connect();
try {
const pid = (await db.query('SELECT pg_backend_pid() AS pid')).rows[0].pid;
const operation =
kind === 'query'
? db.query('SELECT pg_sleep(5)')
: db.transaction(async (sql) => {
await sql.query('SELECT pg_sleep(5)');
});
const outcome = operation.catch((error: Error) => error);
for (let i = 0; i < 100; i++) {
const { rows } = await killer.query('SELECT state FROM pg_stat_activity WHERE pid=$1', [
pid,
]);
if (rows[0]?.state === 'active') break;
await delay(10);
}
await killer.query('SELECT pg_terminate_backend($1)', [pid]);
const error = await outcome;
expect(error).toEqual(new Error(safeMessage));
expect(error).not.toHaveProperty('cause');
const next = await db.query('SELECT pg_backend_pid() AS pid');
expect(next.rows[0].pid).not.toBe(pid);
} finally {
await db.close();
await killer.end();
}
},
);
});
describe('PostgreSQL lease lifecycle', () => {
function setup() {
const client = new EventEmitter();
const idle = () => undefined;
client.on('error', idle);
const query = jest.fn(async (_text: string) => ({ rows: [], rowCount: 0 }));
const release = jest.fn((_discard: boolean) => {
client.on('error', idle);
});
Object.assign(client, { query, release });
const pool = new EventEmitter();
let onHandoff: () => void = () => undefined;
const connect = jest.fn((callback: (error: Error | undefined, client: unknown) => void) => {
client.removeListener('error', idle);
callback(undefined, client);
onHandoff();
});
const end = jest.fn(async () => undefined);
Object.assign(pool, { connect, end, waitingCount: 0 });
jest.spyOn(pg, 'Pool').mockImplementation(() => pool as pg.Pool);
const db = createPostgresDatabase(options());
return {
db,
connect,
client,
query,
release,
onHandoff: (callback: () => void) => {
onHandoff = callback;
},
};
}
it.each(['callback', 'throw'] as const)('sanitizes a driver connect %s failure', async (kind) => {
const { db, connect, release } = setup();
connect.mockImplementation((callback) => {
const original = new Error('postgresql://private:secret@host/private');
if (kind === 'throw') throw original;
callback(original, undefined);
});
const error = await db.query('SELECT 1').catch((error: Error) => error);
expect(error).toEqual(new Error(safeMessage));
expect(error).not.toHaveProperty('cause');
expect(release).not.toHaveBeenCalled();
await db.close();
});
it('protects the synchronous connect handoff and releases exactly once', async () => {
const fixture = setup();
fixture.onHandoff(() => fixture.client.emit('error', new Error('private DSN')));
await expect(fixture.db.query('private query')).rejects.toThrow(safeMessage);
expect(fixture.query).not.toHaveBeenCalled();
expect(fixture.release).toHaveBeenCalledTimes(1);
expect(fixture.release).toHaveBeenCalledWith(true);
expect(fixture.client.listenerCount('error')).toBe(1);
await fixture.db.close();
});
it('leaves only the pool idle listener across successful leases', async () => {
const { db, client, release } = setup();
for (let i = 0; i < 20; i++) {
await db.session(async (sql) => {
expect(client.listenerCount('error')).toBe(1);
await sql.query('SELECT 1');
});
expect(client.listenerCount('error')).toBe(1);
}
expect(release).toHaveBeenCalledTimes(20);
expect(release).toHaveBeenLastCalledWith(false);
await db.close();
});
it('handles an error racing release without returning success or leaking listeners', async () => {
const { db, client, release } = setup();
release.mockImplementation(() => {
client.on('error', () => undefined);
client.emit('error', new Error('private release fault'));
});
await expect(db.query('SELECT 1')).rejects.toThrow(safeMessage);
expect(release).toHaveBeenCalledTimes(1);
expect(client.listenerCount('error')).toBe(1);
await db.close();
});
it('retains local SQL and callback exception identity', async () => {
const { db, query, release } = setup();
const sqlError = Object.assign(new Error('local constraint'), { code: '23505' });
query.mockRejectedValueOnce(sqlError);
await expect(db.query('INSERT')).rejects.toBe(sqlError);
const original = new Error('local callback');
await expect(
db.transaction(async () => {
throw original;
}),
).rejects.toBe(original);
expect(query.mock.calls.map(([text]) => text)).toEqual(['INSERT', 'BEGIN', 'ROLLBACK']);
expect(release).toHaveBeenLastCalledWith(false);
await db.close();
});
it('discards on rollback failure while preserving the original callback exception', async () => {
const { db, query, release } = setup();
const original = new Error('local callback');
query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
query.mockRejectedValueOnce(Object.assign(new Error('rollback fault'), { code: 'XX000' }));
await expect(
db.transaction(async () => {
throw original;
}),
).rejects.toBe(original);
expect(release).toHaveBeenCalledWith(true);
await db.close();
});
it('reports a lost COMMIT acknowledgement without issuing a misleading ROLLBACK', async () => {
const { db, query, release } = setup();
query.mockImplementation(async (text) => {
if (text === 'COMMIT')
throw Object.assign(new Error('private commit detail'), { code: 'ECONNRESET' });
return { rows: [], rowCount: 0 };
});
await expect(db.transaction(async () => 'value')).rejects.toThrow(safeMessage);
expect(query.mock.calls.map(([text]) => text)).toEqual(['BEGIN', 'COMMIT']);
expect(release).toHaveBeenCalledTimes(1);
expect(release).toHaveBeenCalledWith(true);
await db.close();
});
});

View file

@ -0,0 +1,190 @@
import pg from 'pg';
import type { QueryResult, QueryResultRow } from 'pg';
export interface PostgresOptions {
connectionString: string;
max: number;
maxWaiting: number;
acquireTimeoutMs: number;
queryTimeoutMs: number;
idleTimeoutMs: number;
}
export interface SqlExecutor {
query<R extends QueryResultRow = QueryResultRow>(
text: string,
values?: unknown[],
): Promise<QueryResult<R>>;
}
export interface PostgresSession extends SqlExecutor {
transaction<T>(operation: (sql: SqlExecutor) => Promise<T>): Promise<T>;
}
export interface PostgresDatabase extends PostgresSession {
session<T>(operation: (sql: PostgresSession) => Promise<T>): Promise<T>;
transaction<T>(operation: (sql: SqlExecutor) => Promise<T>): Promise<T>;
close(): Promise<void>;
}
export function createPostgresDatabase(options: PostgresOptions): PostgresDatabase {
const pool = new pg.Pool({
connectionString: options.connectionString,
max: options.max,
connectionTimeoutMillis: options.acquireTimeoutMs,
idleTimeoutMillis: options.idleTimeoutMs,
statement_timeout: options.queryTimeoutMs,
query_timeout: options.queryTimeoutMs,
idle_in_transaction_session_timeout: options.queryTimeoutMs,
application_name: 'librechat',
});
// Idle-client errors must not become uncaught exceptions or expose connection details.
pool.on('error', () => undefined);
let closing = false;
let closePromise: Promise<void> | undefined;
function connectionError(): Error {
return new Error('PostgreSQL connection unavailable');
}
function isConnectionError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const code = 'code' in error && typeof error.code === 'string' ? error.code : '';
const severity = 'severity' in error ? error.severity : undefined;
return (
severity === 'FATAL' ||
severity === 'PANIC' ||
code?.startsWith('08') === true ||
['57P01', '57P02', '57P03', 'ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'ECONNREFUSED'].includes(
code ?? '',
) ||
[
'Connection terminated unexpectedly',
'Connection terminated',
'Client has encountered a connection error and is not queryable',
'Client was closed and is not queryable',
'Query read timeout',
].includes(error.message)
);
}
function connect() {
if (closing || pool.waitingCount >= options.maxWaiting) {
return Promise.reject(new Error('PostgreSQL connection admission unavailable'));
}
// The callback installs protection before pg-pool's synchronous handoff returns.
return new Promise<{
query: SqlExecutor['query'];
failure: Promise<never>;
assertHealthy(): void;
discard(): void;
release(): void;
}>((resolve, reject) => {
pool.connect((error, client) => {
if (error || !client) {
reject(connectionError());
return;
}
let failed: Error | undefined;
let released = false;
let discard = false;
let rejectFailure: (error: Error) => void;
const failure = new Promise<never>((_, reject) => {
rejectFailure = reject;
});
void failure.catch(() => undefined);
const onError = () => {
if (failed) return;
failed = connectionError();
rejectFailure(failed);
};
client.on('error', onError);
const assertHealthy = () => {
if (failed) throw failed;
if (released) throw connectionError();
};
resolve({
failure,
assertHealthy,
discard() {
discard = true;
},
async query<R extends QueryResultRow>(text: string, values?: unknown[]) {
assertHealthy();
try {
const result = await client.query<R>(text, values);
assertHealthy();
return result;
} catch (error) {
if (isConnectionError(error)) onError();
throw failed ?? error;
}
},
release() {
if (released) return;
released = true;
try {
client.release(discard || Boolean(failed));
} finally {
// release restores pg-pool's idle listener before ours is removed.
client.removeListener('error', onError);
}
},
});
});
}).catch(() => {
throw connectionError();
});
}
const database: PostgresDatabase = {
query: (text, values) => database.session((session) => session.query(text, values)),
async transaction<T>(operation: (sql: SqlExecutor) => Promise<T>): Promise<T> {
return database.session((session) => session.transaction(operation));
},
async session<T>(operation: (sql: PostgresSession) => Promise<T>): Promise<T> {
const lease = await connect();
let inTransaction = false;
const session: PostgresSession = {
query: lease.query,
async transaction<U>(work: (sql: SqlExecutor) => Promise<U>): Promise<U> {
lease.assertHealthy();
if (inTransaction) throw new Error('Nested PostgreSQL transactions are not supported');
inTransaction = true;
try {
await lease.query('BEGIN');
const value = await work(session);
await lease.query('COMMIT');
return value;
} catch (error) {
// A lost COMMIT acknowledgement has an unknown outcome. Do not claim rollback.
lease.assertHealthy();
try {
await lease.query('ROLLBACK');
} catch {
lease.discard();
}
lease.assertHealthy();
throw error;
} finally {
inTransaction = false;
}
},
};
const work = Promise.resolve()
.then(() => {
lease.assertHealthy();
return operation(session);
})
.then((value) => {
lease.assertHealthy();
return value;
})
.finally(() => lease.release());
// Report disconnects promptly, but retain ownership until the callback settles.
return Promise.race([work, lease.failure]);
},
close() {
closing = true;
closePromise ??= pool.end();
return closePromise;
},
};
return database;
}

View file

@ -0,0 +1,51 @@
import { randomBytes } from 'node:crypto';
import type { PostgresDatabase, PostgresOptions } from './pool';
import { createPostgresDatabase } from './pool';
export const postgresTestOptions: PostgresOptions = {
connectionString: process.env.PG_TEST_URL ?? '',
max: 4,
maxWaiting: 20,
acquireTimeoutMs: 1000,
queryTimeoutMs: 5000,
idleTimeoutMs: 1000,
};
/** Each spec/Stryker worker gets its own schema; network and constraints remain real. */
export async function createPostgresTestDatabase(): Promise<{
db: PostgresDatabase;
options: PostgresOptions;
schema: string;
}> {
const admin = createPostgresDatabase(postgresTestOptions);
const schema = `favorites_test_${randomBytes(8).toString('hex')}`;
try {
await admin.query(`CREATE SCHEMA ${schema}`);
const url = new URL(postgresTestOptions.connectionString);
url.searchParams.set('options', `-c search_path=${schema}`);
const options = { ...postgresTestOptions, connectionString: url.toString() };
const database = createPostgresDatabase(options);
let closing: Promise<void> | undefined;
const db: PostgresDatabase = {
...database,
close() {
closing ??= (async () => {
try {
await database.close();
} finally {
try {
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
} finally {
await admin.close();
}
}
})();
return closing;
},
};
return { db, options, schema };
} catch (error) {
await admin.close();
throw error;
}
}

View file

@ -170,6 +170,10 @@ const userSchema: Schema<IUser> = new Schema<IUser>(
type: Date,
select: false,
},
agentTriggerDeletionAttemptId: {
type: String,
select: false,
},
subagentAdmissionFences: {
type: [
{

View file

@ -33,3 +33,11 @@ export interface RemoveToolFavoriteResult {
ok: boolean;
removed: boolean;
}
export interface ToolFavoriteMethods {
getToolFavorites(userId: string): Promise<IToolFavoriteLean[]>;
addToolFavorite(params: ToolFavoriteParams): Promise<AddToolFavoriteResult>;
removeToolFavorite(params: ToolFavoriteParams): Promise<RemoveToolFavoriteResult>;
deleteUserToolFavorites(userId: string, deletionAttemptId?: string): Promise<number>;
cancelUserToolFavoriteDeletion?(userId: string, deletionAttemptId: string): Promise<void>;
}

View file

@ -69,6 +69,8 @@ export interface IUser extends Document {
termsAcceptedSource?: 'migration';
/** Internal fence that prevents agent-trigger admission during account deletion. */
agentTriggerDeletionStartedAt?: Date;
/** Durable identity shared by every offline recovery of this erasure attempt. */
agentTriggerDeletionAttemptId?: string;
/** Expiring fences closing subagent admission while bulk deletions drain. */
subagentAdmissionFences?: Array<{
token: string;

View file

@ -0,0 +1,331 @@
import { fileURLToPath } from 'node:url';
import { join, resolve } from 'node:path';
import { parseArgs, isDeepStrictEqual } from 'node:util';
import {
appendFileSync,
mkdirSync,
readFileSync,
readdirSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { shardTargets } from './shard.mts';
interface Options {
head: string;
workspace: string;
count: number;
}
type ObjectValue = Record<string, unknown>;
const statuses = [
'Killed',
'Survived',
'Timeout',
'NoCoverage',
'CompileError',
'RuntimeError',
'Ignored',
'Pending',
] as const;
type Status = (typeof statuses)[number];
function requireValue(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
function object(value: unknown): ObjectValue {
requireValue(
value !== null && typeof value === 'object' && !Array.isArray(value),
'Expected JSON object',
);
return value as ObjectValue;
}
function array(value: unknown): unknown[] {
requireValue(Array.isArray(value), 'Expected JSON array');
return value;
}
function text(value: unknown): string {
requireValue(typeof value === 'string' && value.length > 0, 'Expected nonempty string');
return value;
}
function integer(value: unknown, minimum = 0): number {
requireValue(
Number.isSafeInteger(value) && typeof value === 'number' && value >= minimum,
'Expected nonnegative integer',
);
return value;
}
function path(value: unknown): string {
const file = text(value);
requireValue(
!file.includes('\\') &&
!file.startsWith('/') &&
file.split('/').every((part) => part && part !== '.' && part !== '..'),
'Expected repository-relative file',
);
return file;
}
function location(value: unknown): ObjectValue {
const span = object(value);
const start = object(span.start);
const end = object(span.end);
const line = integer(start.line, 1);
const column = integer(start.column, 1);
const endLine = integer(end.line, 1);
const endColumn = integer(end.column, 1);
requireValue(endLine > line || (endLine === line && endColumn >= column), 'Invalid location');
return span;
}
function targets(value: unknown) {
const files = new Set<string>();
return array(value).map((entry) => {
const target = object(entry);
const file = path(target.file);
requireValue(!files.has(file), 'Duplicate target file');
files.add(file);
const functions = array(target.functions);
requireValue(functions.length > 0 && array(target.mutate).length > 0, 'Empty mutation target');
for (const fn of functions) {
const row = object(fn);
text(row.function);
integer(row.line, 1);
integer(row.column, 1);
location(row.location);
}
for (const pattern of array(target.mutate)) text(pattern);
return { file, functions, original: target };
});
}
/** Validate complete file shards without changing their reports or local test/mutant IDs. */
export function aggregate(inputs: readonly unknown[], options: Options) {
integer(options.count, 1);
text(options.head);
text(options.workspace);
requireValue(inputs.length === options.count, 'Wrong shard directory count');
const shards = inputs
.map((input) => {
const row = object(input);
const selection = object(row.selection);
const shard = object(selection.shard);
const index = integer(shard.index, 1);
requireValue(
index <= options.count && shard.total === options.count,
'Wrong shard index or count',
);
requireValue(selection.head === options.head, 'Wrong selection head');
requireValue(selection.workspace === options.workspace, 'Wrong selection workspace');
text(selection.base);
const files = array(selection.files).map((entry) => path(object(entry).file));
requireValue(new Set(files).size === files.length, 'Duplicate selected file');
const all = targets(row.allFunctions);
requireValue(
all.every(({ file }) => files.includes(file)),
'Target outside selected files',
);
targets(row.functions);
const expected = shardTargets(all, { index, total: options.count });
requireValue(
isDeepStrictEqual(
row.functions,
expected.map(({ original }) => original),
),
'Wrong file/function shard partition',
);
return {
index,
selection,
all,
expected,
report: object(row.report),
progress: object(row.progress),
};
})
.sort((a, b) => a.index - b.index);
requireValue(
new Set(shards.map(({ index }) => index)).size === options.count,
'Duplicate shard index',
);
const first = shards[0]!;
const counts: Record<Status, number> = {
Killed: 0,
Survived: 0,
Timeout: 0,
NoCoverage: 0,
CompileError: 0,
RuntimeError: 0,
Ignored: 0,
Pending: 0,
};
const identities = new Set<string>();
let staticTotal = 0;
for (const shard of shards) {
requireValue(shard.selection.base === first.selection.base, 'Inconsistent selection base');
requireValue(
isDeepStrictEqual(shard.selection.files, first.selection.files),
'Inconsistent original selected files',
);
requireValue(
isDeepStrictEqual(
shard.all.map(({ original }) => original),
first.all.map(({ original }) => original),
),
'Inconsistent all-functions manifest',
);
requireValue(shard.report.schemaVersion === '1.0', 'Unsupported mutation report schema');
let total = 0;
let staticCount = 0;
const ids = new Set<string>();
for (const [file, entry] of Object.entries(object(shard.report.files))) {
path(file);
requireValue(
shard.expected.some((target) => target.file === file),
'Report file outside shard targets',
);
for (const value of array(object(entry).mutants)) {
const mutant = object(value);
const id = text(mutant.id);
requireValue(!ids.has(id), 'Duplicate local mutant ID');
ids.add(id);
const span = location(mutant.location);
const mutator = text(mutant.mutatorName);
requireValue(typeof mutant.replacement === 'string', 'Invalid mutant replacement');
requireValue(
mutant.static === undefined || typeof mutant.static === 'boolean',
'Invalid static flag',
);
const status = statuses.find((name) => name === mutant.status);
requireValue(status, 'Unknown mutant status');
requireValue(
status !== 'RuntimeError' && status !== 'Pending',
'Incomplete mutant outcome',
);
const identity = JSON.stringify([
file,
object(span.start).line,
object(span.start).column,
object(span.end).line,
object(span.end).column,
mutator,
mutant.replacement,
]);
requireValue(!identities.has(identity), 'Duplicate mutation identity');
identities.add(identity);
counts[status]++;
total++;
if (mutant.static === true) staticCount++;
}
}
const progress = shard.progress;
requireValue(
progress.complete === true && progress.completed === total && progress.total === total,
'Incomplete mutation progress',
);
requireValue(
progress.staticCompleted === staticCount && progress.staticTotal === staticCount,
'Incomplete static mutation progress',
);
staticTotal += staticCount;
}
const detected = counts.Killed + counts.Timeout;
const tested = detected + counts.Survived + counts.NoCoverage;
const total = Object.values(counts).reduce((sum, count) => sum + count, 0);
return {
head: options.head,
base: first.selection.base,
workspace: options.workspace,
shardCount: options.count,
complete: true,
files: first.selection.files,
functions: first.all.map(({ original }) => original),
total,
completed: total,
staticTotal,
staticCompleted: staticTotal,
counts,
score: tested ? (100 * detected) / tested : null,
shards: shards.map(({ index, report, progress }) => ({ index, report, progress })),
};
}
export function aggregateMarkdown(result: ReturnType<typeof aggregate>): string {
const score = result.score === null ? 'N/A (no valid mutants)' : `${result.score.toFixed(2)}%`;
return [
`## ${result.workspace}: mutation testing (advisory)`,
'',
`Complete: ${result.shardCount} shards, ${result.completed}/${result.total} mutants. Static mutations: ${result.staticCompleted}/${result.staticTotal}.`,
'',
`Mutation score: ${score}. Compile errors and ignored mutants are excluded from the score, not the report.`,
'',
'| Result | Mutants |',
'| --- | ---: |',
...statuses.map((status) => `| ${status} | ${result.counts[status]} |`),
'',
'Aggregate JSON retains each shard report, including source, test metadata, and unchanged local IDs.',
'',
].join('\n');
}
function main() {
const { values } = parseArgs({
options: {
directory: { type: 'string' },
head: { type: 'string' },
workspace: { type: 'string' },
count: { type: 'string' },
output: { type: 'string' },
},
});
requireValue(
values.directory && values.head && values.workspace && values.count && values.output,
'Required: --directory --head --workspace --count --output',
);
for (const name of ['aggregate.json', 'summary.md']) {
rmSync(join(values.output, name), { force: true });
}
requireValue(/^[1-9]\d*$/u.test(values.count), 'Invalid shard count');
const entries = readdirSync(values.directory, { withFileTypes: true });
requireValue(
entries.length === Number(values.count) && entries.every((entry) => entry.isDirectory()),
'Wrong shard directory count',
);
const inputs = entries.map((entry) => {
const read = (name: string): unknown => {
try {
return JSON.parse(readFileSync(join(values.directory!, entry.name, name), 'utf8'));
} catch {
throw new Error(`Missing or malformed shard ${name}`);
}
};
return {
selection: read('selection.json'),
allFunctions: read('all-functions.json'),
functions: read('functions.json'),
report: read('mutation.json'),
progress: read('progress.json'),
};
});
const result = aggregate(inputs, {
head: values.head,
workspace: values.workspace,
count: Number(values.count),
});
mkdirSync(values.output, { recursive: true });
writeFileSync(join(values.output, 'aggregate.json'), JSON.stringify(result, null, 2));
const markdown = aggregateMarkdown(result);
writeFileSync(join(values.output, 'summary.md'), markdown);
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown);
console.log(markdown);
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
try {
main();
} catch {
console.error(
'Mutation aggregation failed: shard evidence is missing, inconsistent, malformed, or incomplete.',
);
process.exitCode = 1;
}
}

View file

@ -0,0 +1,351 @@
import { test } from 'node:test';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { aggregate, aggregateMarkdown } from './aggregate.mts';
import { shardTargets } from './shard.mts';
const options = { head: 'head-sha', workspace: 'api', count: 3 };
const point = (line: number) => ({ line, column: 1 });
const span = (line: number) => ({ start: point(line), end: point(line + 1) });
const functions = Array.from({ length: 5 }, (_, index) => ({
file: `packages/api/src/file${index}.ts`,
functions: Array.from({ length: [2, 4, 1, 3, 2][index] }, (_, fn) => fn * 2 + 1).map((line) => ({
function: `f${line}`,
line,
column: 1,
location: span(line),
})),
mutate: [`packages/api/src/file${index}.ts:1:0-4:0`],
}));
const files = functions.map(({ file }) => ({ file, ranges: [{ start: 1, count: 4 }] }));
const outcomes = ['Killed', 'Survived', 'Timeout', 'NoCoverage', 'CompileError', 'Ignored'];
function fixtures() {
return Array.from({ length: 3 }, (_, shard) => {
const chosen = shardTargets(functions, { index: shard + 1, total: 3 });
const reportFiles = Object.fromEntries(
chosen.map(({ file }, index) => [
file,
{
source: 'function source() {}',
language: 'typescript',
mutants: [0, 1].map((position) => ({
id: String(index * 2 + position),
location: span(position + 1),
mutatorName: 'BooleanLiteral',
replacement: 'false',
status: outcomes[(shard * 2 + position) % outcomes.length],
static: position === 1,
coveredBy: ['0'],
killedBy: position === 0 ? ['0'] : [],
statusReason: 'preserve reason',
})),
},
]),
);
const total = chosen.length * 2;
return {
selection: {
head: options.head,
base: 'base-sha',
workspace: options.workspace,
files: structuredClone(files),
shard: { index: shard + 1, total: 3 },
},
allFunctions: structuredClone(functions),
functions: structuredClone([...chosen]),
report: {
schemaVersion: '1.0',
files: reportFiles,
testFiles: { 'tests/file.ts': { tests: [{ id: '0', name: 'test' }] } },
framework: { name: 'Stryker' },
},
progress: {
complete: true,
completed: total,
total,
staticCompleted: chosen.length,
staticTotal: chosen.length,
},
};
});
}
test('aggregates every file, function, outcome, static mutant, and unchanged raw report', () => {
const inputs = fixtures();
const snapshot = structuredClone(inputs);
const result = aggregate(inputs, options);
assert.equal(result.complete, true);
assert.equal(result.total, 10);
assert.equal(result.completed, 10);
assert.equal(result.staticTotal, 5);
assert.equal(result.staticCompleted, 5);
assert.deepEqual(result.functions, functions);
assert.deepEqual(result.files, files);
assert.deepEqual(result.counts, {
Killed: 2,
Survived: 2,
Timeout: 2,
NoCoverage: 2,
CompileError: 1,
RuntimeError: 0,
Ignored: 1,
Pending: 0,
});
assert.equal(result.score, 50);
assert.deepEqual(
result.shards.map(({ report }) => report),
inputs.map(({ report }) => report),
);
assert.deepEqual(inputs, snapshot);
assert.match(aggregateMarkdown(result), /CompileError \| 1/);
assert.match(aggregateMarkdown(result), /Static mutations: 5\/5/);
});
test('shard directory ordering does not affect results or identities', () => {
const inputs = fixtures();
assert.deepEqual(
aggregate([inputs[2], inputs[0], inputs[1]], options),
aggregate(inputs, options),
);
});
const invalid: [string, (inputs: ReturnType<typeof fixtures>) => void][] = [
[
'missing shard',
(inputs) => {
inputs.pop();
},
],
[
'extra shard',
(inputs) => {
inputs.push(structuredClone(inputs[0]!));
},
],
[
'duplicate shard index',
(inputs) => {
inputs[1]!.selection.shard.index = 1;
},
],
[
'wrong shard count',
(inputs) => {
inputs[0]!.selection.shard.total = 4;
},
],
[
'wrong head',
(inputs) => {
inputs[0]!.selection.head = 'other';
},
],
[
'inconsistent base',
(inputs) => {
inputs[1]!.selection.base = 'other';
},
],
[
'wrong workspace',
(inputs) => {
inputs[0]!.selection.workspace = 'client';
},
],
[
'inconsistent original files',
(inputs) => {
inputs[0]!.selection.files.pop();
},
],
[
'inconsistent full manifest',
(inputs) => {
inputs[1]!.allFunctions[0]!.functions[0]!.function = 'other';
},
],
[
'dropped target file',
(inputs) => {
inputs[0]!.functions.pop();
},
],
[
'dropped target function',
(inputs) => {
inputs[0]!.functions[0]!.functions.pop();
},
],
[
'wrong round-robin partition',
(inputs) => {
inputs[0]!.functions.reverse();
},
],
[
'out-of-shard report file',
(inputs) => {
inputs[0]!.report.files['packages/api/src/other.ts'] = structuredClone(
Object.values(inputs[0]!.report.files)[0]!,
);
},
],
[
'runtime error',
(inputs) => {
Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'RuntimeError';
},
],
[
'pending mutant',
(inputs) => {
Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Pending';
},
],
[
'unknown status',
(inputs) => {
Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Unknown';
},
],
[
'incomplete progress',
(inputs) => {
inputs[0]!.progress.complete = false;
},
],
[
'wrong completed count',
(inputs) => {
inputs[0]!.progress.completed--;
},
],
[
'wrong planned count',
(inputs) => {
inputs[0]!.progress.total++;
},
],
[
'wrong static completed count',
(inputs) => {
inputs[0]!.progress.staticCompleted--;
},
],
[
'wrong static total count',
(inputs) => {
inputs[0]!.progress.staticTotal++;
},
],
[
'duplicate mutant identity',
(inputs) => {
const mutants = Object.values(inputs[0]!.report.files)[0]!.mutants;
mutants[1] = { ...mutants[0]!, id: 'new' };
},
],
[
'duplicate local ID',
(inputs) => {
Object.values(inputs[0]!.report.files)[0]!.mutants[1]!.id = '0';
},
],
];
for (const [name, change] of invalid) {
test(`fails closed: ${name}`, () => {
const inputs = fixtures();
change(inputs);
assert.throws(() => aggregate(inputs, options));
});
}
test('requires explicit complete evidence for zero-mutant shards', () => {
const inputs = fixtures();
for (const input of inputs) {
input.allFunctions = [];
input.functions = [];
input.report.files = {};
input.progress = { complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 };
}
const result = aggregate(inputs, options);
assert.equal(result.total, 0);
assert.equal(result.score, null);
assert.deepEqual(result.functions, []);
const missing: unknown[] = structuredClone(inputs);
missing[0] = { ...inputs[0], progress: undefined };
assert.throws(() => aggregate(missing, options));
});
test('chosen function files may legitimately produce no mutants', () => {
const inputs = fixtures();
inputs[0]!.report.files = {};
inputs[0]!.progress = {
complete: true,
completed: 0,
total: 0,
staticCompleted: 0,
staticTotal: 0,
};
assert.equal(aggregate(inputs, options).total, 6);
});
test('malformed evidence is rejected without trusting TypeScript casts', () => {
for (const bad of [null, [], {}, { selection: null }]) {
assert.throws(() => aggregate([bad, bad, bad], options));
}
const inputs = fixtures();
assert.throws(() => aggregate(inputs, { ...options, count: 0 }));
assert.throws(() => aggregate(inputs, { ...options, head: '' }));
});
test('CLI uses only built-ins, reads unmerged directories, and fails without a green output', (t) => {
const directory = mkdtempSync(join(tmpdir(), 'mutation-aggregate-'));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const input = join(directory, 'reports');
const output = join(directory, 'output');
for (const [index, shard] of fixtures().entries()) {
const folder = join(input, `artifact-${2 - index}`);
mkdirSync(folder, { recursive: true });
for (const [name, value] of Object.entries({
selection: shard.selection,
'all-functions': shard.allFunctions,
functions: shard.functions,
mutation: shard.report,
progress: shard.progress,
})) {
writeFileSync(join(folder, `${name}.json`), JSON.stringify(value));
}
}
const args = [
'scripts/quality/aggregate.mts',
'--directory',
input,
'--head',
options.head,
'--workspace',
options.workspace,
'--count',
'3',
'--output',
output,
];
const stepSummary = join(directory, 'step-summary.md');
const result = spawnSync(process.execPath, args, {
encoding: 'utf8',
env: { ...process.env, GITHUB_STEP_SUMMARY: stepSummary },
});
assert.equal(result.status, 0, result.stderr);
assert.equal(JSON.parse(readFileSync(join(output, 'aggregate.json'), 'utf8')).total, 10);
assert.match(readFileSync(join(output, 'summary.md'), 'utf8'), /50.00%/);
assert.equal(readFileSync(stepSummary, 'utf8'), readFileSync(join(output, 'summary.md'), 'utf8'));
writeFileSync(join(input, 'artifact-0', 'progress.json'), 'not-json-secret-payload');
const failed = spawnSync(process.execPath, args, { encoding: 'utf8' });
assert.equal(failed.status, 1);
assert.doesNotMatch(failed.stderr, /secret-payload/);
assert.throws(() => readFileSync(join(output, 'summary.md')));
});

View file

@ -14,6 +14,7 @@ import {
import { mutationSelection, riskMarkdown, riskReport } from './risk.mts';
import { changes, git, WORKSPACES, workspaceFor } from './changes.mts';
import { parseShard, shardTargets } from './shard.mts';
import { coverageArgs } from './jest.mts';
const { values } = parseArgs({
@ -22,14 +23,17 @@ const { values } = parseArgs({
head: { type: 'string', default: 'HEAD' },
workspace: { type: 'string' },
check: { type: 'string' },
shard: { type: 'string' },
},
});
const workspace = WORKSPACES.find(({ name }) => name === values.workspace);
if (!workspace || !['risk', 'mutation'].includes(values.check ?? '')) {
throw new Error(
'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation',
'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation [--shard INDEX/COUNT]',
);
}
const shard = parseShard(values.shard, values.check);
const shardLabel = shard ? ` (shard ${shard.index}/${shard.total})` : '';
const selection = changes(values.base, values.head);
if (selection.head !== git(['rev-parse', 'HEAD']).trim()) {
throw new Error('Check out the selected head before measuring its source');
@ -38,14 +42,42 @@ const files = selection.files.filter(({ file }) => workspaceFor(file)?.name ===
const output = resolve('coverage/changes', workspace.name, values.check!);
rmSync(output, { recursive: true, force: true });
mkdirSync(output, { recursive: true });
writeFileSync(`${output}/selection.json`, JSON.stringify({ ...selection, files }, null, 2));
writeFileSync(
`${output}/selection.json`,
JSON.stringify(
{ ...selection, workspace: workspace.name, files, ...(shard ? { shard } : {}) },
null,
2,
),
);
const summary = (markdown: string) => {
console.log(markdown);
writeFileSync(`${output}/summary.md`, markdown);
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown);
};
const emptyMutationReports = () => {
writeFileSync(
`${output}/mutation.json`,
JSON.stringify({ schemaVersion: '1.0', files: {} }, null, 2),
);
writeFileSync(
`${output}/progress.json`,
JSON.stringify(
{ complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 },
null,
2,
),
);
};
if (!files.length) {
summary(`## ${workspace.name}: ${values.check}\n\nNo changed source files.\n`);
if (shard) {
writeFileSync(`${output}/all-functions.json`, '[]');
writeFileSync(`${output}/functions.json`, '[]');
emptyMutationReports();
}
summary(
`## ${workspace.name}: ${values.check}${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No changed source files.'}\n`,
);
process.exit(0);
}
// Test binaries and configuration belong to the measured head, not the tooling checkout.
@ -106,10 +138,17 @@ if (values.check === 'risk') {
} else {
process.env.NODE_ENV = env.NODE_ENV;
process.env.BABEL_ENV = env.BABEL_ENV;
const targets = mutationSelection(files, selection.base);
const allTargets = mutationSelection(files, selection.base);
const targets = shardTargets(allTargets, shard);
if (shard) {
writeFileSync(`${output}/all-functions.json`, JSON.stringify(allTargets, null, 2));
}
writeFileSync(`${output}/functions.json`, JSON.stringify(targets, null, 2));
if (!targets.length) {
summary(`## ${workspace.name}: mutation testing\n\nNo added or edited functions.\n`);
if (shard) emptyMutationReports();
summary(
`## ${workspace.name}: mutation testing${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No added or edited functions.'}\n`,
);
process.exit(0);
}
const started = Date.now();
@ -176,7 +215,7 @@ if (values.check === 'risk') {
const score = tested ? `${((100 * detected) / tested).toFixed(2)}%` : 'N/A (no valid mutants)';
summary(
[
`## ${workspace.name}: mutation testing (advisory)`,
`## ${workspace.name}: mutation testing (advisory)${shardLabel}`,
'',
`Changed functions: ${targets.reduce((total, target) => total + target.functions.length, 0)} in ${targets.length} files. Static mutations are included. Duration: ${((Date.now() - started) / 60000).toFixed(1)} minutes.`,
'',
@ -198,7 +237,7 @@ if (values.check === 'risk') {
if (runtimeErrors) process.exitCode = 1;
} catch (error) {
summary(
`## ${workspace.name}: mutation testing\n\nStryker failed. This is not a mutation score. See the test log.\n`,
`## ${workspace.name}: mutation testing${shardLabel}\n\nStryker failed. This is not a mutation score. See the test log.\n`,
);
throw error;
}

30
scripts/quality/shard.mts Normal file
View file

@ -0,0 +1,30 @@
export interface Shard {
index: number;
total: number;
}
export function parseShard(
value: string | undefined,
check: string | undefined,
): Shard | undefined {
if (value === undefined) return undefined;
if (check !== 'mutation') throw new Error('--shard is only allowed for mutation checks');
if (!/^[1-9]\d*\/[1-9]\d*$/u.test(value)) {
throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT');
}
const [index, total] = value.split('/').map(Number);
if (!Number.isSafeInteger(index) || !Number.isSafeInteger(total) || index > total) {
throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT');
}
return { index, total };
}
export function shardTargets<T extends { file: string; functions: readonly unknown[] }>(
targets: readonly T[],
shard?: Shard,
): readonly T[] {
if (!shard) return targets;
return [...targets]
.sort((a, b) => b.functions.length - a.functions.length || a.file.localeCompare(b.file))
.filter((_target, index) => index % shard.total === shard.index - 1);
}

View file

@ -0,0 +1,234 @@
import { test } from 'node:test';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { parseShard, shardTargets } from './shard.mts';
import { mutationTarget } from './risk.mts';
import { git } from './changes.mts';
const root = fileURLToPath(new URL('../../', import.meta.url));
const runner = fileURLToPath(new URL('./run.mts', import.meta.url));
const targets = Object.freeze(
Array.from({ length: 8 }, (_value, index) =>
Object.freeze(
mutationTarget(
`file-${index}.ts`,
'const first = (ok) => ok ? true : false;\nconst second = () => false;',
[{ start: 1, count: 2 }],
),
),
),
);
for (const [index, expected] of [
[1, [0, 3, 6]],
[2, [1, 4, 7]],
[3, [2, 5]],
] as const) {
test(`shard ${index}/3 retains whole files and their full function and mutation ranges`, () => {
const shard = parseShard(`${index}/3`, 'mutation');
const chosen = shardTargets(targets, shard);
assert.deepEqual(
chosen,
expected.map((offset) => targets[offset]),
);
for (const [position, offset] of expected.entries()) {
assert.equal(chosen[position], targets[offset]);
assert.equal(chosen[position].functions.length, 2);
assert.equal(chosen[position].mutate, targets[offset].mutate);
}
assert.deepEqual(shardTargets(targets, shard), chosen);
});
}
test('three shards contain every full target exactly once without mutating input order', () => {
const before = JSON.stringify(targets);
const union = [1, 2, 3].flatMap((index) => shardTargets(targets, { index, total: 3 }));
assert.equal(union.length, targets.length);
assert.equal(new Set(union).size, targets.length);
assert.deepEqual(new Set(union), new Set(targets));
assert.deepEqual(
union.flatMap(({ mutate }) => mutate).sort(),
targets.flatMap(({ mutate }) => mutate).sort(),
);
assert.equal(JSON.stringify(targets), before);
});
test('omitting --shard retains all original targets and allows risk checks', () => {
assert.equal(parseShard(undefined, 'mutation'), undefined);
assert.equal(parseShard(undefined, 'risk'), undefined);
assert.equal(shardTargets(targets), targets);
assert.deepEqual(shardTargets(targets, parseShard('1/1', 'mutation')), targets);
});
test('equal function counts use file order regardless of input order without mutating input', () => {
const reversed = Object.freeze([...targets].reverse());
const before = JSON.stringify(reversed);
for (const index of [1, 2, 3]) {
const shard = parseShard(`${index}/3`, 'mutation');
assert.deepEqual(shardTargets(reversed, shard), shardTargets(targets, shard));
}
assert.equal(JSON.stringify(reversed), before);
assert.equal(shardTargets(reversed), reversed);
assert.deepEqual(shardTargets(reversed, parseShard('1/1', 'mutation')), targets);
});
test('function-weighted round robin spreads the known target file workload across three shards', () => {
const workload = Object.freeze(
(
[
['pool.ts', 25],
['import.ts', 19],
['nativefavorite.ts', 13],
['mongo.ts', 11],
['migrations.ts', 9],
['user.ts', 8],
['backend.ts', 3],
['favorites/mongo.ts', 1],
['methods/createMethods.ts', 1],
] as const
).map(([file, count]) =>
Object.freeze({
file,
functions: Object.freeze(Array.from({ length: count }, (_value, index) => index)),
}),
),
);
const before = JSON.stringify(workload);
const expected = [
['pool.ts', 'mongo.ts', 'backend.ts'],
['import.ts', 'migrations.ts', 'favorites/mongo.ts'],
['nativefavorite.ts', 'user.ts', 'methods/createMethods.ts'],
];
const shuffled = Object.freeze([...workload].reverse());
const shards = [1, 2, 3].map((index) => {
const shard = parseShard(`${index}/3`, 'mutation');
const chosen = shardTargets(shuffled, shard);
assert.deepEqual(
chosen.map(({ file }) => file),
expected[index - 1],
);
assert.deepEqual(chosen, shardTargets(workload, shard));
return chosen;
});
const union = shards.flat();
assert.equal(union.length, workload.length);
assert.equal(new Set(union).size, workload.length);
assert.deepEqual(new Set(union), new Set(workload));
assert.deepEqual(
shards.map((chosen) => chosen.reduce((sum, target) => sum + target.functions.length, 0)),
[39, 29, 22],
);
assert.equal(JSON.stringify(workload), before);
assert.equal(shardTargets(shuffled), shuffled);
});
test('a shard beyond the number of selected target files is empty', () => {
assert.deepEqual(shardTargets(targets.slice(0, 1), parseShard('2/3', 'mutation')), []);
assert.deepEqual(shardTargets([], parseShard('1/3', 'mutation')), []);
});
for (const value of [
'0/2',
'4/3',
'foo',
'1/0',
'1.1/2',
'1/2/3',
'',
'-1/2',
'1/-2',
'01/2',
'1/02',
'+1/2',
' 1/2',
'1/2 ',
'1e1/20',
'1/9007199254740992',
]) {
test(`invalid shard ${JSON.stringify(value)} is rejected`, () => {
assert.throws(() => parseShard(value, 'mutation'), /--shard must be INDEX\/COUNT/u);
});
}
test('risk checks reject --shard before attempting source selection', () => {
assert.throws(() => parseShard('1/3', 'risk'), /only allowed for mutation/u);
const result = spawnSync(
process.execPath,
[
runner,
'--workspace',
'data-schemas',
'--check',
'risk',
'--shard',
'1/3',
'--base',
'nonexistent-shard-test-ref',
],
{ cwd: root, encoding: 'utf8', timeout: 15000 },
);
assert.notEqual(result.status, 0);
assert.match(result.stderr, /--shard is only allowed for mutation checks/u);
assert.doesNotMatch(result.stderr, /Not a valid object name/u);
});
test('empty sharded CLI runs pass with truthful metadata; unsharded output stays unchanged', (t) => {
const directory = mkdtempSync(join(tmpdir(), 'quality-shard-'));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const env = {
...process.env,
GIT_DIR: git(['rev-parse', '--absolute-git-dir'], root).trim(),
GIT_WORK_TREE: directory,
GITHUB_STEP_SUMMARY: '',
};
const output = join(directory, 'coverage/changes/data-schemas/mutation');
for (const shard of ['2/3', undefined]) {
const result = spawnSync(
process.execPath,
[
runner,
'--workspace',
'data-schemas',
'--check',
'mutation',
'--base',
'HEAD',
...(shard ? ['--shard', shard] : []),
],
{ cwd: directory, env, encoding: 'utf8', timeout: 15000 },
);
assert.equal(result.status, 0, `${result.error ?? ''}\n${result.stdout}\n${result.stderr}`);
const selection = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8'));
assert.equal(selection.workspace, 'data-schemas');
assert.deepEqual(selection.files, []);
if (shard) {
assert.deepEqual(selection.shard, { index: 2, total: 3 });
assert.deepEqual(JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')), {
schemaVersion: '1.0',
files: {},
});
assert.deepEqual(JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')), {
complete: true,
completed: 0,
total: 0,
staticCompleted: 0,
staticTotal: 0,
});
assert.deepEqual(JSON.parse(readFileSync(join(output, 'all-functions.json'), 'utf8')), []);
assert.deepEqual(JSON.parse(readFileSync(join(output, 'functions.json'), 'utf8')), []);
assert.match(result.stdout, /mutation \(shard 2\/3\)/u);
assert.match(result.stdout, /No changed functions for shard\./u);
} else {
assert.equal(selection.shard, undefined);
assert.equal(existsSync(join(output, 'all-functions.json')), false);
assert.equal(existsSync(join(output, 'functions.json')), false);
assert.equal(result.stdout, '## data-schemas: mutation\n\nNo changed source files.\n\n');
}
}
});

View file

@ -26,8 +26,15 @@ interface Step {
run?: string;
with?: Record<string, string | number | boolean>;
'working-directory'?: string;
if?: string;
env?: Record<string, string>;
}
interface Job {
name?: string;
needs?: string[];
if?: string;
'timeout-minutes'?: number | string;
strategy?: { matrix: { report?: { check: string; shard: number }[]; workspace: string } };
defaults?: { run: { 'working-directory': string } };
steps: Step[];
}
@ -84,13 +91,60 @@ test('workflow separates tooling, source, installs and artifacts without changin
],
);
assert.equal(runner['working-directory'], undefined);
assert.equal(
runner.run,
'node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK"',
assert.match(runner.run!, /args=\(\)/);
assert.match(runner.run!, /--shard "\$QUALITY_SHARD"/);
assert.match(runner.run!, /--base "\$QUALITY_BASE" --head "\$QUALITY_HEAD"/);
assert.match(
runner.run!,
/--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/,
);
assert.equal(
report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path,
'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.check }}/',
'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/',
);
});
test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => {
assert.equal(report['timeout-minutes'], "${{ matrix.report.check == 'mutation' && 240 || 45 }}");
assert.deepEqual(report.strategy?.matrix.report, [
{ check: 'risk', shard: 0 },
{ check: 'mutation', shard: 1 },
{ check: 'mutation', shard: 2 },
{ check: 'mutation', shard: 3 },
]);
assert.equal(runner.env?.QUALITY_CHECK, '${{ matrix.report.check }}');
assert.equal(
runner.env?.QUALITY_SHARD,
"${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}",
);
const aggregate = workflow.jobs.mutation;
assert.equal(aggregate.name, 'Mutation testing (${{ matrix.workspace }})');
assert.deepEqual(aggregate.needs, ['select', 'reports']);
assert.match(aggregate.if!, /^always\(\)/);
assert.equal(checkouts(aggregate)[0].with?.ref, '${{ github.sha }}');
assert.equal(
aggregate.steps.some(({ run }) => run?.includes('npm ci')),
false,
);
const download = aggregate.steps.find(({ uses }) => uses === 'actions/download-artifact@v7');
assert.equal(
download?.with?.pattern,
'change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}',
);
assert.equal(download?.with?.['merge-multiple'], undefined);
const verify = aggregate.steps.findIndex(({ name }) => name === 'Verify all mutation shards');
const requireSuccess = aggregate.steps.findIndex(
({ name }) => name === 'Require successful report jobs',
);
assert.ok(verify >= 0 && requireSuccess > verify);
assert.match(aggregate.steps[verify].run!, /--head "\$QUALITY_HEAD"/);
assert.match(aggregate.steps[verify].run!, /--count 3/);
assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success');
const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6');
assert.equal(upload?.if, 'always()');
assert.equal(
upload?.with?.name,
'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}',
);
});
@ -189,6 +243,7 @@ for (const olderHead of [true, false]) {
...process.env,
QUALITY_HEAD: head,
QUALITY_WORKSPACE: 'data-schemas',
QUALITY_SHARD: '',
GITHUB_OUTPUT: '',
GITHUB_STEP_SUMMARY: '',
};
@ -249,6 +304,51 @@ for (const olderHead of [true, false]) {
const progress = JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8'));
assert.equal(progress.complete, true);
assert.ok(progress.completed > 0);
const unsharded = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8'));
const identify = (report: { files: Record<string, { mutants: unknown[] }> }) =>
Object.entries(report.files)
.flatMap(([file, { mutants }]) =>
mutants.map((value) => {
const mutant = value as {
location: unknown;
mutatorName: string;
replacement: string;
};
return JSON.stringify([
file,
mutant.location,
mutant.mutatorName,
mutant.replacement,
]);
}),
)
.sort();
const shardReports = join(directory, 'mutation-reports');
const selectedMutants: string[] = [];
for (let index = 1; index <= 3; index++) {
run(runner.run!, source, {
QUALITY_BASE: base,
QUALITY_CHECK: 'mutation',
QUALITY_SHARD: `${index}/3`,
});
const shard = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8'));
assert.deepEqual(shard.shard, { index, total: 3 });
assert.equal(shard.workspace, 'data-schemas');
const mutation = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8'));
selectedMutants.push(...identify(mutation));
cpSync(output, join(shardReports, `shard-${index}`), { recursive: true });
}
assert.deepEqual(selectedMutants.sort(), identify(unsharded));
const aggregateOutput = join(directory, 'aggregate');
run(
`node ../tooling/scripts/quality/aggregate.mts --directory "${shardReports}" --head "${head}" --workspace data-schemas --count 3 --output "${aggregateOutput}"`,
source,
{},
);
const aggregate = JSON.parse(readFileSync(join(aggregateOutput, 'aggregate.json'), 'utf8'));
assert.equal(aggregate.complete, true);
assert.equal(aggregate.head, head);
assert.equal(aggregate.total, selectedMutants.length);
}
assert.equal(git(['rev-parse', 'HEAD'], source).trim(), head);
assert.equal(git(['diff', 'HEAD', '--', file], source), '');