mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-19 22:43:38 +00:00
🚜 feat: Support Native PostgreSQL Tool Favorites (#18032)
* 🚜 feat: Support Native PostgreSQL Tool Favorites
* fix: Guard PostgreSQL Reconciliation and Connection Leases
* fix: Scope Favorites Erasure Fences to Deletion Attempts
* fix: Preserve Account Erasure Ownership Until Cancellation Completes
* fix: Reject Unsupported PostgreSQL Command Flags Before Connecting
* fix: Keep Command Validation Compatible With the TypeScript Target
* fix: Keep Migration Previews Read-Only and Preserve Erasure Recovery Identity
* fix: Fence Erasure Recovery Across Tenant and Ownership Changes
* fix: Isolate Migration Dependencies and Keep Mongo Scans Outside Transactions
* fix: Inject PostgreSQL Backend Factories at Composition Roots
* ci: Shard Complete Mutation Work Without Relaxing the Aggregate Gate
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
dbe9f16a20
commit
42219ec8e9
60 changed files with 6433 additions and 112 deletions
121
.github/workflows/change-quality.yml
vendored
121
.github/workflows/change-quality.yml
vendored
|
|
@ -54,6 +54,7 @@ jobs:
|
|||
outputs:
|
||||
workspaces: ${{ steps.select.outputs.workspaces }}
|
||||
hasChanges: ${{ steps.select.outputs.hasChanges }}
|
||||
postgresChanged: ${{ steps.postgres.outputs.changed }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
|
|
@ -69,13 +70,56 @@ jobs:
|
|||
QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }}
|
||||
run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE"
|
||||
|
||||
- name: Select PostgreSQL schema and lifecycle changes
|
||||
id: postgres
|
||||
run: |
|
||||
if git diff --name-only "$QUALITY_BASE...$QUALITY_HEAD" | grep -Eq '^(packages/data-schemas/src/(postgres/|types/favorite|methods/(favorite|index))|packages/api/src/app/(persistence|postgres)|packages/data-provider/src/config|api/(models/index|server/(index|experimental))|config/(postgres|delete-user)|packages/data-schemas/jest.postgres|package-lock.json|.github/workflows/change-quality.yml)'; then
|
||||
echo 'changed=true' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
postgres:
|
||||
name: PostgreSQL 16 contracts and readiness
|
||||
needs: [select]
|
||||
if: needs.select.outputs.postgresChanged == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
ports: ['5432:5432']
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
|
||||
NODE_OPTIONS: --max-old-space-size=6144
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
ref: ${{ env.QUALITY_HEAD }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: '24.22.1'
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run build:data-provider && npm run build:data-schemas && npm run build:api
|
||||
- run: npm run test:postgres --workspace packages/data-schemas
|
||||
- run: npx --no-install jest src/app/persistence.spec.ts src/app/postgres.spec.ts src/favorites/handlers.spec.ts src/user/erasure.spec.ts --coverage=false --runInBand
|
||||
working-directory: packages/api
|
||||
- run: npx --no-install jest --config config/jest.config.js --runTestsByPath config/__tests__/delete-user.recovery.spec.js --coverage=false --runInBand
|
||||
|
||||
reports:
|
||||
name: ${{ matrix.check == 'risk' && 'Change risk' || 'Mutation testing' }} (${{ matrix.workspace }})
|
||||
name: ${{ matrix.report.check == 'risk' && format('Change risk ({0})', matrix.workspace) || format('Mutation shard {0}/3 ({1})', matrix.report.shard, matrix.workspace) }}
|
||||
needs: [select, verify]
|
||||
if: needs.select.outputs.hasChanges == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
# Whole edited database functions can produce thousands of mutation cases.
|
||||
timeout-minutes: ${{ matrix.check == 'mutation' && 240 || 45 }}
|
||||
timeout-minutes: ${{ matrix.report.check == 'mutation' && 240 || 45 }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: source
|
||||
|
|
@ -83,7 +127,11 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
workspace: ${{ fromJSON(needs.select.outputs.workspaces) }}
|
||||
check: [risk, mutation]
|
||||
report:
|
||||
- { check: risk, shard: 0 }
|
||||
- { check: mutation, shard: 1 }
|
||||
- { check: mutation, shard: 2 }
|
||||
- { check: mutation, shard: 3 }
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
|
|
@ -93,7 +141,18 @@ jobs:
|
|||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
ports: ['5432:5432']
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
|
||||
REDIS_URI: redis://127.0.0.1:6379
|
||||
NODE_OPTIONS: --max-old-space-size=6144
|
||||
steps:
|
||||
|
|
@ -138,15 +197,59 @@ jobs:
|
|||
- name: Generate advisory report
|
||||
env:
|
||||
QUALITY_WORKSPACE: ${{ matrix.workspace }}
|
||||
QUALITY_CHECK: ${{ matrix.check }}
|
||||
run: >-
|
||||
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD"
|
||||
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK"
|
||||
QUALITY_CHECK: ${{ matrix.report.check }}
|
||||
QUALITY_SHARD: ${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}
|
||||
run: |
|
||||
args=()
|
||||
if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi
|
||||
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \
|
||||
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}"
|
||||
- name: Upload reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: change-${{ matrix.check }}-${{ matrix.workspace }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.check }}/
|
||||
name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }}
|
||||
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
mutation:
|
||||
name: Mutation testing (${{ matrix.workspace }})
|
||||
needs: [select, reports]
|
||||
if: always() && needs.select.result == 'success' && needs.select.outputs.hasChanges == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
workspace: ${{ fromJSON(needs.select.outputs.workspaces) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: '24.22.1'
|
||||
- uses: actions/download-artifact@v7
|
||||
with:
|
||||
pattern: change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}
|
||||
path: mutation-reports
|
||||
- name: Verify all mutation shards
|
||||
env:
|
||||
QUALITY_WORKSPACE: ${{ matrix.workspace }}
|
||||
run: >-
|
||||
node scripts/quality/aggregate.mts --directory mutation-reports --head "$QUALITY_HEAD"
|
||||
--workspace "$QUALITY_WORKSPACE" --count 3
|
||||
--output "coverage/changes/$QUALITY_WORKSPACE/mutation"
|
||||
- name: Require successful report jobs
|
||||
env:
|
||||
REPORT_RESULT: ${{ needs.reports.result }}
|
||||
run: test "$REPORT_RESULT" = success
|
||||
- uses: actions/upload-artifact@v6
|
||||
if: always()
|
||||
with:
|
||||
name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}
|
||||
path: coverage/changes/${{ matrix.workspace }}/mutation/
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
|
|
|||
|
|
@ -1,10 +1,16 @@
|
|||
const mongoose = require('mongoose');
|
||||
const { createMethods } = require('@librechat/data-schemas');
|
||||
const {
|
||||
createMethods,
|
||||
createToolFavoriteMethods,
|
||||
createPostgresFavoriteBackend,
|
||||
} = require('@librechat/data-schemas');
|
||||
const {
|
||||
matchModelName,
|
||||
findMatchingPattern,
|
||||
isDeploymentSkillId,
|
||||
createMessageBudgetReader,
|
||||
createFavoriteRuntime,
|
||||
getPersistenceConfig,
|
||||
createChannelStorageConfigReader,
|
||||
} = require('@librechat/api');
|
||||
const getLogStores = require('~/cache/getLogStores');
|
||||
|
|
@ -12,7 +18,14 @@ const getLogStores = require('~/cache/getLogStores');
|
|||
const messageBudget = createMessageBudgetReader();
|
||||
const channelStorageConfig = createChannelStorageConfigReader();
|
||||
|
||||
const favorites = createFavoriteRuntime(
|
||||
createToolFavoriteMethods(mongoose),
|
||||
process.env,
|
||||
createPostgresFavoriteBackend,
|
||||
);
|
||||
|
||||
const methods = createMethods(mongoose, {
|
||||
toolFavorites: favorites.methods,
|
||||
matchModelName,
|
||||
findMatchingPattern,
|
||||
isExternalSkillId: isDeploymentSkillId,
|
||||
|
|
@ -30,6 +43,8 @@ const seedDatabase = async () => {
|
|||
|
||||
module.exports = {
|
||||
...methods,
|
||||
initializeFavorites: (config) => favorites.initialize(getPersistenceConfig(config)),
|
||||
closeFavorites: favorites.close,
|
||||
initializeMessageBudget: messageBudget.initialize,
|
||||
initializeChannelStorageConfig: channelStorageConfig.initialize,
|
||||
seedDatabase,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,18 @@
|
|||
jest.mock('@librechat/data-schemas', () => ({ createMethods: jest.fn(() => ({})) }));
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
createPostgresFavoriteBackend: jest.fn(),
|
||||
createMethods: jest.fn((_mongoose, deps) => ({ ...deps.toolFavorites })),
|
||||
createToolFavoriteMethods: jest.fn(() => ({
|
||||
getToolFavorites: jest.fn().mockResolvedValue([]),
|
||||
addToolFavorite: jest.fn().mockResolvedValue({ ok: true, added: true }),
|
||||
removeToolFavorite: jest.fn().mockResolvedValue({ ok: true, removed: true }),
|
||||
deleteUserToolFavorites: jest.fn().mockResolvedValue(2),
|
||||
})),
|
||||
}));
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('../../packages/api/src/app/persistence'),
|
||||
createFavoriteRuntime: jest.fn(
|
||||
jest.requireActual('../../packages/api/src/app/persistence').createFavoriteRuntime,
|
||||
),
|
||||
matchModelName: jest.fn(),
|
||||
findMatchingPattern: jest.fn(),
|
||||
isDeploymentSkillId: jest.fn(),
|
||||
|
|
@ -11,13 +24,25 @@ jest.mock('@librechat/api', () => ({
|
|||
}));
|
||||
jest.mock('~/cache/getLogStores', () => jest.fn());
|
||||
|
||||
const { createMethods } = require('@librechat/data-schemas');
|
||||
const { createMessageBudgetReader, createChannelStorageConfigReader } = require('@librechat/api');
|
||||
const { initializeMessageBudget, initializeChannelStorageConfig } = require('./index');
|
||||
const {
|
||||
createToolFavoriteMethods,
|
||||
createMethods,
|
||||
createPostgresFavoriteBackend,
|
||||
} = require('@librechat/data-schemas');
|
||||
const {
|
||||
createMessageBudgetReader,
|
||||
createFavoriteRuntime,
|
||||
createChannelStorageConfigReader,
|
||||
} = require('@librechat/api');
|
||||
const models = require('./index');
|
||||
const { initializeMessageBudget, initializeChannelStorageConfig } = models;
|
||||
const reader = createMessageBudgetReader.mock.results[0].value;
|
||||
const channelStorageConfigReader = createChannelStorageConfigReader.mock.results[0].value;
|
||||
const dependencies = createMethods.mock.calls[0][1];
|
||||
const runtimeDependencies = createFavoriteRuntime.mock.calls[0];
|
||||
const { getMCPAppMessageBudget, getChannelStorageConfig: channelStorageConfigDependency } =
|
||||
createMethods.mock.calls[0][1];
|
||||
dependencies;
|
||||
const mongoFavorites = createToolFavoriteMethods.mock.results[0].value;
|
||||
|
||||
describe('message App budget dependency wiring', () => {
|
||||
it('passes the same reader to storage and initializer to the composition root', () => {
|
||||
|
|
@ -27,6 +52,26 @@ describe('message App budget dependency wiring', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('favorites composition root', () => {
|
||||
it('passes stable runtime methods to storage and keeps captured deletion references usable', async () => {
|
||||
expect(runtimeDependencies).toEqual([
|
||||
mongoFavorites,
|
||||
process.env,
|
||||
createPostgresFavoriteBackend,
|
||||
]);
|
||||
const captured = models.deleteUserToolFavorites;
|
||||
expect(dependencies.toolFavorites.deleteUserToolFavorites).toBe(captured);
|
||||
await expect(captured('user')).rejects.toThrow('not initialized');
|
||||
await models.initializeFavorites(null);
|
||||
expect(createPostgresFavoriteBackend).not.toHaveBeenCalled();
|
||||
expect(await captured('user')).toBe(2);
|
||||
expect(mongoFavorites.deleteUserToolFavorites).toHaveBeenCalledWith('user');
|
||||
expect(models.deleteUserToolFavorites).toBe(captured);
|
||||
await models.closeFavorites();
|
||||
await expect(captured('user')).rejects.toThrow('not initialized');
|
||||
});
|
||||
});
|
||||
|
||||
describe('channel storage configuration dependency wiring', () => {
|
||||
it('injects the deployment reader without reading config during module initialization', () => {
|
||||
expect(channelStorageConfigDependency).toBe(channelStorageConfigReader.getConfig);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ const {
|
|||
stopUserGenerations,
|
||||
deleteAllSharedLinksWithCleanup,
|
||||
eraseUserData,
|
||||
createUserDeletionAttempt,
|
||||
deleteUserMCPServers,
|
||||
revokeUserCodeEnvironmentWorkers,
|
||||
finalizeMCPAuthorizationMutation,
|
||||
|
|
@ -370,6 +371,7 @@ const deleteUserController = async (req, res) => {
|
|||
let triggerDeletionFence;
|
||||
let scheduleSuspensionToken;
|
||||
let userDeleted = false;
|
||||
const deletionAttempt = createUserDeletionAttempt();
|
||||
|
||||
try {
|
||||
const existingUser = await db.getUserById(
|
||||
|
|
@ -391,7 +393,11 @@ const deleteUserController = async (req, res) => {
|
|||
// Block new trigger admissions across replicas while preserving the user
|
||||
// principal so a transient cleanup failure remains retryable.
|
||||
triggerDeletionFence = new Date();
|
||||
const fenceState = await db.beginAgentTriggerUserDeletion(user.id, triggerDeletionFence);
|
||||
const fenceState = await db.beginAgentTriggerUserDeletion(
|
||||
user.id,
|
||||
triggerDeletionFence,
|
||||
deletionAttempt.id,
|
||||
);
|
||||
if (fenceState === 'in_progress') {
|
||||
triggerDeletionFence = undefined;
|
||||
throw new Error('Agent trigger account deletion is already in progress');
|
||||
|
|
@ -425,6 +431,7 @@ const deleteUserController = async (req, res) => {
|
|||
await eraseUserData(
|
||||
{
|
||||
userId: user.id,
|
||||
deletionAttemptId: deletionAttempt.id,
|
||||
tenantId,
|
||||
checkpointer: appConfig?.endpoints?.agents?.checkpointer,
|
||||
},
|
||||
|
|
@ -439,7 +446,12 @@ const deleteUserController = async (req, res) => {
|
|||
onSessionsDeleted: () => clearSessionHintCookie(res),
|
||||
},
|
||||
);
|
||||
const deleteResult = await db.deleteUserById(user.id);
|
||||
const deleteResult = await deletionAttempt.commit(() =>
|
||||
db.deleteUserById(user.id, {
|
||||
startedAt: triggerDeletionFence,
|
||||
attemptId: deletionAttempt.id,
|
||||
}),
|
||||
);
|
||||
if (deleteResult.deletedCount !== 1) {
|
||||
throw new Error('User disappeared before account deletion could commit');
|
||||
}
|
||||
|
|
@ -506,7 +518,14 @@ const deleteUserController = async (req, res) => {
|
|||
);
|
||||
}
|
||||
try {
|
||||
await db.cancelAgentTriggerUserDeletion(user.id, triggerDeletionFence);
|
||||
await deletionAttempt.cancel(
|
||||
{
|
||||
userId: user.id,
|
||||
startedAt: triggerDeletionFence,
|
||||
deletionAttemptId: deletionAttempt.id,
|
||||
},
|
||||
db,
|
||||
);
|
||||
} catch (fenceError) {
|
||||
logger.error('[deleteUserController] Failed to release trigger deletion fence', fenceError);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ jest.mock('~/models', () => {
|
|||
deleteAclEntries: jest.fn().mockResolvedValue(undefined),
|
||||
deleteUserChatProjects: jest.fn().mockResolvedValue(0),
|
||||
deleteUserToolFavorites: jest.fn().mockResolvedValue(0),
|
||||
cancelUserToolFavoriteDeletion: jest.fn().mockResolvedValue(undefined),
|
||||
deleteRefreshTokenBridges: jest.fn().mockResolvedValue(undefined),
|
||||
deleteAllAgentQueuedTurnsForUser: jest.fn().mockResolvedValue(0),
|
||||
findUserErasableMCPServers: jest.fn().mockResolvedValue([]),
|
||||
|
|
@ -64,6 +65,7 @@ jest.mock('~/models', () => {
|
|||
deleteUserById: jest.fn().mockResolvedValue({ deletedCount: 1 }),
|
||||
deleteAllUserPluginAuths: jest.fn().mockResolvedValue(undefined),
|
||||
beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),
|
||||
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true),
|
||||
cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),
|
||||
deleteUserPrompts: jest.fn().mockResolvedValue(undefined),
|
||||
deleteUserSkills: jest.fn().mockResolvedValue(undefined),
|
||||
|
|
@ -592,9 +594,20 @@ describe('deleteUserController', () => {
|
|||
|
||||
expect(mockRes.status).toHaveBeenCalledWith(200);
|
||||
expect(mockRes.send).toHaveBeenCalledWith({ message: 'User deleted' });
|
||||
expect(deleteUserById).toHaveBeenCalledWith(userIdStr, {
|
||||
startedAt: beginAgentTriggerUserDeletion.mock.calls[0][1],
|
||||
attemptId: beginAgentTriggerUserDeletion.mock.calls[0][2],
|
||||
});
|
||||
expect(deleteUserById.mock.calls[0][1].attemptId).not.toBe(
|
||||
mockQuiesceUserSchedules.mock.calls[0][1],
|
||||
);
|
||||
expect(mockRes.clearCookie).toHaveBeenCalledWith('session_hint');
|
||||
expect(deletePasskeysByUser).toHaveBeenCalledWith(userIdStr);
|
||||
expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(userId.toString(), expect.any(Date));
|
||||
expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
userId.toString(),
|
||||
expect.any(Date),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(mockPrepareAgentTriggerUserPurge).toHaveBeenCalledWith(
|
||||
userId.toString(),
|
||||
expect.any(Date),
|
||||
|
|
@ -929,7 +942,11 @@ describe('deleteUserController', () => {
|
|||
expect(deleteMessages).not.toHaveBeenCalled();
|
||||
const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];
|
||||
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
userIdString,
|
||||
deletionFence,
|
||||
expect.any(String),
|
||||
);
|
||||
expect(deleteUserById).not.toHaveBeenCalled();
|
||||
// Account survives -> its suspended schedules are restored under the quiesce token.
|
||||
expect(mockRestoreUserSchedules).toHaveBeenCalledWith(
|
||||
|
|
@ -944,6 +961,42 @@ describe('deleteUserController', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each([false, 'error'])(
|
||||
'does not release the favorite marker without a proven Mongo owner: %s',
|
||||
async (result) => {
|
||||
const db = require('~/models');
|
||||
db.deleteUserAgents.mockRejectedValueOnce(new Error('later cascade failure'));
|
||||
if (result === 'error')
|
||||
db.isAgentTriggerUserDeletionOwner.mockRejectedValueOnce(new Error('Mongo unavailable'));
|
||||
else db.isAgentTriggerUserDeletionOwner.mockResolvedValueOnce(false);
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
await deleteUserController({ user: { id: userId, tenantId: 'tenant-1' } }, mockRes);
|
||||
expect(mockRes.status).toHaveBeenCalledWith(500);
|
||||
expect(db.deleteUserToolFavorites).toHaveBeenCalledWith(
|
||||
userId,
|
||||
beginAgentTriggerUserDeletion.mock.calls[0][2],
|
||||
);
|
||||
expect(db.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('releases the cascade UUID before releasing Mongo ownership', async () => {
|
||||
const db = require('~/models');
|
||||
db.deleteUserAgents.mockRejectedValueOnce(new Error('later cascade failure'));
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
await deleteUserController({ user: { id: userId, tenantId: 'tenant-1' } }, mockRes);
|
||||
const token = beginAgentTriggerUserDeletion.mock.calls[0][2];
|
||||
expect(mockRes.status).toHaveBeenCalledWith(500);
|
||||
expect(db.deleteUserToolFavorites).toHaveBeenCalledWith(userId, token);
|
||||
expect(db.cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(userId, token);
|
||||
expect(db.isAgentTriggerUserDeletionOwner.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
db.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(db.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
db.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('fails closed before data cleanup when detached subagents do not drain', async () => {
|
||||
const userId = new mongoose.Types.ObjectId();
|
||||
mockCancelAndDrainSubagentThreads.mockRejectedValueOnce(new Error('child drain timed out'));
|
||||
|
|
@ -963,6 +1016,7 @@ describe('deleteUserController', () => {
|
|||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
userId.toString(),
|
||||
expect.any(Date),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(deleteUserById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -987,7 +1041,11 @@ describe('deleteUserController', () => {
|
|||
expect(mockGetActiveJobIdsForUser).not.toHaveBeenCalled();
|
||||
const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];
|
||||
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
userIdString,
|
||||
deletionFence,
|
||||
expect.any(String),
|
||||
);
|
||||
expect(deleteUserById).not.toHaveBeenCalled();
|
||||
// Account survives -> its suspended schedules are restored under the quiesce token.
|
||||
expect(mockRestoreUserSchedules).toHaveBeenCalledWith(
|
||||
|
|
@ -1074,6 +1132,7 @@ describe('deleteUserController', () => {
|
|||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
userId.toString(),
|
||||
expect.any(Date),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(
|
||||
userId.toString(),
|
||||
|
|
@ -1090,7 +1149,22 @@ describe('deleteUserController', () => {
|
|||
|
||||
expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
|
||||
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
String(userId),
|
||||
expect.any(Date),
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
|
||||
it('retains both erasure fences after an unknown final delete outcome', async () => {
|
||||
const db = require('~/models');
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
db.deleteUserById.mockRejectedValueOnce(new Error('delete acknowledgement lost'));
|
||||
await deleteUserController({ user: { id: userId } }, mockRes);
|
||||
expect(mockRes.status).toHaveBeenCalledWith(500);
|
||||
expect(db.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled();
|
||||
expect(db.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
|
||||
expect(db.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not purge trigger payloads when the user deletion did not commit', async () => {
|
||||
|
|
@ -1101,7 +1175,7 @@ describe('deleteUserController', () => {
|
|||
|
||||
expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
|
||||
expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));
|
||||
expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));
|
||||
expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
expect(mockRes.status).toHaveBeenCalledWith(500);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -70,6 +70,10 @@ jest.mock('@librechat/api', () => ({
|
|||
getNewS3URL: jest.fn(),
|
||||
deleteAllSharedLinksWithCleanup: (...args) => mockDeleteAllSharedLinksWithCleanup(...args),
|
||||
eraseUserData: jest.requireActual('../../../../packages/api/src/user/erasure.ts').eraseUserData,
|
||||
createUserDeletionAttempt: jest.requireActual('../../../../packages/api/src/user/erasure.ts')
|
||||
.createUserDeletionAttempt,
|
||||
cancelUserDataDeletion: jest.requireActual('../../../../packages/api/src/user/erasure.ts')
|
||||
.cancelUserDataDeletion,
|
||||
revokeUserCodeEnvironmentWorkers: (...args) => mockRevokeUserCodeEnvironmentWorkers(...args),
|
||||
GenerationJobManager: {
|
||||
getAccountCleanupJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args),
|
||||
|
|
@ -98,6 +102,7 @@ jest.mock('~/models', () => ({
|
|||
deleteUserById: (...args) => mockDeleteUserById(...args),
|
||||
deleteAllUserPluginAuths: (...args) => mockDeleteAllUserPluginAuths(...args),
|
||||
beginAgentTriggerUserDeletion: (...args) => mockBeginAgentTriggerUserDeletion(...args),
|
||||
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true),
|
||||
cancelAgentTriggerUserDeletion: (...args) => mockCancelAgentTriggerUserDeletion(...args),
|
||||
deleteMessages: (...args) => mockDeleteMessages(...args),
|
||||
deletePresets: (...args) => mockDeletePresets(...args),
|
||||
|
|
@ -429,7 +434,10 @@ describe('deleteUserController - 2FA enforcement', () => {
|
|||
expect(res.status).toHaveBeenCalledWith(200);
|
||||
expect(mockBeginAgentTriggerUserDeletion).toHaveBeenCalledTimes(1);
|
||||
expect(mockDeleteMessages).toHaveBeenCalled();
|
||||
expect(mockDeleteUserById).toHaveBeenCalledWith(USER_ID);
|
||||
expect(mockDeleteUserById).toHaveBeenCalledWith(USER_ID, {
|
||||
startedAt: mockBeginAgentTriggerUserDeletion.mock.calls[0][1],
|
||||
attemptId: mockBeginAgentTriggerUserDeletion.mock.calls[0][2],
|
||||
});
|
||||
expect(mockDeleteUserCodeEnvironments).not.toHaveBeenCalled();
|
||||
expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();
|
||||
expect(mockCancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ const {
|
|||
getRoleByName,
|
||||
updateAccessPermissions,
|
||||
seedDatabase,
|
||||
initializeFavorites,
|
||||
closeFavorites,
|
||||
recordAuditEntry,
|
||||
sweepOrphanedPreviews,
|
||||
} = require('~/models');
|
||||
|
|
@ -104,7 +106,7 @@ const { checkMigrations } = require('./services/start/migration');
|
|||
const initializeMCPs = require('./services/initializeMCPs');
|
||||
const configureSocialLogins = require('./socialLogins');
|
||||
const createSpaFallback = require('./utils/fallback');
|
||||
const { getAppConfig } = require('./services/Config');
|
||||
const { getAppConfig, loadCustomConfig } = require('./services/Config');
|
||||
const staticCache = require('./utils/staticCache');
|
||||
const optionalJwtAuth = require('./middleware/optionalJwtAuth');
|
||||
const noIndex = require('./middleware/noIndex');
|
||||
|
|
@ -524,6 +526,8 @@ if (cluster.isMaster) {
|
|||
* deletion, the expired-file sweep, tool-approval hooks and the other worker-wide settings
|
||||
* below are fixed for the process lifetime, so DB `__base__` overrides must not decide them.
|
||||
*/
|
||||
await initializeFavorites(await loadCustomConfig(false, { storageSelection: true }));
|
||||
registerShutdownTask('favorites persistence', closeFavorites, { priority: -100 });
|
||||
const appConfig = await getAppConfig({ baseOnly: true });
|
||||
initializeFileStorage(appConfig);
|
||||
initializeGitHubSkillSync(appConfig);
|
||||
|
|
@ -791,7 +795,8 @@ if (cluster.isMaster) {
|
|||
setupGracefulShutdown(server);
|
||||
};
|
||||
|
||||
startServer().catch((err) => {
|
||||
startServer().catch(async (err) => {
|
||||
await closeFavorites().catch(() => undefined);
|
||||
logger.error(`Failed to start worker ${process.pid}:`, err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ const {
|
|||
sweepOrphanedPreviews,
|
||||
getRoleByName,
|
||||
seedDatabase,
|
||||
initializeFavorites,
|
||||
closeFavorites,
|
||||
listAllConfigs,
|
||||
recordAuditEntry,
|
||||
} = require('~/models');
|
||||
|
|
@ -111,7 +113,7 @@ const initializeMCPs = require('./services/initializeMCPs');
|
|||
const { configureSubagentTaskRouting } = require('./services/Endpoints/agents/subagentThreadStore');
|
||||
const configureSocialLogins = require('./socialLogins');
|
||||
const createSpaFallback = require('./utils/fallback');
|
||||
const { getAppConfig, rederiveEndpoints } = require('./services/Config');
|
||||
const { getAppConfig, rederiveEndpoints, loadCustomConfig } = require('./services/Config');
|
||||
const staticCache = require('./utils/staticCache');
|
||||
const noIndex = require('./middleware/noIndex');
|
||||
const agentEventMethods = require('~/models');
|
||||
|
|
@ -257,6 +259,8 @@ const startServer = async () => {
|
|||
runAsSystem(sweepOrphanedPreviews).catch((err) => {
|
||||
logger.error('[sweepOrphanedPreviews] Background sweep failed:', err);
|
||||
});
|
||||
await initializeFavorites(await loadCustomConfig(false, { storageSelection: true }));
|
||||
registerShutdownTask('favorites persistence', closeFavorites, { priority: -100 });
|
||||
const appConfig = await getAppConfig({ baseOnly: true });
|
||||
await configureSubagentTaskRouting(appConfig?.endpoints?.agents?.subagentActivity);
|
||||
registerBackgroundTaskShutdown({
|
||||
|
|
@ -618,7 +622,8 @@ const startServer = async () => {
|
|||
* liveness probe eventually times out. Mirrors the pattern in
|
||||
* `experimental.js`.
|
||||
*/
|
||||
startServer().catch((err) => {
|
||||
startServer().catch(async (err) => {
|
||||
await closeFavorites().catch(() => undefined);
|
||||
logger.error('Failed to start server:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
137
api/server/persistence.spec.js
Normal file
137
api/server/persistence.spec.js
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
const { readFileSync } = require('node:fs');
|
||||
const { resolve, dirname } = require('node:path');
|
||||
const { runInNewContext } = require('node:vm');
|
||||
|
||||
describe.each(['index', 'experimental'])('%s entrypoint lifecycle wiring', (entrypoint) => {
|
||||
it.each([false, true])(
|
||||
'retains the original startup error and cleans up (persistence failure: %s)',
|
||||
async (failFavorites) => {
|
||||
const file = resolve(__dirname, `${entrypoint}.js`);
|
||||
const events = [];
|
||||
const errors = [];
|
||||
const tasks = new Map();
|
||||
const bootError = new Error('boot failure');
|
||||
const noop = new Proxy(
|
||||
function () {
|
||||
return noop;
|
||||
},
|
||||
{
|
||||
get(_target, key) {
|
||||
return key === 'then' ? undefined : noop;
|
||||
},
|
||||
},
|
||||
);
|
||||
const closeFavorites = async () => {
|
||||
events.push('close');
|
||||
if (failFavorites) {
|
||||
throw new Error('close failure');
|
||||
}
|
||||
};
|
||||
const api = new Proxy(
|
||||
{
|
||||
cacheConfig: {},
|
||||
isEnabled: () => false,
|
||||
waitForKeyvRedisClient: async () => {},
|
||||
createMetrics: () => ({}),
|
||||
registerShutdownTask: (name, fn) => tasks.set(name, fn),
|
||||
},
|
||||
{
|
||||
get(target, key) {
|
||||
return Reflect.has(target, key) ? Reflect.get(target, key) : noop;
|
||||
},
|
||||
},
|
||||
);
|
||||
const models = new Proxy(
|
||||
{
|
||||
seedDatabase: async () => {},
|
||||
sweepOrphanedPreviews: async () => {},
|
||||
initializeFavorites: async (config) => {
|
||||
events.push('initialize');
|
||||
expect(config.persistence.favorites).toBe('postgres');
|
||||
if (failFavorites) {
|
||||
throw bootError;
|
||||
}
|
||||
},
|
||||
closeFavorites,
|
||||
},
|
||||
{
|
||||
get(target, key) {
|
||||
return Reflect.has(target, key) ? Reflect.get(target, key) : noop;
|
||||
},
|
||||
},
|
||||
);
|
||||
runInNewContext(
|
||||
readFileSync(file, 'utf8'),
|
||||
{
|
||||
require: (id) => {
|
||||
if (id === 'path') {
|
||||
return { resolve, dirname };
|
||||
}
|
||||
if (id === 'cluster') {
|
||||
return { isMaster: false };
|
||||
}
|
||||
if (id === 'express') {
|
||||
return () => ({ locals: {}, disable() {}, set() {}, use() {} });
|
||||
}
|
||||
if (id === '@librechat/api') {
|
||||
return api;
|
||||
}
|
||||
if (id === '@librechat/data-schemas') {
|
||||
return {
|
||||
logger: { info() {}, warn() {}, error: (...args) => errors.push(args) },
|
||||
runAsSystem: async (fn) => fn(),
|
||||
};
|
||||
}
|
||||
if (id === '~/models') {
|
||||
return models;
|
||||
}
|
||||
if (id === '~/db') {
|
||||
return { connectDb: async () => {}, indexSync: async () => {} };
|
||||
}
|
||||
if (id === './services/Config') {
|
||||
return {
|
||||
loadCustomConfig: async (print, options) => {
|
||||
expect(print).toBe(false);
|
||||
expect(options).toEqual({ storageSelection: true });
|
||||
return { persistence: { favorites: 'postgres' } };
|
||||
},
|
||||
getAppConfig: async () => {
|
||||
events.push('config');
|
||||
throw bootError;
|
||||
},
|
||||
};
|
||||
}
|
||||
return noop;
|
||||
},
|
||||
__dirname: dirname(file),
|
||||
module: { exports: {} },
|
||||
console,
|
||||
process: {
|
||||
env: {},
|
||||
pid: 1,
|
||||
on() {},
|
||||
exit: (code) => events.push(`exit:${code}`),
|
||||
},
|
||||
setTimeout,
|
||||
clearTimeout,
|
||||
setInterval,
|
||||
clearInterval,
|
||||
Buffer,
|
||||
},
|
||||
{ filename: file },
|
||||
);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(events[0]).toBe('initialize');
|
||||
expect(events).toContain('close');
|
||||
expect(events).toContain('exit:1');
|
||||
const last = errors[errors.length - 1];
|
||||
expect(last[last.length - 1]).toBe(bootError);
|
||||
if (failFavorites) {
|
||||
expect(events).not.toContain('config');
|
||||
} else {
|
||||
expect(tasks.get('favorites persistence')).toBe(closeFavorites);
|
||||
expect(events).toContain('config');
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -60,6 +60,7 @@ const handlers = createAdminUsersHandlers({
|
|||
findUsers: db.findUsers,
|
||||
countUsers: db.countUsers,
|
||||
beginAgentTriggerUserDeletion: db.beginAgentTriggerUserDeletion,
|
||||
isAgentTriggerUserDeletionOwner: db.isAgentTriggerUserDeletionOwner,
|
||||
cancelAgentTriggerUserDeletion: db.cancelAgentTriggerUserDeletion,
|
||||
drainAgentTriggerDeliveriesForUser,
|
||||
prepareAgentTriggerUserPurge,
|
||||
|
|
|
|||
259
config/__tests__/delete-user.recovery.spec.js
Normal file
259
config/__tests__/delete-user.recovery.spec.js
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const mongoose = require('mongoose');
|
||||
const { MongoMemoryServer } = require('mongodb-memory-server');
|
||||
const schemas = require('@librechat/data-schemas');
|
||||
const api = require('@librechat/api');
|
||||
|
||||
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
realPostgres('delete-user CLI durable erasure recovery', () => {
|
||||
const schema = `cli_erasure_${randomUUID().replace(/-/g, '')}`;
|
||||
const config = {
|
||||
persistence: { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } },
|
||||
};
|
||||
const options = {
|
||||
connectionString: process.env.PG_TEST_URL ?? '',
|
||||
max: 2,
|
||||
maxWaiting: 10,
|
||||
acquireTimeoutMs: 1000,
|
||||
queryTimeoutMs: 5000,
|
||||
idleTimeoutMs: 1000,
|
||||
};
|
||||
let mongo;
|
||||
let admin;
|
||||
let db;
|
||||
const source = fs
|
||||
.readFileSync(path.resolve(__dirname, '../delete-user.js'), 'utf8')
|
||||
.replace(/^#!.*\n/, '');
|
||||
const execute = vm.runInThisContext(
|
||||
`(function(require, __dirname, process, console, Date) { ${source}\n})`,
|
||||
{ filename: 'delete-user-recovery-cli.js' },
|
||||
);
|
||||
beforeAll(async () => {
|
||||
mongo = await MongoMemoryServer.create();
|
||||
schemas.createModels(mongoose);
|
||||
await mongoose.connect(mongo.getUri());
|
||||
admin = schemas.createPostgresDatabase(options);
|
||||
await admin.query(`CREATE SCHEMA ${schema}`);
|
||||
const url = new URL(options.connectionString);
|
||||
url.searchParams.set('options', `-c search_path=${schema}`);
|
||||
options.connectionString = url.toString();
|
||||
db = schemas.createPostgresDatabase(options);
|
||||
await schemas.migratePostgres(db);
|
||||
await schemas.importFavorites(db, async function* () {}, {
|
||||
offline: true,
|
||||
verifiedEmpty: true,
|
||||
batchSize: 10,
|
||||
});
|
||||
});
|
||||
afterEach(async () => {
|
||||
await db.query('DROP TRIGGER IF EXISTS reject_cancel ON tool_favorite_erasure');
|
||||
await db.query('DROP FUNCTION IF EXISTS reject_cancel()');
|
||||
});
|
||||
afterAll(async () => {
|
||||
await db.close();
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
|
||||
await admin.close();
|
||||
await mongoose.disconnect();
|
||||
await mongo.stop();
|
||||
});
|
||||
async function runCli(
|
||||
email,
|
||||
now,
|
||||
{ failCascade = true, failAdopt = false, failCancel = false } = {},
|
||||
) {
|
||||
const runtime = api.createFavoriteRuntime(
|
||||
schemas.createToolFavoriteMethods(mongoose),
|
||||
{
|
||||
PG_TEST_URL: options.connectionString,
|
||||
},
|
||||
schemas.createPostgresFavoriteBackend,
|
||||
);
|
||||
const methods = schemas.createMethods(mongoose, { toolFavorites: runtime.methods });
|
||||
const suspend = jest.spyOn(methods, 'suspendUserSchedulesForDeletion');
|
||||
if (failAdopt)
|
||||
jest
|
||||
.spyOn(methods, 'getAgentTriggerUserDeletionAttemptId')
|
||||
.mockRejectedValueOnce(new Error('identity read unavailable'));
|
||||
const disconnect = jest.spyOn(mongoose, 'disconnect').mockResolvedValue(undefined);
|
||||
class Clock extends Date {
|
||||
constructor(value) {
|
||||
super(value === undefined ? now : value);
|
||||
}
|
||||
}
|
||||
const quiet = Object.assign(
|
||||
Object.create(console),
|
||||
Object.fromEntries(['purple', 'green', 'yellow', 'error'].map((name) => [name, jest.fn()])),
|
||||
);
|
||||
const code = await new Promise((resolve) => {
|
||||
const modules = {
|
||||
mongoose,
|
||||
'module-alias': () => undefined,
|
||||
'@librechat/data-schemas': schemas,
|
||||
'@librechat/api': {
|
||||
...api,
|
||||
createStreamServices: () => ({ isRedis: false }),
|
||||
waitForKeyvRedisClient: async () => undefined,
|
||||
initializeFileStorage: () => undefined,
|
||||
openCheckpointDeletion: async () => ({
|
||||
remember: async () => undefined,
|
||||
cleanup: async () => undefined,
|
||||
acknowledge: async () => undefined,
|
||||
}),
|
||||
deleteAllSharedLinksWithCleanup: async () => {
|
||||
if (failCascade) throw new Error('shared links unavailable');
|
||||
},
|
||||
revokeUserCodeEnvironmentWorkers: async () => 0,
|
||||
},
|
||||
'~/models': {
|
||||
...methods,
|
||||
initializeFavorites: async (value) => {
|
||||
await runtime.initialize(value.persistence);
|
||||
if (failCancel) await installFault();
|
||||
},
|
||||
closeFavorites: async () => {
|
||||
if (failCancel) await removeFault();
|
||||
await runtime.close();
|
||||
},
|
||||
},
|
||||
'~/config': {},
|
||||
'~/cache/getLogStores': () => undefined,
|
||||
'~/server/services/Config': { getAppConfig: async () => ({}) },
|
||||
'~/server/services/Config/loadCustomConfig': async () => config,
|
||||
'~/server/services/Files/process': {
|
||||
processDeleteRequest: async () => ({ deletedFileIds: [], failedFileIds: [] }),
|
||||
},
|
||||
'~/server/services/Config/getCachedTools': { invalidateCachedTools: async () => undefined },
|
||||
'./helpers': { askQuestion: async () => 'y', silentExit: resolve },
|
||||
'./connect': async () => undefined,
|
||||
};
|
||||
execute(
|
||||
(name) => (Object.hasOwn(modules, name) ? modules[name] : require(name)),
|
||||
path.resolve(__dirname, '..'),
|
||||
{ argv: ['node', 'delete-user.js', email] },
|
||||
quiet,
|
||||
Clock,
|
||||
);
|
||||
});
|
||||
disconnect.mockRestore();
|
||||
return { code, suspend };
|
||||
}
|
||||
async function owned(id) {
|
||||
return mongoose.models.User.findById(id)
|
||||
.select('+agentTriggerDeletionStartedAt +agentTriggerDeletionAttemptId')
|
||||
.lean();
|
||||
}
|
||||
async function installFault() {
|
||||
await db.query(
|
||||
`CREATE FUNCTION reject_cancel() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'cancel unavailable'; END $$`,
|
||||
);
|
||||
await db.query(
|
||||
'CREATE TRIGGER reject_cancel BEFORE DELETE ON tool_favorite_erasure FOR EACH ROW EXECUTE FUNCTION reject_cancel()',
|
||||
);
|
||||
}
|
||||
async function removeFault() {
|
||||
await db.query('DROP TRIGGER reject_cancel ON tool_favorite_erasure');
|
||||
await db.query('DROP FUNCTION reject_cancel()');
|
||||
}
|
||||
it.each([
|
||||
{ outcome: 'cancel', origin: 'system' },
|
||||
{ outcome: 'commit', origin: 'system' },
|
||||
{ outcome: 'cancel', origin: 'tenant' },
|
||||
{ outcome: 'commit', origin: 'tenant' },
|
||||
])(
|
||||
'adopts the canonical UUID from $origin through real offline CLI recovery and then $outcome',
|
||||
async ({ outcome, origin }) => {
|
||||
const id = new mongoose.Types.ObjectId();
|
||||
const email = `${id}@example.com`;
|
||||
await mongoose.models.User.create({
|
||||
_id: id,
|
||||
email,
|
||||
provider: 'local',
|
||||
...(origin === 'tenant' ? { tenantId: 'tenant-a' } : {}),
|
||||
});
|
||||
const time = Date.now();
|
||||
const stale = 15 * 60_000 + 1;
|
||||
const priorToken = randomUUID();
|
||||
if (origin === 'tenant') {
|
||||
const priorMethods = schemas.createMethods(mongoose);
|
||||
await schemas.runAsSystem(() =>
|
||||
priorMethods.beginAgentTriggerUserDeletion(
|
||||
id.toString(),
|
||||
new Date(time - stale),
|
||||
priorToken,
|
||||
),
|
||||
);
|
||||
await schemas.tenantStorage.run({ tenantId: 'tenant-a' }, () =>
|
||||
schemas
|
||||
.createPostgresFavoriteMethods(db)
|
||||
.deleteUserToolFavorites(id.toString(), priorToken),
|
||||
);
|
||||
}
|
||||
const first = await runCli(email, time, { failCancel: true });
|
||||
expect(first.code).toBe(1);
|
||||
const original = await owned(id);
|
||||
const token = original.agentTriggerDeletionAttemptId;
|
||||
expect(token).toMatch(/^[0-9a-f-]{36}$/);
|
||||
if (origin === 'tenant') expect(token).toBe(priorToken);
|
||||
// A takeover that loses its identity read must retain the adopted identity and owner.
|
||||
if (outcome === 'cancel') {
|
||||
expect((await runCli(email, time + stale, { failAdopt: true })).code).toBe(1);
|
||||
expect((await owned(id)).agentTriggerDeletionAttemptId).toBe(token);
|
||||
expect((await runCli(email, time + 2 * stale, { failCancel: true })).code).toBe(1);
|
||||
expect((await owned(id)).agentTriggerDeletionAttemptId).toBe(token);
|
||||
}
|
||||
const result = await runCli(email, time + 3 * stale, { failCascade: outcome === 'cancel' });
|
||||
expect(result.code).toBe(outcome === 'cancel' ? 1 : 0);
|
||||
expect(result.suspend.mock.calls[0][1]).not.toBe(first.suspend.mock.calls[0][1]);
|
||||
expect(result.suspend.mock.calls[0][1]).not.toBe(token);
|
||||
const methods = schemas.createMethods(mongoose);
|
||||
expect(
|
||||
await methods.cancelAgentTriggerUserDeletion(
|
||||
id.toString(),
|
||||
original.agentTriggerDeletionStartedAt,
|
||||
token,
|
||||
),
|
||||
).toBe(false);
|
||||
const restarted = api.createFavoriteRuntime(
|
||||
methods,
|
||||
{
|
||||
PG_TEST_URL: options.connectionString,
|
||||
},
|
||||
schemas.createPostgresFavoriteBackend,
|
||||
);
|
||||
try {
|
||||
await restarted.initialize(config.persistence);
|
||||
const addFavorite = () =>
|
||||
restarted.methods.addToolFavorite({
|
||||
userId: id.toString(),
|
||||
itemType: 'tool',
|
||||
itemId: 'after-recovery',
|
||||
});
|
||||
const add =
|
||||
origin === 'tenant'
|
||||
? schemas.tenantStorage.run({ tenantId: 'tenant-a' }, addFavorite)
|
||||
: addFavorite();
|
||||
if (outcome === 'cancel') {
|
||||
expect((await owned(id)).agentTriggerDeletionStartedAt).toBeUndefined();
|
||||
expect((await owned(id)).agentTriggerDeletionAttemptId).toBeUndefined();
|
||||
await expect(add).resolves.toEqual({ ok: true, added: true });
|
||||
expect(
|
||||
(
|
||||
await db.query(
|
||||
'SELECT scope FROM tool_favorite_erasure WHERE user_id=$1 AND attempt_id=$2',
|
||||
[id.toString(), token],
|
||||
)
|
||||
).rows,
|
||||
).toEqual([]);
|
||||
} else {
|
||||
expect(await owned(id)).toBeNull();
|
||||
await expect(add).rejects.toThrow('erasure');
|
||||
}
|
||||
} finally {
|
||||
await restarted.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -25,6 +25,7 @@ const mockMethods = {
|
|||
deleteUserById: jest.fn(),
|
||||
deleteAgentTriggerDeliveriesByUser: jest.fn(),
|
||||
cancelAgentTriggerUserPurge: jest.fn(),
|
||||
isAgentTriggerUserDeletionOwner: jest.fn(),
|
||||
cancelAgentTriggerUserDeletion: jest.fn(),
|
||||
deleteUserPrompts: jest.fn(),
|
||||
};
|
||||
|
|
@ -34,6 +35,7 @@ const mockDestroy = jest.fn();
|
|||
const mockSilentExit = jest.fn();
|
||||
const mockAskQuestion = jest.fn();
|
||||
const mockGetAppConfig = jest.fn();
|
||||
const mockLoadCustomConfig = jest.fn();
|
||||
const mockRevokeUserCodeEnvironmentWorkers = jest.fn();
|
||||
const mockEraseUserData = jest.fn();
|
||||
const mockProcessDeleteRequest = jest.fn();
|
||||
|
|
@ -41,16 +43,25 @@ const mockInvalidateCachedTools = jest.fn();
|
|||
const mockInitializeFileStorage = jest.fn();
|
||||
const mockOpenCheckpointDeletion = jest.fn();
|
||||
const mockDeleteAllSharedLinksWithCleanup = jest.fn();
|
||||
const mockMongoFavorites = {
|
||||
getToolFavorites: jest.fn(),
|
||||
addToolFavorite: jest.fn(),
|
||||
removeToolFavorite: jest.fn(),
|
||||
deleteUserToolFavorites: jest.fn(),
|
||||
cancelUserToolFavoriteDeletion: jest.fn(),
|
||||
};
|
||||
|
||||
jest.mock('../connect', () => jest.fn(async () => undefined));
|
||||
jest.mock('mongoose', () => ({ disconnect: jest.fn(async () => undefined) }));
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
createModels: () => new Proxy({}, { get: (_target, prop) => mockModelFor(prop) }),
|
||||
createMethods: () => mockMethods,
|
||||
createMethods: (_mongoose, deps) => ({ ...mockMethods, ...deps.toolFavorites }),
|
||||
createToolFavoriteMethods: () => mockMongoFavorites,
|
||||
runAsSystem: (operation) => operation(),
|
||||
tenantStorage: mockTenantStorage,
|
||||
}));
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
waitForKeyvRedisClient: jest.fn(async () => undefined),
|
||||
createStreamServices: jest.fn(() => ({ isRedis: true })),
|
||||
GenerationJobManager: {
|
||||
|
|
@ -67,6 +78,7 @@ jest.mock('@librechat/api', () => ({
|
|||
deleteAllSharedLinksWithCleanup: mockDeleteAllSharedLinksWithCleanup,
|
||||
}));
|
||||
jest.mock('~/cache/getLogStores', () => jest.fn());
|
||||
jest.mock('~/server/services/Config/loadCustomConfig', () => mockLoadCustomConfig);
|
||||
jest.mock('~/server/services/Config', () => ({
|
||||
getAppConfig: (...args) => mockGetAppConfig(...args),
|
||||
}));
|
||||
|
|
@ -129,12 +141,16 @@ describe('Delete user CLI', () => {
|
|||
mockMethods.deleteUserById.mockReset().mockResolvedValue({ deletedCount: 1 });
|
||||
mockMethods.deleteAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(undefined);
|
||||
mockMethods.cancelAgentTriggerUserPurge.mockReset().mockResolvedValue(true);
|
||||
mockMethods.isAgentTriggerUserDeletionOwner.mockReset().mockResolvedValue(true);
|
||||
mockMethods.cancelAgentTriggerUserDeletion.mockReset().mockResolvedValue(true);
|
||||
mockMethods.deleteUserPrompts.mockReset().mockResolvedValue(undefined);
|
||||
mockGetCleanupBlockingJobIdsForUser.mockReset().mockResolvedValue([]);
|
||||
mockAbortJob.mockReset().mockResolvedValue({ success: true });
|
||||
mockDestroy.mockReset().mockResolvedValue(undefined);
|
||||
mockGetAppConfig.mockReset().mockResolvedValue({});
|
||||
mockLoadCustomConfig.mockReset().mockResolvedValue({});
|
||||
mockMongoFavorites.deleteUserToolFavorites.mockReset();
|
||||
mockMongoFavorites.cancelUserToolFavoriteDeletion.mockReset().mockResolvedValue(undefined);
|
||||
mockRevokeUserCodeEnvironmentWorkers.mockReset().mockResolvedValue(0);
|
||||
mockEraseUserData.mockReset().mockResolvedValue(undefined);
|
||||
mockProcessDeleteRequest.mockReset().mockResolvedValue({ deletedFileIds: [] });
|
||||
|
|
@ -149,12 +165,38 @@ describe('Delete user CLI', () => {
|
|||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('initializes the injected favorites methods and closes captured references on exit', async () => {
|
||||
let deleteFavorites;
|
||||
mockMongoFavorites.deleteUserToolFavorites.mockResolvedValueOnce(3);
|
||||
mockEraseUserData.mockImplementationOnce(async (_params, deps) => {
|
||||
deleteFavorites = deps.deleteUserToolFavorites;
|
||||
expect(await deleteFavorites(USER_ID)).toBe(3);
|
||||
});
|
||||
|
||||
expect(await runCli()).toBe(0);
|
||||
|
||||
expect(mockLoadCustomConfig).toHaveBeenCalledWith(false, { storageSelection: true });
|
||||
expect(mockMongoFavorites.deleteUserToolFavorites).toHaveBeenCalledWith(USER_ID);
|
||||
await expect(deleteFavorites(USER_ID)).rejects.toThrow('not initialized');
|
||||
});
|
||||
|
||||
it('deletes nothing when favorites persistence configuration is invalid', async () => {
|
||||
mockLoadCustomConfig.mockResolvedValueOnce({ persistence: { favorites: 'invalid' } });
|
||||
|
||||
expect(await runCli()).toBe(1);
|
||||
|
||||
expect(mockMethods.beginAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
expect(mockEraseUserData).not.toHaveBeenCalled();
|
||||
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fences triggers and schedules before inspecting active work', async () => {
|
||||
expect(await runCli()).toBe(0);
|
||||
|
||||
expect(mockMethods.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
USER_ID,
|
||||
expect.any(Date),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(mockMethods.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
|
||||
USER_ID,
|
||||
|
|
@ -206,7 +248,11 @@ describe('Delete user CLI', () => {
|
|||
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
|
||||
const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
|
||||
expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
USER_ID,
|
||||
deletionFence,
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
@ -229,6 +275,34 @@ describe('Delete user CLI', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each([false, 'error'])(
|
||||
'keeps favorites fenced when Mongo ownership is not proven: %s',
|
||||
async (result) => {
|
||||
mockEraseUserData.mockRejectedValueOnce(new Error('shared link failure'));
|
||||
if (result === 'error')
|
||||
mockMethods.isAgentTriggerUserDeletionOwner.mockRejectedValueOnce(
|
||||
new Error('Mongo unavailable'),
|
||||
);
|
||||
else mockMethods.isAgentTriggerUserDeletionOwner.mockResolvedValueOnce(false);
|
||||
expect(await runCli()).toBe(1);
|
||||
expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('releases only the cascade UUID before releasing Mongo ownership', async () => {
|
||||
mockEraseUserData.mockRejectedValueOnce(new Error('shared link failure'));
|
||||
expect(await runCli()).toBe(1);
|
||||
const token = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2];
|
||||
expect(mockEraseUserData.mock.calls[0][0].deletionAttemptId).toBe(token);
|
||||
expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(USER_ID, token);
|
||||
expect(mockMethods.isAgentTriggerUserDeletionOwner.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockMongoFavorites.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0],
|
||||
);
|
||||
expect(
|
||||
mockMongoFavorites.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0],
|
||||
).toBeLessThan(mockMethods.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0]);
|
||||
});
|
||||
|
||||
it('never restores schedules after a committed deletion', async () => {
|
||||
expect(await runCli()).toBe(0);
|
||||
|
||||
|
|
@ -240,6 +314,13 @@ describe('Delete user CLI', () => {
|
|||
expect(await runCli()).toBe(0);
|
||||
|
||||
expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true });
|
||||
expect(mockMethods.deleteUserById).toHaveBeenCalledWith(USER_ID, {
|
||||
startedAt: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1],
|
||||
attemptId: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2],
|
||||
});
|
||||
expect(mockMethods.deleteUserById.mock.calls[0][1].attemptId).not.toBe(
|
||||
mockMethods.suspendUserSchedulesForDeletion.mock.calls[0][1],
|
||||
);
|
||||
expect(mockRevokeUserCodeEnvironmentWorkers).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ userId: USER_ID }),
|
||||
);
|
||||
|
|
@ -279,6 +360,7 @@ describe('Delete user CLI', () => {
|
|||
tenantId: 'tenant-1',
|
||||
checkpointer: { ttl: 60 },
|
||||
keepTransactions: true,
|
||||
deletionAttemptId: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2],
|
||||
});
|
||||
expect(deps).toEqual(
|
||||
expect.objectContaining({
|
||||
|
|
@ -337,6 +419,8 @@ describe('Delete user CLI', () => {
|
|||
});
|
||||
|
||||
it('supplies shared-user MCP OAuth cleanup to the cascade', async () => {
|
||||
mockMethods.findMCPServerByObjectId = jest.fn(async () => ({ config: { type: 'sse' } }));
|
||||
mockMethods.getUserById = jest.fn(async () => ({ role: 'VIEWER' }));
|
||||
expect(await runCli()).toBe(0);
|
||||
|
||||
const [, deps] = mockEraseUserData.mock.calls[0];
|
||||
|
|
@ -347,10 +431,8 @@ describe('Delete user CLI', () => {
|
|||
invalidateUserTools: expect.any(Function),
|
||||
uninstallOAuthMCP: expect.any(Function),
|
||||
});
|
||||
mockMethods.findMCPServerByObjectId = jest.fn(async () => ({ config: { type: 'sse' } }));
|
||||
expect(await deps.mcpOAuth.getServerConfig('server-1', 'server-id-1')).toEqual({ type: 'sse' });
|
||||
expect(mockMethods.findMCPServerByObjectId).toHaveBeenCalledWith('server-id-1');
|
||||
mockMethods.getUserById = jest.fn(async () => ({ role: 'VIEWER' }));
|
||||
mockGetAppConfig.mockImplementation(async (options) =>
|
||||
options?.userId === 'viewer-1' ? { mcpSettings: { allowedDomains: ['viewer.example'] } } : {},
|
||||
);
|
||||
|
|
@ -380,6 +462,19 @@ describe('Delete user CLI', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it.each(['unknown', 'absent'])(
|
||||
'keeps both erasure fences after final delete starts: %s',
|
||||
async (outcome) => {
|
||||
if (outcome === 'unknown')
|
||||
mockMethods.deleteUserById.mockRejectedValueOnce(new Error('delete acknowledgement lost'));
|
||||
else mockMethods.deleteUserById.mockResolvedValueOnce({ deletedCount: 0 });
|
||||
expect(await runCli()).toBe(1);
|
||||
expect(mockMethods.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled();
|
||||
expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps the user and releases both fences when the cascade fails', async () => {
|
||||
mockEraseUserData.mockRejectedValueOnce(new Error('storage unavailable'));
|
||||
|
||||
|
|
@ -388,7 +483,11 @@ describe('Delete user CLI', () => {
|
|||
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
|
||||
const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
|
||||
expect(mockMethods.restoreUserSchedulesFromDeletion).toHaveBeenCalled();
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
USER_ID,
|
||||
deletionFence,
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
|
||||
it('passes deleteUserPrompts through to the shared cascade, which keeps a synced group with a synthetic author untouched', async () => {
|
||||
|
|
@ -409,6 +508,10 @@ describe('Delete user CLI', () => {
|
|||
expect(mockMethods.deleteUserById).not.toHaveBeenCalled();
|
||||
const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1];
|
||||
expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence);
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence);
|
||||
expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
USER_ID,
|
||||
deletionFence,
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
jest.mock('../connect', () => jest.fn().mockResolvedValue(true));
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
ensureRequiredCollectionsExist: jest.fn().mockResolvedValue(undefined),
|
||||
createChannelStorageConfigReader: jest.fn(() => ({
|
||||
initialize: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -4,16 +4,12 @@
|
|||
const path = require('path');
|
||||
const mongoose = require('mongoose');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const {
|
||||
createModels,
|
||||
createMethods,
|
||||
runAsSystem,
|
||||
tenantStorage,
|
||||
} = require('@librechat/data-schemas');
|
||||
const { createModels, runAsSystem, tenantStorage } = require('@librechat/data-schemas');
|
||||
const { User } = createModels(mongoose);
|
||||
require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });
|
||||
const {
|
||||
eraseUserData,
|
||||
createUserDeletionAttempt,
|
||||
GenerationJobManager,
|
||||
createStreamServices,
|
||||
initializeFileStorage,
|
||||
|
|
@ -36,13 +32,16 @@ const connect = require('./connect');
|
|||
|
||||
const TRIGGER_DRAIN_TIMEOUT_MS = 35_000;
|
||||
const TRIGGER_DRAIN_POLL_MS = 100;
|
||||
const methods = createMethods(mongoose, { getCache: getLogStores });
|
||||
const { initializeFavorites, closeFavorites, ...methods } = require('~/models');
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function gracefulExit(code = 0) {
|
||||
try {
|
||||
await mongoose.disconnect();
|
||||
const cleanup = await Promise.allSettled([closeFavorites(), mongoose.disconnect()]);
|
||||
if (cleanup.some((result) => result.status === 'rejected')) {
|
||||
throw new Error('Persistence cleanup failed');
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error disconnecting from MongoDB:', err);
|
||||
}
|
||||
|
|
@ -51,6 +50,9 @@ async function gracefulExit(code = 0) {
|
|||
|
||||
(async () => {
|
||||
await connect();
|
||||
await initializeFavorites(
|
||||
await require('~/server/services/Config/loadCustomConfig')(false, { storageSelection: true }),
|
||||
);
|
||||
|
||||
console.purple('---------------');
|
||||
console.purple('Deleting a user and all related data');
|
||||
|
|
@ -115,11 +117,12 @@ async function gracefulExit(code = 0) {
|
|||
let deletionFence;
|
||||
let scheduleSuspensionToken;
|
||||
let userDeleted = false;
|
||||
const deletionAttempt = createUserDeletionAttempt();
|
||||
|
||||
try {
|
||||
deletionFence = new Date();
|
||||
let fenceState = await runAsSystem(() =>
|
||||
methods.beginAgentTriggerUserDeletion(uid, deletionFence),
|
||||
methods.beginAgentTriggerUserDeletion(uid, deletionFence, deletionAttempt.id),
|
||||
);
|
||||
if (fenceState === 'in_progress') {
|
||||
deletionFence = undefined;
|
||||
|
|
@ -135,7 +138,7 @@ async function gracefulExit(code = 0) {
|
|||
|
||||
const recoveredAt = new Date();
|
||||
fenceState = await runAsSystem(() =>
|
||||
methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt),
|
||||
methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt, deletionAttempt.id),
|
||||
);
|
||||
if (fenceState !== 'acquired') {
|
||||
throw new Error(
|
||||
|
|
@ -145,6 +148,7 @@ async function gracefulExit(code = 0) {
|
|||
);
|
||||
}
|
||||
deletionFence = recoveredAt;
|
||||
await runAsSystem(() => deletionAttempt.adopt(uid, recoveredAt, methods));
|
||||
}
|
||||
if (fenceState === 'missing') {
|
||||
deletionFence = undefined;
|
||||
|
|
@ -214,6 +218,7 @@ async function gracefulExit(code = 0) {
|
|||
eraseUserData(
|
||||
{
|
||||
userId: uid,
|
||||
deletionAttemptId: deletionAttempt.id,
|
||||
tenantId: user.tenantId,
|
||||
checkpointer: userAppConfig?.endpoints?.agents?.checkpointer,
|
||||
keepTransactions: !deleteTx,
|
||||
|
|
@ -283,7 +288,14 @@ async function gracefulExit(code = 0) {
|
|||
);
|
||||
|
||||
// 6) Finally delete the user document itself
|
||||
const deletedUser = await runAsSystem(() => methods.deleteUserById(uid));
|
||||
const deletedUser = await deletionAttempt.commit(() =>
|
||||
runAsSystem(() =>
|
||||
methods.deleteUserById(uid, {
|
||||
startedAt: deletionFence,
|
||||
attemptId: deletionAttempt.id,
|
||||
}),
|
||||
),
|
||||
);
|
||||
if (deletedUser.deletedCount !== 1) {
|
||||
throw new Error('User disappeared before account deletion could commit');
|
||||
}
|
||||
|
|
@ -326,9 +338,12 @@ async function gracefulExit(code = 0) {
|
|||
await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch(
|
||||
(error) => console.error('Failed to disarm trigger purge recovery:', error),
|
||||
);
|
||||
await runAsSystem(() => methods.cancelAgentTriggerUserDeletion(uid, deletionFence)).catch(
|
||||
(error) => console.error('Failed to release account-deletion fence:', error),
|
||||
);
|
||||
await runAsSystem(() =>
|
||||
deletionAttempt.cancel(
|
||||
{ userId: uid, startedAt: deletionFence, deletionAttemptId: deletionAttempt.id },
|
||||
methods,
|
||||
),
|
||||
).catch((error) => console.error('Failed to release account-deletion fence:', error));
|
||||
}
|
||||
if (hasSharedGenerationStore) {
|
||||
await GenerationJobManager.destroy().catch((error) =>
|
||||
|
|
@ -343,11 +358,7 @@ async function gracefulExit(code = 0) {
|
|||
}
|
||||
|
||||
return gracefulExit(0);
|
||||
})().catch(async (err) => {
|
||||
if (!err.message.includes('fetch failed')) {
|
||||
console.error('There was an uncaught error:');
|
||||
console.error(err);
|
||||
await mongoose.disconnect();
|
||||
process.exit(1);
|
||||
}
|
||||
})().catch(async () => {
|
||||
console.error('Account deletion failed; retry before considering the account erased.');
|
||||
await gracefulExit(1);
|
||||
});
|
||||
|
|
|
|||
21
config/postgres.js
Normal file
21
config/postgres.js
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env node
|
||||
require('dotenv').config();
|
||||
require('module-alias/register');
|
||||
const path = require('node:path');
|
||||
require('module-alias').addAlias('~', path.resolve(__dirname, '../api'));
|
||||
const { runPostgresCommand } = require('@librechat/api');
|
||||
const mongoose = require('mongoose');
|
||||
const { createMongoFavoriteStore, createPostgresDatabase } = require('@librechat/data-schemas');
|
||||
const loadCustomConfig = require('~/server/services/Config/loadCustomConfig');
|
||||
runPostgresCommand({
|
||||
favorites: createMongoFavoriteStore(mongoose),
|
||||
createDatabase: createPostgresDatabase,
|
||||
connectDb: () => mongoose.connect(process.env.MONGO_URI, { autoCreate: false, autoIndex: false }),
|
||||
disconnectDb: () => mongoose.disconnect(),
|
||||
loadCustomConfig,
|
||||
args: process.argv.slice(2),
|
||||
env: process.env,
|
||||
}).catch(() => {
|
||||
console.error('PostgreSQL operation failed. No automatic storage cutover was performed.');
|
||||
process.exitCode = 1;
|
||||
});
|
||||
154
package-lock.json
generated
154
package-lock.json
generated
|
|
@ -19958,6 +19958,18 @@
|
|||
"@types/passport": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pg": {
|
||||
"version": "8.23.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz",
|
||||
"integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"pg-protocol": "*",
|
||||
"pg-types": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/prop-types": {
|
||||
"version": "15.7.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
|
||||
|
|
@ -35414,6 +35426,95 @@
|
|||
"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.23.1",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz",
|
||||
"integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-connection-string": "^2.14.1",
|
||||
"pg-pool": "^3.14.0",
|
||||
"pg-protocol": "^1.16.1",
|
||||
"pg-types": "2.2.0",
|
||||
"pgpass": "1.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 16.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"pg-cloudflare": "^1.4.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"pg-native": ">=3.0.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"pg-native": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/pg-cloudflare": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz",
|
||||
"integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/pg-connection-string": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz",
|
||||
"integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-int8": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
||||
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-pool": {
|
||||
"version": "3.14.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
|
||||
"integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"pg": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg-protocol": {
|
||||
"version": "1.16.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz",
|
||||
"integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pg-types": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
|
||||
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pg-int8": "1.0.1",
|
||||
"postgres-array": "~2.0.0",
|
||||
"postgres-bytea": "~1.0.0",
|
||||
"postgres-date": "~1.0.4",
|
||||
"postgres-interval": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/pgpass": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
|
||||
"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
|
|
@ -35639,6 +35740,45 @@
|
|||
"node": "^10 || ^12 || >=14"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-array": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
|
||||
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-bytea": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
|
||||
"integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-date": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
|
||||
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/postgres-interval": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
|
||||
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/powershell-utils": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.2.1.tgz",
|
||||
|
|
@ -38744,6 +38884,15 @@
|
|||
"node": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
|
|
@ -41977,7 +42126,6 @@
|
|||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4"
|
||||
|
|
@ -45298,13 +45446,15 @@
|
|||
"micromark-extension-directive": "^3.0.1",
|
||||
"micromark-extension-gfm": "^3.0.0",
|
||||
"micromark-extension-llm-math": "^3.1.0",
|
||||
"micromark-util-decode-string": "^2.0.0"
|
||||
"micromark-util-decode-string": "^2.0.0",
|
||||
"pg": "^8.23.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@stryker-mutator/core": "10.1.1",
|
||||
"@stryker-mutator/jest-runner": "10.1.1",
|
||||
"@types/jest": "^29.5.2",
|
||||
"@types/node": "^24.12.4",
|
||||
"@types/pg": "^8.23.1",
|
||||
"jest": "^30.2.0",
|
||||
"jest-junit": "^17.0.0",
|
||||
"mongodb-memory-server": "^11.3.0",
|
||||
|
|
|
|||
|
|
@ -166,7 +166,8 @@
|
|||
"migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50",
|
||||
"lighthouse": "npm run e2e:prepare && npm run lighthouse:run",
|
||||
"lighthouse:run": "playwright test --config=e2e/playwright.config.lighthouse.ts",
|
||||
"lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run"
|
||||
"lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run",
|
||||
"postgres": "node config/postgres.js"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { createAdminUsersHandlers } from './users';
|
|||
import { eraseUserData } from '~/user/erasure';
|
||||
|
||||
jest.mock('~/user/erasure', () => ({
|
||||
...jest.requireActual('~/user/erasure'),
|
||||
eraseUserData: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
|
|
@ -60,6 +61,7 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {
|
|||
findUsers: jest.fn().mockResolvedValue([]),
|
||||
countUsers: jest.fn().mockResolvedValue(0),
|
||||
beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),
|
||||
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true),
|
||||
cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),
|
||||
drainAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined),
|
||||
prepareAgentTriggerUserPurge: jest.fn().mockResolvedValue(undefined),
|
||||
|
|
@ -392,6 +394,7 @@ describe('createAdminUsersHandlers', () => {
|
|||
expect(deps.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
validUserId,
|
||||
expect.any(Date),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(deps.drainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId);
|
||||
expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
|
||||
|
|
@ -461,7 +464,10 @@ describe('createAdminUsersHandlers', () => {
|
|||
await handlers.deleteUser(req, res);
|
||||
|
||||
expect(status).toHaveBeenCalledWith(200);
|
||||
expect(deps.deleteUserById).toHaveBeenCalledWith(targetId);
|
||||
expect(deps.deleteUserById).toHaveBeenCalledWith(targetId, {
|
||||
startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1],
|
||||
attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not check admin count when target is a regular user', async () => {
|
||||
|
|
@ -559,7 +565,12 @@ describe('createAdminUsersHandlers', () => {
|
|||
}),
|
||||
);
|
||||
expect(eraseUserData).toHaveBeenCalledWith(
|
||||
{ userId: validUserId, tenantId: 'tenant-1', checkpointer },
|
||||
{
|
||||
userId: validUserId,
|
||||
tenantId: 'tenant-1',
|
||||
checkpointer,
|
||||
deletionAttemptId: expect.any(String),
|
||||
},
|
||||
erasureDeps,
|
||||
);
|
||||
expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan(
|
||||
|
|
@ -582,6 +593,34 @@ describe('createAdminUsersHandlers', () => {
|
|||
expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'releases the resolved favorite UUID only after a proven Mongo owner: %s',
|
||||
async (owned) => {
|
||||
const cancelUserToolFavoriteDeletion = jest.fn().mockResolvedValue(undefined);
|
||||
jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('later cascade failure'));
|
||||
const deps = createDeps({
|
||||
isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(owned),
|
||||
resolveErasure: jest.fn().mockResolvedValue({ deps: { cancelUserToolFavoriteDeletion } }),
|
||||
});
|
||||
const { req, res } = createReqRes({ params: { id: validUserId } });
|
||||
await createAdminUsersHandlers(deps).deleteUser(req, res);
|
||||
const token = jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2];
|
||||
expect(eraseUserData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ deletionAttemptId: token }),
|
||||
expect.anything(),
|
||||
);
|
||||
if (owned) {
|
||||
expect(cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(validUserId, token);
|
||||
expect(
|
||||
jest.mocked(deps.isAgentTriggerUserDeletionOwner).mock.invocationCallOrder[0],
|
||||
).toBeLessThan(cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]);
|
||||
expect(cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0],
|
||||
);
|
||||
} else expect(cancelUserToolFavoriteDeletion).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('suspends schedules before erasing and restores them when the deletion fails', async () => {
|
||||
jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));
|
||||
const deps = createDeps();
|
||||
|
|
@ -677,7 +716,10 @@ describe('createAdminUsersHandlers', () => {
|
|||
|
||||
await handlers.deleteUser(req, res);
|
||||
|
||||
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId);
|
||||
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId, {
|
||||
startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1],
|
||||
attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2],
|
||||
});
|
||||
expect(deps.deleteUserCodeEnvironments).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -694,10 +736,7 @@ describe('createAdminUsersHandlers', () => {
|
|||
expect(deps.deleteUserCodeEnvironments).not.toHaveBeenCalled();
|
||||
expect(deps.deleteAclEntries).not.toHaveBeenCalled();
|
||||
expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
|
||||
expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
validUserId,
|
||||
expect.any(Date),
|
||||
);
|
||||
expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));
|
||||
});
|
||||
|
||||
|
|
@ -735,10 +774,8 @@ describe('createAdminUsersHandlers', () => {
|
|||
|
||||
expect(status).toHaveBeenCalledWith(500);
|
||||
expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
|
||||
expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
|
||||
validUserId,
|
||||
expect.any(Date),
|
||||
);
|
||||
expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
expect(deps.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled();
|
||||
expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));
|
||||
expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -815,7 +852,10 @@ describe('createAdminUsersHandlers', () => {
|
|||
|
||||
expect(status).toHaveBeenCalledWith(500);
|
||||
expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
|
||||
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId);
|
||||
expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId, {
|
||||
startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1],
|
||||
attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2],
|
||||
});
|
||||
expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalled();
|
||||
expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled();
|
||||
expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import type {
|
|||
ConfigRecord,
|
||||
AdminUserListItem,
|
||||
AdminUserSearchResult,
|
||||
UserDeleteResult,
|
||||
AllMethods,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { TCheckpointerConfig } from 'librechat-data-provider';
|
||||
import type { FilterQuery } from 'mongoose';
|
||||
|
|
@ -15,10 +15,10 @@ import type { Response } from 'express';
|
|||
import type { UserGenerationControl } from '~/user/generations';
|
||||
import type { UserErasureDeps } from '~/user/erasure';
|
||||
import type { ServerRequest } from '~/types/http';
|
||||
import { eraseUserData, createUserDeletionAttempt } from '~/user/erasure';
|
||||
import { stopUserGenerations } from '~/user/generations';
|
||||
import { invalidatingConfigStore } from './cascade';
|
||||
import { parsePagination } from './pagination';
|
||||
import { eraseUserData } from '~/user/erasure';
|
||||
|
||||
const MAX_SEARCH_LENGTH = 200;
|
||||
|
||||
|
|
@ -75,8 +75,18 @@ export interface AdminUsersDeps {
|
|||
beginAgentTriggerUserDeletion: (
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
) => Promise<'acquired' | 'in_progress' | 'missing'>;
|
||||
cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;
|
||||
isAgentTriggerUserDeletionOwner: (
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
) => Promise<boolean>;
|
||||
cancelAgentTriggerUserDeletion: (
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
) => Promise<boolean>;
|
||||
drainAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;
|
||||
prepareAgentTriggerUserPurge: (
|
||||
userId: string,
|
||||
|
|
@ -102,7 +112,7 @@ export interface AdminUsersDeps {
|
|||
*/
|
||||
resolveErasure: (target: AdminErasureTarget) => Promise<AdminErasureContext>;
|
||||
/** Thin data-layer delete: removes the User document after the erasure cascade has run. */
|
||||
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
|
||||
deleteUserById: AllMethods['deleteUserById'];
|
||||
deleteUserCodeEnvironments: (userId: string | Types.ObjectId) => Promise<number>;
|
||||
invalidateCodeEnvironmentConfigCache: (tenantId?: string) => Promise<void>;
|
||||
deleteConfig: (
|
||||
|
|
@ -132,6 +142,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
|
|||
findUsers,
|
||||
countUsers,
|
||||
beginAgentTriggerUserDeletion,
|
||||
isAgentTriggerUserDeletionOwner,
|
||||
cancelAgentTriggerUserDeletion,
|
||||
drainAgentTriggerDeliveriesForUser,
|
||||
prepareAgentTriggerUserPurge,
|
||||
|
|
@ -232,6 +243,8 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
|
|||
let triggerDeletionFence: Date | undefined;
|
||||
let scheduleSuspensionToken: string | undefined;
|
||||
let userDeleted = false;
|
||||
const deletionAttempt = createUserDeletionAttempt();
|
||||
let erasureDeps: UserErasureDeps | undefined;
|
||||
|
||||
try {
|
||||
const { id } = req.params as { id: string };
|
||||
|
|
@ -255,7 +268,11 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
|
|||
}
|
||||
|
||||
triggerDeletionFence = new Date();
|
||||
const fenceState = await beginAgentTriggerUserDeletion(id, triggerDeletionFence);
|
||||
const fenceState = await beginAgentTriggerUserDeletion(
|
||||
id,
|
||||
triggerDeletionFence,
|
||||
deletionAttempt.id,
|
||||
);
|
||||
if (fenceState === 'in_progress') {
|
||||
triggerDeletionFence = undefined;
|
||||
return res.status(409).json({ error: 'User deletion is already in progress' });
|
||||
|
|
@ -274,22 +291,40 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
|
|||
}
|
||||
await stopUserGenerations(id, targetUser?.tenantId, generationJobs);
|
||||
|
||||
const { deps: erasureDeps, checkpointer } = await resolveErasure({
|
||||
const erasure = await resolveErasure({
|
||||
id,
|
||||
role: targetUser?.role,
|
||||
tenantId: targetUser?.tenantId,
|
||||
idOnTheSource: targetUser?.idOnTheSource ?? null,
|
||||
profile: targetUser,
|
||||
});
|
||||
erasureDeps = erasure.deps;
|
||||
await eraseUserData(
|
||||
{ userId: id, tenantId: targetUser?.tenantId, checkpointer },
|
||||
{
|
||||
userId: id,
|
||||
tenantId: targetUser?.tenantId,
|
||||
checkpointer: erasure.checkpointer,
|
||||
deletionAttemptId: deletionAttempt.id,
|
||||
},
|
||||
erasureDeps,
|
||||
);
|
||||
const result = await deleteUserById(id);
|
||||
const deletionFence = { startedAt: triggerDeletionFence, attemptId: deletionAttempt.id };
|
||||
const result = await deletionAttempt.commit(() => deleteUserById(id, deletionFence));
|
||||
|
||||
if (result.deletedCount === 0) {
|
||||
await cancelAgentTriggerUserPurge(id, triggerDeletionFence);
|
||||
await cancelAgentTriggerUserDeletion(id, triggerDeletionFence);
|
||||
await deletionAttempt.cancel(
|
||||
{
|
||||
userId: id,
|
||||
startedAt: triggerDeletionFence,
|
||||
deletionAttemptId: deletionAttempt.id,
|
||||
},
|
||||
{
|
||||
isAgentTriggerUserDeletionOwner,
|
||||
cancelAgentTriggerUserDeletion,
|
||||
cancelUserToolFavoriteDeletion: erasureDeps?.cancelUserToolFavoriteDeletion,
|
||||
},
|
||||
);
|
||||
triggerDeletionFence = undefined;
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
|
|
@ -353,7 +388,18 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
|
|||
logger.error('[adminUsers] failed to disarm trigger purge recovery:', purgeFenceError);
|
||||
}
|
||||
try {
|
||||
await cancelAgentTriggerUserDeletion(targetUserId, triggerDeletionFence);
|
||||
await deletionAttempt.cancel(
|
||||
{
|
||||
userId: targetUserId,
|
||||
startedAt: triggerDeletionFence,
|
||||
deletionAttemptId: deletionAttempt.id,
|
||||
},
|
||||
{
|
||||
isAgentTriggerUserDeletionOwner,
|
||||
cancelAgentTriggerUserDeletion,
|
||||
cancelUserToolFavoriteDeletion: erasureDeps?.cancelUserToolFavoriteDeletion,
|
||||
},
|
||||
);
|
||||
} catch (fenceError) {
|
||||
logger.error('[adminUsers] failed to release trigger deletion fence:', fenceError);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,3 +22,7 @@ export { resolveBuildInfo } from './build';
|
|||
export type { BuildInfo } from './build';
|
||||
export { buildPreLoginInterface, withSafePolicyLinks } from './interface';
|
||||
export * from './migrations';
|
||||
|
||||
export * from './persistence';
|
||||
|
||||
export * from './postgres';
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { createHash } from 'node:crypto';
|
|||
import { logger } from '@librechat/data-schemas';
|
||||
import {
|
||||
configSchema,
|
||||
persistenceConfigSchema,
|
||||
EModelEndpoint,
|
||||
EImageOutputType,
|
||||
getMaxSubagents,
|
||||
|
|
@ -21,6 +22,8 @@ export type CustomConfigLoadMode = 'startup' | 'reload';
|
|||
|
||||
export interface CustomConfigLoadOptions {
|
||||
mode?: CustomConfigLoadMode;
|
||||
/** Storage initialization must distinguish an absent default from an unreadable source. */
|
||||
storageSelection?: boolean;
|
||||
}
|
||||
|
||||
export interface LoadedCustomConfig {
|
||||
|
|
@ -192,6 +195,17 @@ export function createCustomConfigLoader({
|
|||
cause?: unknown,
|
||||
startupLogLevel: 'error' | 'info' = 'error',
|
||||
): null => {
|
||||
if (
|
||||
options.storageSelection &&
|
||||
!(
|
||||
cause instanceof Error &&
|
||||
'code' in cause &&
|
||||
cause.code === 'ENOENT' &&
|
||||
!process.env.CONFIG_PATH
|
||||
)
|
||||
) {
|
||||
throw new ConfigReloadError('Deployment config unavailable for storage selection');
|
||||
}
|
||||
if (mode === 'reload') {
|
||||
if (cause == null) {
|
||||
logger.error(message);
|
||||
|
|
@ -261,6 +275,16 @@ export function createCustomConfigLoader({
|
|||
}
|
||||
|
||||
/** The checks below return copies, but `modelSpecs` is assigned onto this object later. */
|
||||
if (
|
||||
typeof loadedConfig === 'object' &&
|
||||
loadedConfig != null &&
|
||||
'persistence' in loadedConfig
|
||||
) {
|
||||
const persistence = persistenceConfigSchema.safeParse(loadedConfig.persistence);
|
||||
if (!persistence.success) {
|
||||
throw new ConfigReloadError('Invalid persistence configuration');
|
||||
}
|
||||
}
|
||||
const parsedConfig = loadedConfig;
|
||||
const themeCheck = checkConfigTheme(loadedConfig);
|
||||
loadedConfig = themeCheck.config;
|
||||
|
|
@ -302,6 +326,13 @@ export function createCustomConfigLoader({
|
|||
);
|
||||
}
|
||||
|
||||
if (
|
||||
typeof loadedConfig === 'object' &&
|
||||
loadedConfig != null &&
|
||||
'persistence' in loadedConfig
|
||||
) {
|
||||
throw new ConfigReloadError('Invalid configuration containing persistence selection');
|
||||
}
|
||||
if (process.env.CONFIG_BYPASS_VALIDATION === 'true') {
|
||||
logger.warn(
|
||||
'CONFIG_BYPASS_VALIDATION is enabled. Continuing with default configuration despite validation errors.',
|
||||
|
|
|
|||
319
packages/api/src/app/persistence.spec.ts
Normal file
319
packages/api/src/app/persistence.spec.ts
Normal file
|
|
@ -0,0 +1,319 @@
|
|||
import pg from 'pg';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import {
|
||||
createPostgresFavoriteBackend,
|
||||
createPostgresDatabase,
|
||||
migratePostgres,
|
||||
importFavorites,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { ToolFavoriteMethods } from '@librechat/data-schemas';
|
||||
import type { Response } from 'express';
|
||||
import type { ServerRequest } from '~/types';
|
||||
import { createFavoriteRuntime, getPersistenceConfig } from './persistence';
|
||||
import { createToolFavoritesHandlers } from '~/favorites/handlers';
|
||||
import { createCustomConfigLoader } from './loader';
|
||||
|
||||
const mongo: ToolFavoriteMethods = {
|
||||
getToolFavorites: jest.fn().mockResolvedValue([]),
|
||||
addToolFavorite: jest.fn().mockResolvedValue({ ok: true, added: true }),
|
||||
removeToolFavorite: jest.fn().mockResolvedValue({ ok: true, removed: true }),
|
||||
deleteUserToolFavorites: jest.fn().mockResolvedValue(2),
|
||||
};
|
||||
|
||||
describe('favorites process lifecycle', () => {
|
||||
it('retains a captured method across initialization and erases through the selected domain', async () => {
|
||||
const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend);
|
||||
const captured = runtime.methods.deleteUserToolFavorites;
|
||||
await expect(captured('user')).rejects.toThrow('not initialized');
|
||||
await runtime.initialize(undefined);
|
||||
expect(await captured('user')).toBe(2);
|
||||
await runtime.close();
|
||||
await expect(captured('user')).rejects.toThrow('not initialized');
|
||||
await expect(runtime.initialize(undefined)).rejects.toThrow('closed');
|
||||
});
|
||||
it('keeps the optional Mongo cancellation hook a no-op without changing captured references', async () => {
|
||||
const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend);
|
||||
const cancel = runtime.methods.cancelUserToolFavoriteDeletion!;
|
||||
await expect(cancel('user', 'attempt')).rejects.toThrow('not initialized');
|
||||
await runtime.initialize(undefined);
|
||||
await expect(cancel('user', 'attempt')).resolves.toBeUndefined();
|
||||
await runtime.close();
|
||||
await expect(cancel('user', 'attempt')).rejects.toThrow('not initialized');
|
||||
});
|
||||
it('fails closed on invalid explicit storage config and missing DSN', async () => {
|
||||
await expect(
|
||||
createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend).initialize({
|
||||
favorites: 'invalid',
|
||||
}),
|
||||
).rejects.toThrow('Invalid');
|
||||
await expect(
|
||||
createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend).initialize({
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'PG_DSN' },
|
||||
}),
|
||||
).rejects.toThrow('missing');
|
||||
});
|
||||
it('selects once for a process lifetime, never a request or config reload', async () => {
|
||||
const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend);
|
||||
await runtime.initialize(undefined);
|
||||
await runtime.initialize({ favorites: 'postgres' });
|
||||
expect(await runtime.methods.getToolFavorites('user')).toEqual([]);
|
||||
await runtime.close();
|
||||
});
|
||||
it('constructs only the selected injected backend and retains captured method references', async () => {
|
||||
const backend = {
|
||||
methods: {
|
||||
...mongo,
|
||||
getToolFavorites: jest.fn(async () => [{ itemType: 'tool' as const, itemId: 'injected' }]),
|
||||
},
|
||||
initialize: jest.fn(async () => undefined),
|
||||
close: jest.fn(async () => undefined),
|
||||
};
|
||||
const factory = jest.fn(() => backend);
|
||||
const runtime = createFavoriteRuntime(mongo, { INJECTED_PG: 'owned-connection' }, factory);
|
||||
const captured = runtime.methods.getToolFavorites;
|
||||
expect(factory).not.toHaveBeenCalled();
|
||||
await expect(captured('user')).rejects.toThrow('not initialized');
|
||||
await runtime.initialize({
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'INJECTED_PG' },
|
||||
});
|
||||
await runtime.initialize(undefined);
|
||||
expect(factory).toHaveBeenCalledTimes(1);
|
||||
expect(factory).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
connectionString: 'owned-connection',
|
||||
max: 10,
|
||||
queryTimeoutMs: 15000,
|
||||
}),
|
||||
);
|
||||
expect(backend.initialize).toHaveBeenCalledTimes(1);
|
||||
expect(runtime.methods.getToolFavorites).toBe(captured);
|
||||
expect(await captured('user')).toEqual([{ itemType: 'tool', itemId: 'injected' }]);
|
||||
await runtime.close();
|
||||
await runtime.close();
|
||||
expect(backend.close).toHaveBeenCalledTimes(1);
|
||||
await expect(captured('user')).rejects.toThrow('not initialized');
|
||||
const unused = jest.fn(() => backend);
|
||||
const defaultRuntime = createFavoriteRuntime(mongo, {}, unused);
|
||||
await defaultRuntime.initialize(undefined);
|
||||
expect(unused).not.toHaveBeenCalled();
|
||||
await defaultRuntime.close();
|
||||
});
|
||||
it.each(['construct', 'initialize'] as const)(
|
||||
'sanitizes an injected %s failure without publishing methods or falling back',
|
||||
async (stage) => {
|
||||
const privateError = new Error('private backend detail');
|
||||
const backend = {
|
||||
methods: mongo,
|
||||
initialize: jest.fn().mockRejectedValue(privateError),
|
||||
close: jest.fn(async () => undefined),
|
||||
};
|
||||
const factory = jest.fn(() => {
|
||||
if (stage === 'construct') throw privateError;
|
||||
return backend;
|
||||
});
|
||||
const runtime = createFavoriteRuntime(mongo, { INJECTED_PG: 'owned-connection' }, factory);
|
||||
await expect(
|
||||
runtime.initialize({
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'INJECTED_PG' },
|
||||
}),
|
||||
).rejects.toThrow('PostgreSQL favorites initialization failed');
|
||||
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
|
||||
await runtime.close();
|
||||
expect(factory).toHaveBeenCalledTimes(1);
|
||||
expect(backend.close).toHaveBeenCalledTimes(stage === 'initialize' ? 1 : 0);
|
||||
},
|
||||
);
|
||||
it('waits for the injected initializer during shutdown and never publishes its methods', async () => {
|
||||
let ready!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
ready = resolve;
|
||||
});
|
||||
const backend = {
|
||||
methods: mongo,
|
||||
initialize: jest.fn(() => gate),
|
||||
close: jest.fn(async () => undefined),
|
||||
};
|
||||
const runtime = createFavoriteRuntime(
|
||||
mongo,
|
||||
{ INJECTED_PG: 'owned-connection' },
|
||||
() => backend,
|
||||
);
|
||||
const initialization = runtime.initialize({
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'INJECTED_PG' },
|
||||
});
|
||||
const result = initialization.catch((error: Error) => error);
|
||||
const closing = runtime.close();
|
||||
expect(backend.close).not.toHaveBeenCalled();
|
||||
ready();
|
||||
expect(await result).toEqual(new Error('PostgreSQL favorites initialization failed'));
|
||||
await closing;
|
||||
expect(backend.close).toHaveBeenCalledTimes(1);
|
||||
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
|
||||
});
|
||||
it('extracts only deployment persistence and rejects malformed deployment data', () => {
|
||||
expect(getPersistenceConfig(null)).toBeUndefined();
|
||||
expect(getPersistenceConfig({ persistence: { favorites: 'mongo' } })).toEqual({
|
||||
favorites: 'mongo',
|
||||
});
|
||||
expect(() => getPersistenceConfig('invalid')).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('storage selection source handling', () => {
|
||||
const create = (loadLocal: () => unknown) =>
|
||||
createCustomConfigLoader({
|
||||
defaultConfigPath: 'librechat.yaml',
|
||||
loadLocal,
|
||||
redactConfig: (config) => config,
|
||||
});
|
||||
it('permits an absent default file, but never malformed YAML or an explicit missing source', async () => {
|
||||
const original = process.env.CONFIG_PATH;
|
||||
delete process.env.CONFIG_PATH;
|
||||
const missing = Object.assign(new Error('missing'), { code: 'ENOENT' });
|
||||
try {
|
||||
await expect(create(() => missing)(false, { storageSelection: true })).resolves.toBeNull();
|
||||
await expect(
|
||||
create(() => new Error('malformed'))(false, { storageSelection: true }),
|
||||
).rejects.toThrow('unavailable');
|
||||
await expect(
|
||||
create(() => Object.assign(new Error('unreadable'), { code: 'EACCES' }))(false, {
|
||||
storageSelection: true,
|
||||
}),
|
||||
).rejects.toThrow('unavailable');
|
||||
process.env.CONFIG_PATH = 'explicit.yaml';
|
||||
await expect(create(() => missing)(false, { storageSelection: true })).rejects.toThrow(
|
||||
'unavailable',
|
||||
);
|
||||
} finally {
|
||||
if (original == null) {
|
||||
delete process.env.CONFIG_PATH;
|
||||
} else {
|
||||
process.env.CONFIG_PATH = original;
|
||||
}
|
||||
}
|
||||
});
|
||||
it('refuses invalid persistence even with CONFIG_BYPASS_VALIDATION', async () => {
|
||||
const original = process.env.CONFIG_BYPASS_VALIDATION;
|
||||
process.env.CONFIG_BYPASS_VALIDATION = 'true';
|
||||
try {
|
||||
await expect(
|
||||
create(() => ({ version: '1.3.1', persistence: { favorites: 'postgress' } }))(false),
|
||||
).rejects.toThrow('Invalid persistence');
|
||||
} finally {
|
||||
if (original == null) {
|
||||
delete process.env.CONFIG_BYPASS_VALIDATION;
|
||||
} else {
|
||||
process.env.CONFIG_BYPASS_VALIDATION = original;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
realPostgres('selected PostgreSQL lifecycle and HTTP boundary', () => {
|
||||
const env = { PG_TEST_URL: process.env.PG_TEST_URL };
|
||||
const config = {
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'PG_TEST_URL', acquireTimeoutMs: 1000, queryTimeoutMs: 5000 },
|
||||
};
|
||||
const options = {
|
||||
connectionString: process.env.PG_TEST_URL ?? '',
|
||||
max: 2,
|
||||
maxWaiting: 10,
|
||||
acquireTimeoutMs: 1000,
|
||||
queryTimeoutMs: 5000,
|
||||
idleTimeoutMs: 1000,
|
||||
};
|
||||
const schema = `favorites_lifecycle_${randomBytes(6).toString('hex')}`;
|
||||
const admin = createPostgresDatabase(options);
|
||||
beforeAll(async () => {
|
||||
await admin.query(`CREATE SCHEMA ${schema}`);
|
||||
const url = new URL(options.connectionString);
|
||||
url.searchParams.set('options', `-c search_path=${schema}`);
|
||||
options.connectionString = url.toString();
|
||||
env.PG_TEST_URL = options.connectionString;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
|
||||
await admin.close();
|
||||
});
|
||||
it('refuses schema-only data, releases failed pools, and serves captured HTTP handlers after verification', async () => {
|
||||
const db = createPostgresDatabase(options);
|
||||
try {
|
||||
await migratePostgres(db);
|
||||
await db.query('TRUNCATE tool_favorites, tool_favorite_erasure');
|
||||
await db.query(
|
||||
'UPDATE tool_favorite_readiness SET verified=false, activated=false, checkpoint=NULL, snapshot=NULL',
|
||||
);
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
const failed = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend);
|
||||
await expect(failed.initialize(config)).rejects.toThrow('initialization failed');
|
||||
expect(end).toHaveBeenCalled();
|
||||
await failed.close();
|
||||
await importFavorites(db, async function* () {}, {
|
||||
offline: true,
|
||||
batchSize: 10,
|
||||
verifiedEmpty: true,
|
||||
});
|
||||
const runtime = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend);
|
||||
const handlers = createToolFavoritesHandlers(runtime.methods);
|
||||
await runtime.initialize(config);
|
||||
const uid = '0123456789abcdef01234567';
|
||||
const req = {
|
||||
user: { id: uid },
|
||||
params: { itemType: 'tool', itemId: 'native' },
|
||||
} as ServerRequest;
|
||||
const res = {
|
||||
status: jest.fn().mockReturnThis(),
|
||||
json: jest.fn().mockReturnThis(),
|
||||
} as unknown as Response;
|
||||
try {
|
||||
await handlers.addToolFavorite(req, res);
|
||||
expect(res.status).toHaveBeenCalledWith(200);
|
||||
expect(res.json).toHaveBeenCalledWith({ itemType: 'tool', itemId: 'native' });
|
||||
expect(await runtime.methods.deleteUserToolFavorites(uid)).toBe(1);
|
||||
expect(await runtime.methods.getToolFavorites(uid)).toEqual([]);
|
||||
await db.query('ALTER TABLE tool_favorites RENAME TO unavailable_favorites');
|
||||
await handlers.listToolFavorites(req, res);
|
||||
expect(res.status).toHaveBeenLastCalledWith(500);
|
||||
expect(res.json).toHaveBeenLastCalledWith({ message: 'Failed to retrieve favorites' });
|
||||
await db.query('ALTER TABLE unavailable_favorites RENAME TO tool_favorites');
|
||||
await expect(
|
||||
runtime.methods.addToolFavorite({ userId: uid, itemType: 'tool', itemId: 'late' }),
|
||||
).rejects.toThrow('erasure');
|
||||
} finally {
|
||||
await runtime.close();
|
||||
await expect(runtime.methods.getToolFavorites(uid)).rejects.toThrow('not initialized');
|
||||
}
|
||||
} finally {
|
||||
await db.close();
|
||||
}
|
||||
});
|
||||
it('fails startup on a selected PostgreSQL outage without using Mongo', async () => {
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
const runtime = createFavoriteRuntime(
|
||||
mongo,
|
||||
{
|
||||
PG_TEST_URL: 'postgresql://postgres@127.0.0.1:1/postgres',
|
||||
},
|
||||
createPostgresFavoriteBackend,
|
||||
);
|
||||
await expect(runtime.initialize(config)).rejects.toThrow('initialization failed');
|
||||
expect(end).toHaveBeenCalled();
|
||||
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
|
||||
await runtime.close();
|
||||
});
|
||||
it('does not publish methods when shutdown races initialization', async () => {
|
||||
const runtime = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend);
|
||||
const initializing = runtime.initialize(config);
|
||||
const stopping = runtime.close();
|
||||
await expect(initializing).rejects.toThrow('initialization failed');
|
||||
await stopping;
|
||||
await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized');
|
||||
expect(runtime.close()).toBe(stopping);
|
||||
});
|
||||
});
|
||||
100
packages/api/src/app/persistence.ts
Normal file
100
packages/api/src/app/persistence.ts
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
import { persistenceConfigSchema } from 'librechat-data-provider';
|
||||
import type {
|
||||
ToolFavoriteMethods,
|
||||
PostgresOptions,
|
||||
PostgresFavoriteBackend,
|
||||
} from '@librechat/data-schemas';
|
||||
|
||||
export interface FavoriteRuntime {
|
||||
methods: ToolFavoriteMethods;
|
||||
initialize(config: unknown): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/** Captured route references never change, and no request chooses its storage engine. */
|
||||
export function createFavoriteRuntime(
|
||||
mongo: ToolFavoriteMethods,
|
||||
env: NodeJS.ProcessEnv,
|
||||
createPostgres: (options: PostgresOptions) => PostgresFavoriteBackend,
|
||||
): FavoriteRuntime {
|
||||
let selected: ToolFavoriteMethods | undefined;
|
||||
let backend: PostgresFavoriteBackend | undefined;
|
||||
let backendShutdown: Promise<void> | undefined;
|
||||
const closeBackend = () => (backendShutdown ??= backend?.close() ?? Promise.resolve());
|
||||
let initialization: Promise<void> | undefined;
|
||||
let closed = false;
|
||||
let shutdown: Promise<void> | undefined;
|
||||
function methods(): ToolFavoriteMethods {
|
||||
if (!selected || closed) {
|
||||
throw new Error('Favorites persistence is not initialized');
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
return {
|
||||
methods: {
|
||||
getToolFavorites: async (userId) => methods().getToolFavorites(userId),
|
||||
addToolFavorite: async (params) => methods().addToolFavorite(params),
|
||||
removeToolFavorite: async (params) => methods().removeToolFavorite(params),
|
||||
deleteUserToolFavorites: async (userId, deletionAttemptId) =>
|
||||
deletionAttemptId === undefined
|
||||
? methods().deleteUserToolFavorites(userId)
|
||||
: methods().deleteUserToolFavorites(userId, deletionAttemptId),
|
||||
cancelUserToolFavoriteDeletion: async (userId, deletionAttemptId) =>
|
||||
methods().cancelUserToolFavoriteDeletion?.(userId, deletionAttemptId),
|
||||
},
|
||||
initialize(config) {
|
||||
if (closed) {
|
||||
return Promise.reject(new Error('Favorites persistence is closed'));
|
||||
}
|
||||
initialization ??= (async () => {
|
||||
if (closed) {
|
||||
throw new Error('Favorites persistence is closed');
|
||||
}
|
||||
const parsed = persistenceConfigSchema.safeParse(config);
|
||||
if (!parsed.success) {
|
||||
throw new Error('Invalid persistence configuration');
|
||||
}
|
||||
if (parsed.data.favorites === 'mongo') {
|
||||
selected = mongo;
|
||||
return;
|
||||
}
|
||||
const options = parsed.data.postgres;
|
||||
const connectionString = options && env[options.connectionStringEnv];
|
||||
if (!options || !connectionString) {
|
||||
throw new Error('PostgreSQL connection environment variable is missing');
|
||||
}
|
||||
try {
|
||||
backend = createPostgres({ ...options, connectionString });
|
||||
await backend.initialize();
|
||||
if (closed) {
|
||||
throw new Error('Favorites persistence is closed');
|
||||
}
|
||||
selected = backend.methods;
|
||||
} catch {
|
||||
await closeBackend().catch(() => undefined);
|
||||
throw new Error('PostgreSQL favorites initialization failed');
|
||||
}
|
||||
})();
|
||||
return initialization;
|
||||
},
|
||||
close() {
|
||||
closed = true;
|
||||
shutdown ??= (async () => {
|
||||
await initialization?.catch(() => undefined);
|
||||
await closeBackend();
|
||||
})();
|
||||
return shutdown;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Storage selection is deployment-only, never a tenant config override. */
|
||||
export function getPersistenceConfig(config: unknown): unknown {
|
||||
if (config == null) {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof config !== 'object' || Array.isArray(config)) {
|
||||
throw new Error('Invalid deployment configuration');
|
||||
}
|
||||
return 'persistence' in config ? config.persistence : undefined;
|
||||
}
|
||||
648
packages/api/src/app/postgres.spec.ts
Normal file
648
packages/api/src/app/postgres.spec.ts
Normal file
|
|
@ -0,0 +1,648 @@
|
|||
import pg from 'pg';
|
||||
import mongoose from 'mongoose';
|
||||
import { resolve } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { execFile, spawnSync } from 'node:child_process';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs';
|
||||
import {
|
||||
createModels,
|
||||
createMongoFavoriteStore,
|
||||
createPostgresDatabase,
|
||||
fingerprintFavorites,
|
||||
readPostgresFavorites,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { MongoFavoriteStore } from '@librechat/data-schemas';
|
||||
import { runPostgresCommand } from './postgres';
|
||||
|
||||
describe('PostgreSQL command arguments', () => {
|
||||
const favorites: MongoFavoriteStore = {
|
||||
source: async function* () {},
|
||||
stage: jest.fn(),
|
||||
};
|
||||
const disconnectDb = jest.fn(async () => undefined);
|
||||
|
||||
const commands = ['migrate', 'import', 'verify', 'reconcile-mongo'];
|
||||
const flags = ['--offline', '--dry-run', '--verified-empty'];
|
||||
const combinations = commands.flatMap((command) =>
|
||||
Array.from({ length: 8 }, (_, mask) => ({
|
||||
args: [command, ...flags.filter((_flag, bit) => (mask & (1 << bit)) !== 0)],
|
||||
})),
|
||||
);
|
||||
it.each(combinations)('validates $args before configuration or connections', async ({ args }) => {
|
||||
const [command, ...flags] = args;
|
||||
const unsupported =
|
||||
(command === 'migrate' && flags.length > 0) ||
|
||||
(command === 'verify' && flags.some((flag) => flag !== '--offline')) ||
|
||||
(command === 'reconcile-mongo' && flags.includes('--verified-empty'));
|
||||
const missingOffline = command !== 'migrate' && !flags.includes('--offline');
|
||||
const sentinel = new Error('configuration not loaded');
|
||||
const loadCustomConfig = jest.fn().mockRejectedValue(sentinel);
|
||||
const connectDb = jest.fn();
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
const operation = runPostgresCommand({
|
||||
createDatabase: createPostgresDatabase,
|
||||
favorites,
|
||||
disconnectDb,
|
||||
connectDb,
|
||||
loadCustomConfig,
|
||||
args,
|
||||
env: {},
|
||||
});
|
||||
if (unsupported) await expect(operation).rejects.toThrow('Usage:');
|
||||
else if (missingOffline) await expect(operation).rejects.toThrow('pass --offline');
|
||||
else await expect(operation).rejects.toBe(sentinel);
|
||||
expect(loadCustomConfig).toHaveBeenCalledTimes(unsupported || missingOffline ? 0 : 1);
|
||||
expect(connectDb).not.toHaveBeenCalled();
|
||||
expect(end).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([
|
||||
{ args: [] },
|
||||
{ args: ['unknown'] },
|
||||
{ args: ['__proto__'] },
|
||||
{ args: ['constructor'] },
|
||||
...commands.map((command) => ({ args: [command, '--unknown'] })),
|
||||
...commands.map((command) => ({ args: [command, '--offline', '--offline'] })),
|
||||
])('rejects unknown commands, flags, and duplicates: $args', async ({ args }) => {
|
||||
const loadCustomConfig = jest.fn();
|
||||
const connectDb = jest.fn();
|
||||
await expect(
|
||||
runPostgresCommand({
|
||||
createDatabase: createPostgresDatabase,
|
||||
favorites,
|
||||
disconnectDb,
|
||||
connectDb,
|
||||
loadCustomConfig,
|
||||
args,
|
||||
env: {},
|
||||
}),
|
||||
).rejects.toThrow('Usage:');
|
||||
expect(loadCustomConfig).not.toHaveBeenCalled();
|
||||
expect(connectDb).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
realPostgres('PostgreSQL CLI with real Mongo and PostgreSQL', () => {
|
||||
const schema = `favorites_cli_${randomBytes(6).toString('hex')}`;
|
||||
const baseUrl = process.env.PG_TEST_URL ?? '';
|
||||
const url = new URL(baseUrl || 'postgresql://postgres@localhost/postgres');
|
||||
url.searchParams.set('options', `-c search_path=${schema}`);
|
||||
const env = { PG_TEST_URL: url.toString() };
|
||||
const config = {
|
||||
persistence: { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } },
|
||||
};
|
||||
const options = {
|
||||
connectionString: env.PG_TEST_URL,
|
||||
max: 2,
|
||||
maxWaiting: 10,
|
||||
acquireTimeoutMs: 1000,
|
||||
queryTimeoutMs: 5000,
|
||||
idleTimeoutMs: 1000,
|
||||
};
|
||||
const admin = createPostgresDatabase({ ...options, connectionString: baseUrl });
|
||||
const db = createPostgresDatabase(options);
|
||||
let mongo: MongoMemoryServer;
|
||||
const connectDb = jest.fn(async () =>
|
||||
mongoose.connect(mongo.getUri(), { autoCreate: false, autoIndex: false }),
|
||||
);
|
||||
const disconnectDb = () => mongoose.disconnect();
|
||||
const favorites = createMongoFavoriteStore(mongoose);
|
||||
const loadCustomConfig = jest.fn(async () => config);
|
||||
const run = (args: string[]) =>
|
||||
runPostgresCommand({
|
||||
createDatabase: createPostgresDatabase,
|
||||
favorites,
|
||||
connectDb,
|
||||
disconnectDb,
|
||||
loadCustomConfig,
|
||||
args,
|
||||
env,
|
||||
});
|
||||
const collection = () => createModels(mongoose).ToolFavorite.collection;
|
||||
const reconnect = () => mongoose.connect(mongo.getUri());
|
||||
const uid = new mongoose.Types.ObjectId('0123456789abcdef01234567');
|
||||
const row = (index: number) => ({
|
||||
_id: new mongoose.Types.ObjectId(index.toString(16).padStart(24, '0')),
|
||||
user: uid,
|
||||
tenantId: index % 2 ? 'tenant-one' : null,
|
||||
itemType: 'tool',
|
||||
itemId: `tool-${index}`,
|
||||
createdAt: new Date('2025-01-01T00:00:00.000Z'),
|
||||
updatedAt: new Date('2025-01-02T00:00:00.000Z'),
|
||||
});
|
||||
beforeAll(async () => {
|
||||
await admin.query(`CREATE SCHEMA ${schema}`);
|
||||
mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks();
|
||||
await run(['migrate']);
|
||||
await db.query('TRUNCATE tool_favorites, tool_favorite_erasure');
|
||||
await db.query(
|
||||
'UPDATE tool_favorite_readiness SET verified=false, activated=false, checkpoint=NULL, snapshot=NULL, imported=0',
|
||||
);
|
||||
await reconnect();
|
||||
await collection().deleteMany({});
|
||||
await collection().createIndex({ user: 1, itemType: 1, itemId: 1 }, { unique: true });
|
||||
await collection().createIndex({ tenantId: 1 });
|
||||
await mongoose.disconnect();
|
||||
connectDb.mockClear();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongo?.stop();
|
||||
await db.close();
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
|
||||
await admin.close();
|
||||
});
|
||||
it('imports and reconciles through plain snapshot, staging, and lifecycle interfaces', async () => {
|
||||
const snapshot = {
|
||||
id: row(1)._id.toHexString(),
|
||||
userId: uid.toHexString(),
|
||||
tenantId: 'tenant-one',
|
||||
itemType: 'tool',
|
||||
itemId: 'portable-store',
|
||||
createdAt: row(1).createdAt,
|
||||
updatedAt: row(1).updatedAt,
|
||||
} as const;
|
||||
const source = async function* () {
|
||||
yield snapshot;
|
||||
};
|
||||
const insert = jest.fn(async () => undefined);
|
||||
const replace = jest.fn(async () => undefined);
|
||||
const discard = jest.fn(async () => undefined);
|
||||
const store: MongoFavoriteStore = {
|
||||
source,
|
||||
stage: jest.fn(async () => ({ source, insert, replace, discard })),
|
||||
};
|
||||
const connect = jest.fn(async () => undefined);
|
||||
const disconnect = jest.fn(async () => undefined);
|
||||
const createDatabase = jest.fn(createPostgresDatabase);
|
||||
for (const command of ['import', 'verify', 'reconcile-mongo']) {
|
||||
await runPostgresCommand({
|
||||
createDatabase,
|
||||
favorites: store,
|
||||
connectDb: connect,
|
||||
disconnectDb: disconnect,
|
||||
loadCustomConfig,
|
||||
args: [command, '--offline'],
|
||||
env,
|
||||
});
|
||||
}
|
||||
expect(createDatabase).toHaveBeenCalledTimes(3);
|
||||
expect(createDatabase).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ connectionString: env.PG_TEST_URL }),
|
||||
);
|
||||
expect(connect.mock.calls).toEqual([[], [], []]);
|
||||
expect(disconnect).toHaveBeenCalledTimes(3);
|
||||
expect(store.stage).toHaveBeenCalledTimes(1);
|
||||
expect(insert).toHaveBeenCalledWith([snapshot]);
|
||||
expect(replace).toHaveBeenCalledTimes(1);
|
||||
expect(discard).not.toHaveBeenCalled();
|
||||
expect(connectDb).not.toHaveBeenCalled();
|
||||
});
|
||||
it('runs the actual migrate entrypoint with MONGO_URI unset', () => {
|
||||
const scratchRoot = resolve(__dirname, '../../../../node_modules/.cache/pg-lifecycle');
|
||||
mkdirSync(scratchRoot, { recursive: true });
|
||||
const scratch = mkdtempSync(resolve(scratchRoot, 'cli-'));
|
||||
try {
|
||||
const configPath = resolve(scratch, 'config.yaml');
|
||||
writeFileSync(
|
||||
configPath,
|
||||
'version: 1.3.1\npersistence:\n favorites: postgres\n postgres:\n connectionStringEnv: PG_TEST_URL\n',
|
||||
);
|
||||
const script = resolve(__dirname, '../../../../config/postgres.js');
|
||||
const child = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
`delete process.env.MONGO_URI; process.argv = ['node', ${JSON.stringify(script)}, 'migrate']; require(${JSON.stringify(script)});`,
|
||||
],
|
||||
{
|
||||
cwd: scratch,
|
||||
env: { ...process.env, ...env, CONFIG_PATH: configPath },
|
||||
encoding: 'utf8',
|
||||
timeout: 15000,
|
||||
killSignal: 'SIGKILL',
|
||||
},
|
||||
);
|
||||
expect({ status: child.status, error: child.error?.message, stderr: child.stderr }).toEqual({
|
||||
status: 0,
|
||||
error: undefined,
|
||||
stderr: '',
|
||||
});
|
||||
expect(child.stdout).toContain('PostgreSQL schema migrated');
|
||||
} finally {
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
it.each([
|
||||
{ args: ['import', '--offline', '--dry-run'], seeded: true },
|
||||
{ args: ['import', '--offline', '--dry-run', '--verified-empty'], seeded: false },
|
||||
{ args: ['reconcile-mongo', '--offline', '--dry-run'], seeded: true },
|
||||
{ args: ['reconcile-mongo', '--offline', '--dry-run'], seeded: false },
|
||||
{ args: ['import', '--offline'], seeded: true },
|
||||
{ args: ['verify', '--offline'], seeded: true },
|
||||
{ args: ['reconcile-mongo', '--offline'], seeded: true },
|
||||
])(
|
||||
'isolates actual CLI $args from Mongo schema initialization (seeded=$seeded)',
|
||||
async ({ args, seeded }) => {
|
||||
const native = await mongoose
|
||||
.createConnection(mongo.getUri(`cli_${randomBytes(6).toString('hex')}`), {
|
||||
autoCreate: false,
|
||||
autoIndex: false,
|
||||
})
|
||||
.asPromise();
|
||||
const scratchRoot = resolve(__dirname, '../../../../node_modules/.cache/pg-lifecycle');
|
||||
mkdirSync(scratchRoot, { recursive: true });
|
||||
const scratch = mkdtempSync(resolve(scratchRoot, 'cli-'));
|
||||
const favorites = native.collection(collection().name);
|
||||
const mongoSnapshot = async () => {
|
||||
const collections = await native.db!.listCollections().toArray();
|
||||
return Promise.all(
|
||||
collections
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map(async (metadata) => ({
|
||||
metadata,
|
||||
indexes: await native.collection(metadata.name).indexes(),
|
||||
rows: await native.collection(metadata.name).find().sort({ _id: 1 }).toArray(),
|
||||
})),
|
||||
);
|
||||
};
|
||||
const postgresSnapshot = async () => {
|
||||
const tables = await db.query<{ tablename: string }>(
|
||||
'SELECT tablename FROM pg_tables WHERE schemaname=current_schema() ORDER BY tablename',
|
||||
);
|
||||
return {
|
||||
columns: (
|
||||
await db.query(
|
||||
'SELECT * FROM information_schema.columns WHERE table_schema=current_schema() ORDER BY table_name,ordinal_position',
|
||||
)
|
||||
).rows,
|
||||
indexes: (
|
||||
await db.query(
|
||||
'SELECT * FROM pg_indexes WHERE schemaname=current_schema() ORDER BY tablename,indexname',
|
||||
)
|
||||
).rows,
|
||||
tables: await Promise.all(
|
||||
tables.rows.map(async ({ tablename }) => ({
|
||||
tablename,
|
||||
rows: (await db.query(`SELECT * FROM "${tablename}" ORDER BY 1`)).rows,
|
||||
})),
|
||||
),
|
||||
};
|
||||
};
|
||||
try {
|
||||
if (seeded) await favorites.insertOne(row(1));
|
||||
if (args[0] !== 'import') {
|
||||
await db.query(
|
||||
'INSERT INTO tool_favorites(id,user_id,tenant_id,item_type,item_id,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7)',
|
||||
[
|
||||
row(1)._id.toHexString(),
|
||||
uid.toHexString(),
|
||||
row(1).tenantId,
|
||||
'tool',
|
||||
row(1).itemId,
|
||||
row(1).createdAt,
|
||||
row(1).updatedAt,
|
||||
],
|
||||
);
|
||||
const snapshot = await fingerprintFavorites(readPostgresFavorites(db, 250));
|
||||
await db.query(
|
||||
'UPDATE tool_favorite_readiness SET verified=true,activated=true,snapshot=$1',
|
||||
[snapshot.digest],
|
||||
);
|
||||
}
|
||||
const beforeMongo = await mongoSnapshot();
|
||||
const beforePostgres = await postgresSnapshot();
|
||||
const configPath = resolve(scratch, 'config.yaml');
|
||||
writeFileSync(
|
||||
configPath,
|
||||
'version: 1.3.1\npersistence:\n favorites: postgres\n postgres:\n connectionStringEnv: PG_TEST_URL\n',
|
||||
);
|
||||
const guardPath = resolve(scratch, 'guard.cjs');
|
||||
writeFileSync(
|
||||
guardPath,
|
||||
`
|
||||
const assert = require('node:assert/strict');
|
||||
const mongoose = require(${JSON.stringify(require.resolve('mongoose'))});
|
||||
const connect = mongoose.connect.bind(mongoose);
|
||||
mongoose.connect = (uri, options) => {
|
||||
assert.equal(options.autoCreate, false);
|
||||
assert.equal(options.autoIndex, false);
|
||||
return connect(uri, options);
|
||||
};
|
||||
mongoose.model = () => { throw new Error('CLI registered a connected model'); };
|
||||
process.on('beforeExit', () => {
|
||||
assert.deepEqual(mongoose.modelNames(), []);
|
||||
assert.equal(Object.keys(require.cache).some(path => path.endsWith('/api/db/index.js')), false);
|
||||
});
|
||||
`,
|
||||
);
|
||||
const child = await promisify(execFile)(
|
||||
process.execPath,
|
||||
['--require', guardPath, resolve(__dirname, '../../../../config/postgres.js'), ...args],
|
||||
{
|
||||
cwd: scratch,
|
||||
env: {
|
||||
...process.env,
|
||||
...env,
|
||||
CONFIG_PATH: configPath,
|
||||
MONGO_URI: mongo.getUri(native.name),
|
||||
MONGO_AUTO_CREATE: 'true',
|
||||
MONGO_AUTO_INDEX: 'true',
|
||||
},
|
||||
encoding: 'utf8',
|
||||
timeout: 15000,
|
||||
killSignal: 'SIGKILL',
|
||||
},
|
||||
);
|
||||
expect(child.stderr).toBe('');
|
||||
const afterMongo = await mongoSnapshot();
|
||||
if (args[0] === 'reconcile-mongo' && !args.includes('--dry-run')) {
|
||||
expect(afterMongo).toHaveLength(1);
|
||||
const before = beforeMongo[0].metadata;
|
||||
const after = afterMongo[0].metadata;
|
||||
if (!('info' in before) || !before.info || !('info' in after) || !after.info) {
|
||||
throw new Error('Full Mongo collection metadata is required');
|
||||
}
|
||||
expect(after.info.uuid).not.toEqual(before.info.uuid);
|
||||
before.info.uuid = after.info.uuid;
|
||||
}
|
||||
expect(afterMongo).toEqual(beforeMongo);
|
||||
if (args[0] !== 'import' || args.includes('--dry-run')) {
|
||||
expect(await postgresSnapshot()).toEqual(beforePostgres);
|
||||
} else {
|
||||
expect((await db.query('SELECT count(*) FROM tool_favorites')).rows[0].count).toBe('1');
|
||||
}
|
||||
} finally {
|
||||
await native.dropDatabase();
|
||||
await native.close();
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
it('rejects a migration dry run before creating any schema objects or opening connections', async () => {
|
||||
const emptySchema = `favorites_dry_run_${randomBytes(6).toString('hex')}`;
|
||||
await admin.query(`CREATE SCHEMA ${emptySchema}`);
|
||||
const target = new URL(baseUrl);
|
||||
target.searchParams.set('options', `-c search_path=${emptySchema}`);
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
loadCustomConfig.mockClear();
|
||||
try {
|
||||
const result = await runPostgresCommand({
|
||||
createDatabase: createPostgresDatabase,
|
||||
favorites,
|
||||
disconnectDb,
|
||||
connectDb,
|
||||
loadCustomConfig,
|
||||
args: ['migrate', '--dry-run'],
|
||||
env: { PG_TEST_URL: target.toString() },
|
||||
}).then(
|
||||
() => 'success',
|
||||
(error: Error) => error.message,
|
||||
);
|
||||
const objects = await admin.query('SELECT tablename FROM pg_tables WHERE schemaname=$1', [
|
||||
emptySchema,
|
||||
]);
|
||||
expect(objects.rows).toEqual([]);
|
||||
expect(result).toContain('Usage:');
|
||||
expect(loadCustomConfig).not.toHaveBeenCalled();
|
||||
expect(connectDb).not.toHaveBeenCalled();
|
||||
expect(end).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await admin.query(`DROP SCHEMA ${emptySchema} CASCADE`);
|
||||
}
|
||||
});
|
||||
it('migrates without connecting Mongo and releases its pool', async () => {
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
await run(['migrate']);
|
||||
expect(connectDb).not.toHaveBeenCalled();
|
||||
expect(end).toHaveBeenCalledTimes(1);
|
||||
expect(loadCustomConfig).toHaveBeenCalledWith(false, {
|
||||
mode: 'reload',
|
||||
storageSelection: true,
|
||||
});
|
||||
});
|
||||
it.each(['import', 'verify', 'reconcile-mongo'])(
|
||||
'requires offline for %s before opening connections',
|
||||
async (command) => {
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
const disconnect = jest.spyOn(mongoose, 'disconnect');
|
||||
await expect(run([command])).rejects.toThrow('--offline');
|
||||
expect(connectDb).not.toHaveBeenCalled();
|
||||
expect(end).not.toHaveBeenCalled();
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
it('validates an import dry run without changing data or readiness', async () => {
|
||||
await reconnect();
|
||||
await collection().insertMany([row(1), row(2)]);
|
||||
const original = await collection().find().sort({ _id: 1 }).toArray();
|
||||
await mongoose.disconnect();
|
||||
const state = await db.query('SELECT * FROM tool_favorite_readiness');
|
||||
await run(['import', '--offline', '--dry-run']);
|
||||
expect((await db.query('SELECT * FROM tool_favorites')).rows).toEqual([]);
|
||||
expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(state.rows);
|
||||
await reconnect();
|
||||
expect(await collection().find().sort({ _id: 1 }).toArray()).toEqual(original);
|
||||
});
|
||||
it('imports and verifies identity, tenant and dates, then reconciles deletes and preserves indexes without changing authority', async () => {
|
||||
await reconnect();
|
||||
await collection().insertMany([row(1), row(2)]);
|
||||
const indexes = await collection().indexes();
|
||||
await mongoose.disconnect();
|
||||
await run(['import', '--offline']);
|
||||
await run(['verify', '--offline']);
|
||||
const expected = await fingerprintFavorites(readPostgresFavorites(db, 250));
|
||||
await db.query('DELETE FROM tool_favorites WHERE id=$1', [row(1)._id.toHexString()]);
|
||||
const state = await db.query('SELECT * FROM tool_favorite_readiness');
|
||||
await expect(run(['verify', '--offline'])).rejects.toThrow('differ');
|
||||
await run(['reconcile-mongo', '--offline', '--dry-run']);
|
||||
await reconnect();
|
||||
expect(await collection().countDocuments()).toBe(2);
|
||||
await mongoose.disconnect();
|
||||
await run(['reconcile-mongo', '--offline']);
|
||||
await run(['verify', '--offline']);
|
||||
await reconnect();
|
||||
expect(await collection().findOne({ _id: row(1)._id })).toBeNull();
|
||||
expect(await collection().findOne({ _id: row(2)._id })).toEqual(row(2));
|
||||
expect(await collection().indexes()).toEqual(indexes);
|
||||
expect(
|
||||
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
|
||||
).toEqual([]);
|
||||
expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(state.rows);
|
||||
expect(expected.count).toBe(2);
|
||||
});
|
||||
it.each(['schema-only', 'partial', 'missing-marker'] as const)(
|
||||
'refuses destructive reconciliation from %s PostgreSQL without modifying Mongo',
|
||||
async (state) => {
|
||||
await reconnect();
|
||||
await collection().insertMany([row(1), row(2)]);
|
||||
const original = await collection().find().sort({ _id: 1 }).toArray();
|
||||
const indexes = await collection().indexes();
|
||||
await mongoose.disconnect();
|
||||
if (state !== 'schema-only') {
|
||||
await run(['import', '--offline']);
|
||||
await db.query('DELETE FROM tool_favorites WHERE id=$1', [row(2)._id.toHexString()]);
|
||||
await db.query('UPDATE tool_favorite_readiness SET verified=false,checkpoint=$1', [
|
||||
row(1)._id.toHexString(),
|
||||
]);
|
||||
}
|
||||
if (state === 'missing-marker') await db.query('DELETE FROM tool_favorite_readiness');
|
||||
const authority = await db.query('SELECT * FROM tool_favorite_readiness');
|
||||
for (const flags of [[], ['--dry-run']]) {
|
||||
await expect(run(['reconcile-mongo', '--offline', ...flags])).rejects.toThrow(
|
||||
'not verified',
|
||||
);
|
||||
await reconnect();
|
||||
expect(await collection().find().sort({ _id: 1 }).toArray()).toEqual(original);
|
||||
expect(await collection().indexes()).toEqual(indexes);
|
||||
expect(
|
||||
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
|
||||
).toEqual([]);
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(
|
||||
authority.rows,
|
||||
);
|
||||
if (state === 'missing-marker')
|
||||
await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)');
|
||||
},
|
||||
);
|
||||
it('allows verified activated PostgreSQL to reconcile subsequent deletions, including an empty snapshot', async () => {
|
||||
await reconnect();
|
||||
await collection().insertOne(row(1));
|
||||
await mongoose.disconnect();
|
||||
await run(['import', '--offline']);
|
||||
await db.query('UPDATE tool_favorite_readiness SET activated=true');
|
||||
await db.query('DELETE FROM tool_favorites');
|
||||
await run(['reconcile-mongo', '--offline']);
|
||||
await reconnect();
|
||||
expect(await collection().countDocuments()).toBe(0);
|
||||
expect((await db.query('SELECT verified,activated FROM tool_favorite_readiness')).rows).toEqual(
|
||||
[{ verified: true, activated: true }],
|
||||
);
|
||||
});
|
||||
it('refuses a same-count corrupted staging snapshot, retains Mongo, and removes staging', async () => {
|
||||
await reconnect();
|
||||
await collection().insertOne(row(1));
|
||||
await mongoose.disconnect();
|
||||
await run(['import', '--offline']);
|
||||
const original = mongoose.connection.collection.bind(mongoose.connection);
|
||||
const staging = jest
|
||||
.spyOn(mongoose.connection, 'collection')
|
||||
.mockImplementation((name, options) => {
|
||||
const target = original(name, options);
|
||||
if (name.includes('_reconcile_')) {
|
||||
const insert = target.insertMany.bind(target);
|
||||
jest.spyOn(target, 'insertMany').mockImplementation(async (docs, options) => {
|
||||
const result = await insert(docs, options);
|
||||
await target.updateOne({}, { $set: { itemId: 'corrupt' } });
|
||||
return result;
|
||||
});
|
||||
}
|
||||
return target;
|
||||
});
|
||||
await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow(
|
||||
'changed during reconciliation',
|
||||
);
|
||||
staging.mockRestore();
|
||||
await reconnect();
|
||||
expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1));
|
||||
expect(
|
||||
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
|
||||
).toEqual([]);
|
||||
});
|
||||
it('rechecks PostgreSQL readiness before Mongo replacement and discards untrusted staging', async () => {
|
||||
await reconnect();
|
||||
await collection().insertOne(row(1));
|
||||
await mongoose.disconnect();
|
||||
await run(['import', '--offline']);
|
||||
const original = mongoose.connection.collection.bind(mongoose.connection);
|
||||
jest.spyOn(mongoose.connection, 'collection').mockImplementation((name, options) => {
|
||||
const target = original(name, options);
|
||||
if (name.includes('_reconcile_')) {
|
||||
const insert = target.insertMany.bind(target);
|
||||
jest.spyOn(target, 'insertMany').mockImplementation(async (docs, options) => {
|
||||
const result = await insert(docs, options);
|
||||
await db.query('UPDATE tool_favorite_readiness SET verified=false');
|
||||
return result;
|
||||
});
|
||||
}
|
||||
return target;
|
||||
});
|
||||
await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow('not verified');
|
||||
jest.restoreAllMocks();
|
||||
await reconnect();
|
||||
expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1));
|
||||
expect(
|
||||
await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(),
|
||||
).toEqual([]);
|
||||
});
|
||||
it('does not race a migration or import holding the same schema authority', async () => {
|
||||
await reconnect();
|
||||
await collection().insertOne(row(1));
|
||||
await mongoose.disconnect();
|
||||
await run(['import', '--offline']);
|
||||
await db.session(async (session) => {
|
||||
await session.query('SELECT pg_advisory_lock(17490261,hashtext(current_schema()))');
|
||||
try {
|
||||
await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow(
|
||||
'Another favorite migration',
|
||||
);
|
||||
} finally {
|
||||
await session.query('SELECT pg_advisory_unlock(17490261,hashtext(current_schema()))');
|
||||
}
|
||||
});
|
||||
await reconnect();
|
||||
expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1));
|
||||
});
|
||||
it('reports cleanup failure after a successful command instead of silently succeeding', async () => {
|
||||
const original = pg.Pool.prototype.end;
|
||||
jest.spyOn(pg.Pool.prototype, 'end').mockImplementationOnce(async function (this: pg.Pool) {
|
||||
await original.bind(this)();
|
||||
throw new Error('cleanup failed');
|
||||
});
|
||||
const disconnect = jest.spyOn(mongoose, 'disconnect');
|
||||
await expect(run(['migrate'])).rejects.toThrow('command cleanup failed');
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it.each([{ user: null }, { tenantId: 42 }, { createdAt: null }, { updatedAt: 'invalid' }])(
|
||||
'closes pools and Mongo after malformed source validation without exposing the record: %j',
|
||||
async (invalid) => {
|
||||
await reconnect();
|
||||
await collection().insertOne({ ...row(1), ...invalid });
|
||||
await mongoose.disconnect();
|
||||
const end = jest.spyOn(pg.Pool.prototype, 'end');
|
||||
const disconnect = jest.spyOn(mongoose, 'disconnect');
|
||||
await expect(run(['import', '--offline'])).rejects.toThrow(
|
||||
'Invalid favorite snapshot record',
|
||||
);
|
||||
expect(end).toHaveBeenCalledTimes(1);
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
expect((await db.query('SELECT count(*) FROM tool_favorites')).rows[0].count).toBe('0');
|
||||
},
|
||||
);
|
||||
it('keeps the operational error when pool shutdown also fails, and still disconnects Mongo', async () => {
|
||||
await reconnect();
|
||||
await collection().insertOne({ ...row(1), user: null });
|
||||
await mongoose.disconnect();
|
||||
const original = pg.Pool.prototype.end;
|
||||
jest.spyOn(pg.Pool.prototype, 'end').mockImplementationOnce(async function (this: pg.Pool) {
|
||||
await original.bind(this)();
|
||||
throw new Error('cleanup failed');
|
||||
});
|
||||
const disconnect = jest.spyOn(mongoose, 'disconnect');
|
||||
await expect(run(['import', '--offline'])).rejects.toThrow('Invalid favorite snapshot record');
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('disconnects Mongo even when configuration loading fails before a pool exists', async () => {
|
||||
loadCustomConfig.mockRejectedValueOnce(new Error('Deployment config unavailable'));
|
||||
const disconnect = jest.spyOn(mongoose, 'disconnect');
|
||||
await expect(run(['migrate'])).rejects.toThrow('Deployment config unavailable');
|
||||
expect(disconnect).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
164
packages/api/src/app/postgres.ts
Normal file
164
packages/api/src/app/postgres.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import { persistenceConfigSchema } from 'librechat-data-provider';
|
||||
import {
|
||||
migratePostgres,
|
||||
importFavorites,
|
||||
fingerprintFavorites,
|
||||
readPostgresFavorites,
|
||||
assertPostgresDataReady,
|
||||
} from '@librechat/data-schemas';
|
||||
import type {
|
||||
FavoriteSnapshotRow,
|
||||
MongoFavoriteStore,
|
||||
PostgresDatabase,
|
||||
PostgresOptions,
|
||||
} from '@librechat/data-schemas';
|
||||
import { getPersistenceConfig } from './persistence';
|
||||
|
||||
export async function runPostgresCommand(deps: {
|
||||
favorites: MongoFavoriteStore;
|
||||
createDatabase(options: PostgresOptions): PostgresDatabase;
|
||||
connectDb(): Promise<unknown>;
|
||||
disconnectDb(): Promise<void>;
|
||||
loadCustomConfig(
|
||||
print: boolean,
|
||||
options: { mode: 'reload'; storageSelection: true },
|
||||
): Promise<unknown>;
|
||||
args: string[];
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): Promise<void> {
|
||||
let db: PostgresDatabase | undefined;
|
||||
const close = () => Promise.allSettled([db?.close(), deps.disconnectDb()]);
|
||||
const execute = async () => {
|
||||
const [command, ...flags] = deps.args;
|
||||
const commands: Record<string, string[]> = {
|
||||
migrate: [],
|
||||
import: ['--offline', '--dry-run', '--verified-empty'],
|
||||
verify: ['--offline'],
|
||||
'reconcile-mongo': ['--offline', '--dry-run'],
|
||||
};
|
||||
const allowed = Object.prototype.hasOwnProperty.call(commands, command)
|
||||
? commands[command]
|
||||
: undefined;
|
||||
if (
|
||||
!allowed ||
|
||||
flags.some((flag) => !allowed.includes(flag)) ||
|
||||
new Set(flags).size !== flags.length
|
||||
) {
|
||||
throw new Error(
|
||||
'Usage: postgres migrate | import --offline [--dry-run] [--verified-empty] | verify --offline | reconcile-mongo --offline [--dry-run]',
|
||||
);
|
||||
}
|
||||
if (command !== 'migrate' && !flags.includes('--offline')) {
|
||||
throw new Error('Stop all app, worker, and migration writers and pass --offline');
|
||||
}
|
||||
const config = persistenceConfigSchema.parse(
|
||||
getPersistenceConfig(
|
||||
await deps.loadCustomConfig(false, { mode: 'reload', storageSelection: true }),
|
||||
),
|
||||
);
|
||||
const options = config.postgres;
|
||||
const connectionString = options && deps.env[options.connectionStringEnv];
|
||||
if (!options || !connectionString) {
|
||||
throw new Error('Missing PostgreSQL connection environment variable');
|
||||
}
|
||||
db = deps.createDatabase({ ...options, connectionString });
|
||||
if (command === 'migrate') {
|
||||
await migratePostgres(db);
|
||||
console.info('PostgreSQL schema migrated. Favorites data is not yet verified.');
|
||||
return;
|
||||
}
|
||||
await deps.connectDb();
|
||||
const mongo = deps.favorites;
|
||||
const source = mongo.source;
|
||||
if (command === 'import') {
|
||||
const result = await importFavorites(db, source, {
|
||||
offline: true,
|
||||
batchSize: 250,
|
||||
dryRun: flags.includes('--dry-run'),
|
||||
verifiedEmpty: flags.includes('--verified-empty'),
|
||||
});
|
||||
console.info(
|
||||
`Favorites snapshot ${flags.includes('--dry-run') ? 'validated' : 'imported and verified'}: ${result.count} rows`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (command === 'verify') {
|
||||
const postgres = await fingerprintFavorites(readPostgresFavorites(db, 250));
|
||||
const mongo = await fingerprintFavorites(source);
|
||||
if (postgres.digest !== mongo.digest || postgres.count !== mongo.count) {
|
||||
throw new Error('Favorite snapshots differ');
|
||||
}
|
||||
console.info(`Favorites snapshots match: ${postgres.count} rows`);
|
||||
return;
|
||||
}
|
||||
await db.session(async (session) => {
|
||||
const lock = await session.query<{ acquired: boolean }>(
|
||||
'SELECT pg_try_advisory_lock(17490261, hashtext(current_schema())) AS acquired',
|
||||
);
|
||||
if (!lock.rows[0].acquired)
|
||||
throw new Error('Another favorite migration or import is running');
|
||||
try {
|
||||
await assertPostgresDataReady(session);
|
||||
const pgSource = readPostgresFavorites(session, 250);
|
||||
const postgres = await fingerprintFavorites(pgSource);
|
||||
// Build and validate a replacement before an atomic rename. Deletes are reconciled too.
|
||||
if (flags.includes('--dry-run')) {
|
||||
console.info(
|
||||
`Offline reconciliation would replace Mongo favorites with ${postgres.count} PostgreSQL rows`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const staging = await mongo.stage();
|
||||
let renamed = false;
|
||||
try {
|
||||
let batch: FavoriteSnapshotRow[] = [];
|
||||
const write = async () => {
|
||||
await staging.insert(batch);
|
||||
batch = [];
|
||||
};
|
||||
for await (const row of pgSource()) {
|
||||
batch.push(row);
|
||||
if (batch.length === 250) {
|
||||
await write();
|
||||
}
|
||||
}
|
||||
if (batch.length) {
|
||||
await write();
|
||||
}
|
||||
const staged = await fingerprintFavorites(staging.source);
|
||||
const current = await fingerprintFavorites(pgSource);
|
||||
if (
|
||||
current.digest !== postgres.digest ||
|
||||
current.count !== postgres.count ||
|
||||
staged.digest !== postgres.digest ||
|
||||
staged.count !== postgres.count
|
||||
) {
|
||||
throw new Error('PostgreSQL changed during reconciliation');
|
||||
}
|
||||
await assertPostgresDataReady(session);
|
||||
await staging.replace();
|
||||
renamed = true;
|
||||
console.info(
|
||||
`Mongo favorites reconciled: ${postgres.count} rows. Storage selection was not changed.`,
|
||||
);
|
||||
} finally {
|
||||
if (!renamed) {
|
||||
await staging.discard();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await session.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))');
|
||||
}
|
||||
});
|
||||
};
|
||||
try {
|
||||
await execute();
|
||||
} catch (error) {
|
||||
await close();
|
||||
throw error;
|
||||
}
|
||||
const cleanup = await close();
|
||||
if (cleanup.some((result) => result.status === 'rejected')) {
|
||||
throw new Error('PostgreSQL command cleanup failed');
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +1,35 @@
|
|||
import { Keyv } from 'keyv';
|
||||
import mongoose from 'mongoose';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { ResourceType, PrincipalType } from 'librechat-data-provider';
|
||||
import {
|
||||
createPostgresFavoriteBackend,
|
||||
createPostgresDatabase,
|
||||
migratePostgres,
|
||||
importFavorites,
|
||||
createModels,
|
||||
createMethods,
|
||||
tenantStorage,
|
||||
createSkillDeletionPersistenceAdapter,
|
||||
logger,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { PostgresDatabase } from '@librechat/data-schemas';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { CheckpointDeletion } from '~/agents/checkpoints/deletion';
|
||||
import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure';
|
||||
import type { MCPOAuthTokens } from '~/mcp/oauth/types';
|
||||
import {
|
||||
deleteUserMCPServers,
|
||||
eraseUserData,
|
||||
cancelUserDataDeletion,
|
||||
createUserDeletionAttempt,
|
||||
} from './erasure';
|
||||
import { MCPOAuthHandler, MCPTokenStorage, getMCPOAuthLeaseId } from '~/mcp/oauth';
|
||||
import { deleteAllSharedLinksWithCleanup } from '~/shared-links/service';
|
||||
import { createSkillDeletionService } from '../skills/deletionService';
|
||||
import { deleteUserMCPServers, eraseUserData } from './erasure';
|
||||
import { cleanupMCPServerOAuth } from '~/mcp/oauth/cleanup';
|
||||
import { createFavoriteRuntime } from '~/app/persistence';
|
||||
import { FlowStateManager } from '~/flow/manager';
|
||||
|
||||
/** Stored OAuth credentials are encrypted with a key read when the module loads. */
|
||||
|
|
@ -903,3 +915,332 @@ describe('removing a shared MCP server with its owner', () => {
|
|||
expect(await count('MCPServer', { _id: mcpServerId })).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
realPostgres('attempt-scoped PostgreSQL account erasure', () => {
|
||||
const schema = `erasure_${randomUUID().replace(/-/g, '')}`;
|
||||
const config = { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } };
|
||||
const options = {
|
||||
connectionString: process.env.PG_TEST_URL ?? '',
|
||||
max: 2,
|
||||
maxWaiting: 10,
|
||||
acquireTimeoutMs: 1000,
|
||||
queryTimeoutMs: 5000,
|
||||
idleTimeoutMs: 1000,
|
||||
};
|
||||
let admin: PostgresDatabase;
|
||||
let db: PostgresDatabase;
|
||||
beforeAll(async () => {
|
||||
admin = createPostgresDatabase(options);
|
||||
await admin.query(`CREATE SCHEMA ${schema}`);
|
||||
const url = new URL(options.connectionString);
|
||||
url.searchParams.set('options', `-c search_path=${schema}`);
|
||||
options.connectionString = url.toString();
|
||||
db = createPostgresDatabase(options);
|
||||
await migratePostgres(db);
|
||||
await importFavorites(db, async function* () {}, {
|
||||
offline: true,
|
||||
verifiedEmpty: true,
|
||||
batchSize: 10,
|
||||
});
|
||||
});
|
||||
afterAll(async () => {
|
||||
await db.close();
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
|
||||
await admin.close();
|
||||
});
|
||||
async function setup() {
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
const startedAt = new Date();
|
||||
const deletionAttemptId = randomUUID();
|
||||
await model('User').insertOne({
|
||||
_id: new mongoose.Types.ObjectId(userId),
|
||||
agentTriggerDeletionStartedAt: startedAt,
|
||||
agentTriggerDeletionAttemptId: deletionAttemptId,
|
||||
});
|
||||
const runtime = createFavoriteRuntime(
|
||||
methods,
|
||||
{ PG_TEST_URL: options.connectionString },
|
||||
createPostgresFavoriteBackend,
|
||||
);
|
||||
await runtime.initialize(config);
|
||||
const { deps } = createDeps();
|
||||
const erasure = { ...deps, ...runtime.methods };
|
||||
const cancel = { ...methods, ...runtime.methods };
|
||||
const params = { userId, startedAt, deletionAttemptId };
|
||||
const favorite = { userId, itemType: 'tool', itemId: 'saved' } as const;
|
||||
await runtime.methods.addToolFavorite(favorite);
|
||||
return { runtime, erasure, cancel, params, favorite };
|
||||
}
|
||||
it('unfences a retained account after a later cascade failure, including after restart', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
erasure.deleteSharedLinks = jest.fn().mockRejectedValue(new Error('shared links unavailable'));
|
||||
try {
|
||||
await expect(eraseUserData(params, erasure)).rejects.toThrow('shared links unavailable');
|
||||
expect(await count('User', { _id: new mongoose.Types.ObjectId(params.userId) })).toBe(1);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
await cancelUserDataDeletion(params, cancel);
|
||||
await runtime.close();
|
||||
const restarted = createFavoriteRuntime(
|
||||
methods,
|
||||
{ PG_TEST_URL: options.connectionString },
|
||||
createPostgresFavoriteBackend,
|
||||
);
|
||||
try {
|
||||
await restarted.initialize(config);
|
||||
await expect(restarted.methods.addToolFavorite(favorite)).resolves.toEqual({
|
||||
ok: true,
|
||||
added: true,
|
||||
});
|
||||
expect(await restarted.methods.getToolFavorites(params.userId)).toEqual([
|
||||
{ itemType: 'tool', itemId: 'saved' },
|
||||
]);
|
||||
} finally {
|
||||
await restarted.close();
|
||||
}
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it.each(['deleted', 'other-attempt', 'mongo-error'] as const)(
|
||||
'keeps the marker when cancellation cannot prove ownership: %s',
|
||||
async (reason) => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
if (reason === 'deleted') await methods.deleteUserById(params.userId);
|
||||
if (reason === 'other-attempt')
|
||||
await model('User').updateOne(
|
||||
{ _id: new mongoose.Types.ObjectId(params.userId) },
|
||||
{ $set: { agentTriggerDeletionStartedAt: new Date(params.startedAt.getTime() + 1) } },
|
||||
);
|
||||
if (reason === 'mongo-error')
|
||||
jest
|
||||
.spyOn(mongoose.models.User.collection, 'findOne')
|
||||
.mockRejectedValueOnce(new Error('Mongo unavailable'));
|
||||
if (reason === 'mongo-error')
|
||||
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow('Mongo unavailable');
|
||||
else await cancelUserDataDeletion(params, cancel);
|
||||
if (reason === 'mongo-error')
|
||||
expect(
|
||||
await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt),
|
||||
).toBe(true);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
await runtime.close();
|
||||
const restarted = createFavoriteRuntime(
|
||||
methods,
|
||||
{ PG_TEST_URL: options.connectionString },
|
||||
createPostgresFavoriteBackend,
|
||||
);
|
||||
try {
|
||||
await restarted.initialize(config);
|
||||
await expect(restarted.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
} finally {
|
||||
await restarted.close();
|
||||
}
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
it('retains Mongo ownership through a PostgreSQL cancellation fault, then retries after restart', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
await db.query(
|
||||
`CREATE FUNCTION reject_cancel() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'cancel unavailable'; END $$`,
|
||||
);
|
||||
await db.query(
|
||||
'CREATE TRIGGER reject_cancel BEFORE DELETE ON tool_favorite_erasure FOR EACH ROW EXECUTE FUNCTION reject_cancel()',
|
||||
);
|
||||
try {
|
||||
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow('cancel unavailable');
|
||||
} finally {
|
||||
await db.query('DROP TRIGGER reject_cancel ON tool_favorite_erasure');
|
||||
await db.query('DROP FUNCTION reject_cancel()');
|
||||
}
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
|
||||
true,
|
||||
);
|
||||
await runtime.close();
|
||||
const restarted = createFavoriteRuntime(
|
||||
methods,
|
||||
{ PG_TEST_URL: options.connectionString },
|
||||
createPostgresFavoriteBackend,
|
||||
);
|
||||
try {
|
||||
await restarted.initialize(config);
|
||||
await cancelUserDataDeletion(params, { ...methods, ...restarted.methods });
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
|
||||
false,
|
||||
);
|
||||
await expect(restarted.methods.addToolFavorite(favorite)).resolves.toEqual({
|
||||
ok: true,
|
||||
added: true,
|
||||
});
|
||||
} finally {
|
||||
await restarted.close();
|
||||
}
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('retries Mongo release after PostgreSQL cleanup without losing the durable owner', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
// Fail the real release write after the ownership read and PostgreSQL cancellation.
|
||||
jest
|
||||
.spyOn(mongoose.models.User.collection, 'updateOne')
|
||||
.mockRejectedValueOnce(new Error('Mongo release unavailable'));
|
||||
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow(
|
||||
'Mongo release unavailable',
|
||||
);
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
|
||||
true,
|
||||
);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).resolves.toEqual({
|
||||
ok: true,
|
||||
added: true,
|
||||
});
|
||||
await cancelUserDataDeletion(params, cancel);
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('tolerates a lost Mongo release acknowledgement after PostgreSQL cleanup', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
cancel.cancelAgentTriggerUserDeletion = async (userId, startedAt) => {
|
||||
await methods.cancelAgentTriggerUserDeletion(userId, startedAt, params.deletionAttemptId);
|
||||
throw new Error('release acknowledgement lost');
|
||||
};
|
||||
await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow(
|
||||
'release acknowledgement lost',
|
||||
);
|
||||
await cancelUserDataDeletion(params, { ...methods, ...runtime.methods });
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).resolves.toEqual({
|
||||
ok: true,
|
||||
added: true,
|
||||
});
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('makes concurrent cancellation idempotent without releasing a successor UUID', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
const successor = randomUUID();
|
||||
await runtime.methods.deleteUserToolFavorites(params.userId, successor);
|
||||
await Promise.all([
|
||||
cancelUserDataDeletion(params, cancel),
|
||||
cancelUserDataDeletion(params, cancel),
|
||||
]);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
expect(
|
||||
(
|
||||
await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [
|
||||
params.userId,
|
||||
])
|
||||
).rows,
|
||||
).toEqual([{ attempt_id: successor }]);
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('preserves a successor owner and UUID acquired after the ownership proof', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
const successor = randomUUID();
|
||||
const nextStartedAt = new Date(params.startedAt.getTime() + 1);
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
cancel.isAgentTriggerUserDeletionOwner = async (userId, startedAt) => {
|
||||
const owned = await methods.isAgentTriggerUserDeletionOwner(userId, startedAt);
|
||||
await methods.cancelAgentTriggerUserDeletion(userId, startedAt, params.deletionAttemptId);
|
||||
await methods.beginAgentTriggerUserDeletion(userId, nextStartedAt, successor);
|
||||
await runtime.methods.deleteUserToolFavorites(userId, successor);
|
||||
return owned;
|
||||
};
|
||||
await cancelUserDataDeletion(params, cancel);
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, nextStartedAt)).toBe(
|
||||
true,
|
||||
);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
expect(
|
||||
(
|
||||
await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [
|
||||
params.userId,
|
||||
])
|
||||
).rows,
|
||||
).toEqual([{ attempt_id: successor }]);
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('keeps both fences when a final delete is pending or loses its acknowledgement', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
const attempt = createUserDeletionAttempt();
|
||||
let releaseCommit!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
releaseCommit = resolve;
|
||||
});
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
const committing = attempt.commit(async () => {
|
||||
await gate;
|
||||
await methods.deleteUserById(params.userId, {
|
||||
startedAt: params.startedAt,
|
||||
attemptId: params.deletionAttemptId,
|
||||
});
|
||||
throw new Error('delete acknowledgement lost');
|
||||
});
|
||||
await attempt.cancel(params, cancel);
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
|
||||
true,
|
||||
);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
releaseCommit();
|
||||
await expect(committing).rejects.toThrow('delete acknowledgement lost');
|
||||
await attempt.cancel(params, cancel);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
} finally {
|
||||
releaseCommit();
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('keeps both fences after a final delete throws without a known outcome', async () => {
|
||||
const { runtime, erasure, cancel, params, favorite } = await setup();
|
||||
const attempt = createUserDeletionAttempt();
|
||||
try {
|
||||
await eraseUserData(params, erasure);
|
||||
await expect(
|
||||
attempt.commit(async () => {
|
||||
throw new Error('delete outcome unknown');
|
||||
}),
|
||||
).rejects.toThrow('delete outcome unknown');
|
||||
await attempt.cancel(params, cancel);
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe(
|
||||
true,
|
||||
);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
it('never cancels a permanent marker when no UUID was supplied', async () => {
|
||||
const { runtime, cancel, params, favorite } = await setup();
|
||||
try {
|
||||
await runtime.methods.deleteUserToolFavorites(params.userId);
|
||||
await cancelUserDataDeletion({ userId: params.userId, startedAt: params.startedAt }, cancel);
|
||||
await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure');
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type { TCheckpointerConfig, MCPOptions } from 'librechat-data-provider';
|
||||
import type { AllMethods, IMongoFile } from '@librechat/data-schemas';
|
||||
|
|
@ -47,6 +48,7 @@ type ErasureMethods = Pick<
|
|||
| 'deleteSchedulesByUser'
|
||||
| 'deleteUserChatProjects'
|
||||
| 'deleteUserToolFavorites'
|
||||
| 'cancelUserToolFavoriteDeletion'
|
||||
| 'deleteRefreshTokenBridges'
|
||||
| 'deleteAllAgentQueuedTurnsForUser'
|
||||
>;
|
||||
|
|
@ -99,10 +101,76 @@ export interface UserErasureParams {
|
|||
userId: string;
|
||||
tenantId?: string;
|
||||
checkpointer?: TCheckpointerConfig;
|
||||
/** UUID owned by this deletion attempt, distinct from the Mongo ownership timestamp. */
|
||||
deletionAttemptId?: string;
|
||||
/** Operator option to keep the transaction ledger; account self-deletion always erases it. */
|
||||
keepTransactions?: boolean;
|
||||
}
|
||||
|
||||
type UserDeletionCancellationParams = {
|
||||
userId: string;
|
||||
startedAt: Date;
|
||||
deletionAttemptId?: string;
|
||||
};
|
||||
type UserDeletionCancellationDeps = Pick<
|
||||
AllMethods,
|
||||
| 'isAgentTriggerUserDeletionOwner'
|
||||
| 'cancelAgentTriggerUserDeletion'
|
||||
| 'cancelUserToolFavoriteDeletion'
|
||||
>;
|
||||
|
||||
export interface UserDeletionAttempt {
|
||||
readonly id: string;
|
||||
adopt(
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
deps: Pick<AllMethods, 'getAgentTriggerUserDeletionAttemptId'>,
|
||||
): Promise<void>;
|
||||
commit<T>(operation: () => Promise<T>): Promise<T>;
|
||||
cancel(params: UserDeletionCancellationParams, deps: UserDeletionCancellationDeps): Promise<void>;
|
||||
}
|
||||
|
||||
/** A started user delete may still commit after an error or a lost acknowledgement. */
|
||||
export function createUserDeletionAttempt(): UserDeletionAttempt {
|
||||
let commitStarted = false;
|
||||
let identityKnown = true;
|
||||
let attemptId: string = randomUUID();
|
||||
return {
|
||||
get id() {
|
||||
return attemptId;
|
||||
},
|
||||
async adopt(userId, startedAt, deps) {
|
||||
identityKnown = false;
|
||||
const stored = await deps.getAgentTriggerUserDeletionAttemptId(userId, startedAt);
|
||||
if (stored === null) throw new Error('User deletion attempt identity unavailable');
|
||||
attemptId = stored;
|
||||
identityKnown = true;
|
||||
},
|
||||
async commit<T>(operation: () => Promise<T>): Promise<T> {
|
||||
commitStarted = true;
|
||||
return operation();
|
||||
},
|
||||
async cancel(params, deps) {
|
||||
if (!commitStarted && identityKnown) await cancelUserDataDeletion(params, deps);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Keep Mongo ownership retryable until this attempt's favorites fence is released. */
|
||||
export async function cancelUserDataDeletion(
|
||||
params: UserDeletionCancellationParams,
|
||||
deps: UserDeletionCancellationDeps,
|
||||
): Promise<void> {
|
||||
const { userId, startedAt, deletionAttemptId } = params;
|
||||
if (deletionAttemptId !== undefined && deps.cancelUserToolFavoriteDeletion) {
|
||||
if ((await deps.isAgentTriggerUserDeletionOwner(userId, startedAt, deletionAttemptId)) !== true)
|
||||
return;
|
||||
await deps.cancelUserToolFavoriteDeletion(userId, deletionAttemptId);
|
||||
}
|
||||
if (deletionAttemptId === undefined) await deps.cancelAgentTriggerUserDeletion(userId, startedAt);
|
||||
else await deps.cancelAgentTriggerUserDeletion(userId, startedAt, deletionAttemptId);
|
||||
}
|
||||
|
||||
export type MCPServerCleanupDeps = Pick<
|
||||
UserErasureDeps,
|
||||
| 'findUserErasableMCPServers'
|
||||
|
|
@ -307,7 +375,9 @@ export async function eraseUserData(
|
|||
deps.deleteAllUserPluginAuths(userId),
|
||||
deps.deleteToolCalls(userId),
|
||||
deps.deleteConversationTags({ user: userId }),
|
||||
deps.deleteUserToolFavorites(userId),
|
||||
params.deletionAttemptId === undefined
|
||||
? deps.deleteUserToolFavorites(userId)
|
||||
: deps.deleteUserToolFavorites(userId, params.deletionAttemptId),
|
||||
deps.deleteAllUserMemories(userId),
|
||||
]);
|
||||
await deps.deleteSharedLinks(userId);
|
||||
|
|
|
|||
|
|
@ -102,6 +102,8 @@ describe('createErasureResolver', () => {
|
|||
const { deps } = createDeps({
|
||||
db: {
|
||||
deleteUserAgents: jest.fn(record('agents')),
|
||||
deleteUserToolFavorites: jest.fn(record('favorites')),
|
||||
cancelUserToolFavoriteDeletion: jest.fn(record('cancelFavorites')),
|
||||
findMCPServerOAuthTokenUserIds: jest.fn(record('tokens')),
|
||||
findMCPServerViewerEntries: jest.fn(record('viewers')),
|
||||
findMCPServerViewerUserIds: jest.fn(record('viewerUsers')),
|
||||
|
|
@ -118,6 +120,8 @@ describe('createErasureResolver', () => {
|
|||
|
||||
await tenantStorage.run({ tenantId: 'admin-tenant' }, async () => {
|
||||
await erasure.deleteUserAgents('user-1');
|
||||
await erasure.deleteUserToolFavorites('user-1', 'attempt');
|
||||
await erasure.cancelUserToolFavoriteDeletion?.('user-1', 'attempt');
|
||||
await erasure.findMCPServerOAuthTokenUserIds?.('srv');
|
||||
await erasure.findMCPServerViewerEntries?.('srv-id');
|
||||
await erasure.findMCPServerViewerUserIds?.([]);
|
||||
|
|
@ -130,7 +134,7 @@ describe('createErasureResolver', () => {
|
|||
await erasure.mcpOAuth?.disconnectUser?.('other', 'srv');
|
||||
});
|
||||
|
||||
expect(Object.keys(seen)).toHaveLength(10);
|
||||
expect(Object.keys(seen)).toHaveLength(12);
|
||||
expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,16 @@ export function createErasureResolver(deps: ErasureResolverDeps) {
|
|||
...deps.db,
|
||||
openCheckpointDeletion: deps.openCheckpointDeletion,
|
||||
deleteSharedLinks: deps.deleteSharedLinks,
|
||||
deleteUserToolFavorites: (userId, deletionAttemptId) =>
|
||||
inTargetTenant(async () =>
|
||||
deletionAttemptId === undefined
|
||||
? deps.db.deleteUserToolFavorites(userId)
|
||||
: deps.db.deleteUserToolFavorites(userId, deletionAttemptId),
|
||||
),
|
||||
cancelUserToolFavoriteDeletion: (userId, deletionAttemptId) =>
|
||||
inTargetTenant(async () =>
|
||||
deps.db.cancelUserToolFavoriteDeletion?.(userId, deletionAttemptId),
|
||||
),
|
||||
deleteUserAgents: (userId) => inTargetTenant(async () => deps.db.deleteUserAgents(userId)),
|
||||
findMCPServerOAuthTokenUserIds: (serverName) =>
|
||||
inTargetTenant(async () => deps.db.findMCPServerOAuthTokenUserIds(serverName)),
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ export type DeploymentOnlyConfigReason = 'startup' | 'operator' | 'unused';
|
|||
* a stored override had no effect rather than a dangerous one.
|
||||
*/
|
||||
export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly<Record<string, DeploymentOnlyConfigReason>> = {
|
||||
persistence: 'operator',
|
||||
'telemetry.stackTraceLimit': 'operator',
|
||||
'telemetry.rum': 'operator',
|
||||
clientLimits: 'operator',
|
||||
|
|
@ -4666,7 +4667,35 @@ export const toolCallPreviewsConfigSchema = z.object({
|
|||
|
||||
export type TToolCallPreviewsConfig = z.infer<typeof toolCallPreviewsConfigSchema>;
|
||||
|
||||
export const persistenceConfigSchema = z
|
||||
.object({
|
||||
favorites: z.enum(['mongo', 'postgres']).default('mongo'),
|
||||
postgres: z
|
||||
.object({
|
||||
connectionStringEnv: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/),
|
||||
max: z.number().int().min(1).max(100).default(10),
|
||||
maxWaiting: z.number().int().min(1).max(1000).default(100),
|
||||
acquireTimeoutMs: z.number().int().min(1).max(60000).default(5000),
|
||||
queryTimeoutMs: z.number().int().min(1).max(300000).default(15000),
|
||||
idleTimeoutMs: z.number().int().min(1).max(300000).default(30000),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
})
|
||||
.strict()
|
||||
.default({ favorites: 'mongo' })
|
||||
.superRefine((config, context) => {
|
||||
if (config.favorites === 'postgres' && !config.postgres) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ['postgres'],
|
||||
message: 'PostgreSQL configuration is required',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export const configSchema = z.object({
|
||||
persistence: persistenceConfigSchema,
|
||||
version: z.string(),
|
||||
telemetry: telemetrySchema.optional(),
|
||||
clientLimits: clientLimitsSchema.optional(),
|
||||
|
|
|
|||
33
packages/data-provider/src/persistence.spec.ts
Normal file
33
packages/data-provider/src/persistence.spec.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import { configSchema, persistenceConfigSchema, DEPLOYMENT_ONLY_CONFIG_PATHS } from './config';
|
||||
|
||||
describe('deployment persistence configuration', () => {
|
||||
it('defaults existing deployments to Mongo and keeps selection deployment-only', () => {
|
||||
expect(configSchema.parse({ version: '1.3.1' }).persistence).toEqual({ favorites: 'mongo' });
|
||||
expect(DEPLOYMENT_ONLY_CONFIG_PATHS.persistence).toBe('operator');
|
||||
});
|
||||
it('accepts bounded pool settings and an environment-variable name, never a DSN', () => {
|
||||
const config = persistenceConfigSchema.parse({
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'LIBRECHAT_POSTGRES_DSN' },
|
||||
});
|
||||
expect(config.postgres).toMatchObject({
|
||||
max: 10,
|
||||
maxWaiting: 100,
|
||||
acquireTimeoutMs: 5000,
|
||||
queryTimeoutMs: 15000,
|
||||
});
|
||||
expect(
|
||||
persistenceConfigSchema.safeParse({
|
||||
favorites: 'postgres',
|
||||
postgres: { connectionStringEnv: 'postgres://secret' },
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
it.each([
|
||||
{ favorites: 'postgres' },
|
||||
{ favorites: 'invalid' },
|
||||
{ favorites: 'postgres', postgres: { connectionStringEnv: 'PG', max: 0 } },
|
||||
])('refuses an explicit invalid selection: %j', (config) => {
|
||||
expect(persistenceConfigSchema.safeParse(config).success).toBe(false);
|
||||
});
|
||||
});
|
||||
9
packages/data-schemas/jest.postgres.config.mjs
Normal file
9
packages/data-schemas/jest.postgres.config.mjs
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import config from './jest.config.mjs';
|
||||
|
||||
export default {
|
||||
...config,
|
||||
testMatch: ['<rootDir>/src/postgres/*.spec.ts'],
|
||||
globalSetup: undefined,
|
||||
testResultsProcessor: undefined,
|
||||
maxWorkers: 1,
|
||||
};
|
||||
|
|
@ -47,7 +47,8 @@
|
|||
"test:quality:favorites": "jest --config jest.favorite.config.mjs --coverage && node ../../scripts/crap.mts coverage/favorites/coverage-final.json src/methods/favorite.ts",
|
||||
"test:contracts:leases": "jest --config jest.lease.config.mjs --coverage=false",
|
||||
"test:mutations:leases": "cd ../.. && stryker run packages/data-schemas/stryker.lease.config.mjs",
|
||||
"test:quality:leases": "jest --config jest.lease.config.mjs --coverage && node ../../scripts/crap.mts coverage/leases/coverage-final.json src/methods/resourceLock.ts"
|
||||
"test:quality:leases": "jest --config jest.lease.config.mjs --coverage && node ../../scripts/crap.mts coverage/leases/coverage-final.json src/methods/resourceLock.ts",
|
||||
"test:postgres": "jest --config jest.postgres.config.mjs --coverage=false --runInBand"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
@ -67,13 +68,15 @@
|
|||
"micromark-extension-directive": "^3.0.1",
|
||||
"micromark-extension-gfm": "^3.0.0",
|
||||
"micromark-extension-llm-math": "^3.1.0",
|
||||
"micromark-util-decode-string": "^2.0.0"
|
||||
"micromark-util-decode-string": "^2.0.0",
|
||||
"pg": "^8.23.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@stryker-mutator/core": "10.1.1",
|
||||
"@stryker-mutator/jest-runner": "10.1.1",
|
||||
"@types/jest": "^29.5.2",
|
||||
"@types/node": "^24.12.4",
|
||||
"@types/pg": "^8.23.1",
|
||||
"jest": "^30.2.0",
|
||||
"jest-junit": "^17.0.0",
|
||||
"mongodb-memory-server": "^11.3.0",
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ export type {
|
|||
} from './methods/response';
|
||||
export {
|
||||
createMethods,
|
||||
createToolFavoriteMethods,
|
||||
createChannelIdentityStorage,
|
||||
createChannelFeedbackStorage,
|
||||
ChannelFeedbackAuthorityError,
|
||||
|
|
@ -182,3 +183,10 @@ export type { SearchTokenBackfillResult } from './migrations';
|
|||
export { createBrowserSetupStore, createBrowserSetupBackend } from './methods/browserSetup';
|
||||
export type * from './types/browserSetup';
|
||||
export type * from './types/codeEnvironmentSetup';
|
||||
|
||||
export * from './postgres/backend';
|
||||
export * from './postgres/pool';
|
||||
export * from './postgres/migrations';
|
||||
export * from './postgres/favorite';
|
||||
export * from './postgres/import';
|
||||
export * from './postgres/mongo';
|
||||
|
|
|
|||
|
|
@ -19,12 +19,9 @@ function capError(): Error {
|
|||
return error;
|
||||
}
|
||||
|
||||
export function createToolFavoriteMethods(mongoose: typeof import('mongoose')): {
|
||||
getToolFavorites: (userId: string) => Promise<t.IToolFavoriteLean[]>;
|
||||
addToolFavorite: (params: t.ToolFavoriteParams) => Promise<t.AddToolFavoriteResult>;
|
||||
removeToolFavorite: (params: t.ToolFavoriteParams) => Promise<t.RemoveToolFavoriteResult>;
|
||||
deleteUserToolFavorites: (userId: string) => Promise<number>;
|
||||
} {
|
||||
export function createToolFavoriteMethods(
|
||||
mongoose: typeof import('mongoose'),
|
||||
): t.ToolFavoriteMethods {
|
||||
async function getToolFavorites(userId: string): Promise<t.IToolFavoriteLean[]> {
|
||||
const ToolFavorite = mongoose.models.ToolFavorite;
|
||||
return ToolFavorite.find({ user: userId })
|
||||
|
|
@ -96,4 +93,4 @@ export function createToolFavoriteMethods(mongoose: typeof import('mongoose')):
|
|||
};
|
||||
}
|
||||
|
||||
export type ToolFavoriteMethods = ReturnType<typeof createToolFavoriteMethods>;
|
||||
export type { ToolFavoriteMethods } from '~/types/favorite';
|
||||
|
|
|
|||
|
|
@ -368,6 +368,7 @@ export type AllMethods = PermissionPrincipalStorage &
|
|||
|
||||
/** Dependencies injected from the api layer into createMethods */
|
||||
export interface CreateMethodsDeps {
|
||||
toolFavorites?: ToolFavoriteMethods;
|
||||
getChannelStorageConfig?: () => Promise<Partial<ChannelStorageConfig>>;
|
||||
/** Matches a model name to a canonical key. From @librechat/api. */
|
||||
matchModelName?: (model: string, endpoint?: string) => string | undefined;
|
||||
|
|
@ -575,7 +576,7 @@ export function createMethods(
|
|||
...createFileMethods(mongoose),
|
||||
...createFileStorageMethods(mongoose),
|
||||
...createMemoryMethods(mongoose),
|
||||
...createToolFavoriteMethods(mongoose),
|
||||
...(deps.toolFavorites ?? createToolFavoriteMethods(mongoose)),
|
||||
...createAgentCategoryMethods(mongoose),
|
||||
...createAgentApiKeyMethods(mongoose),
|
||||
...createAgentChannelMethods(mongoose, deps.getChannelStorageConfig),
|
||||
|
|
@ -730,3 +731,5 @@ export type {
|
|||
};
|
||||
|
||||
export { recordAgentEventActorReceiptMetric, setAgentEventActorReceiptMetricObserver };
|
||||
|
||||
export { createToolFavoriteMethods } from './favorite';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import {
|
||||
CacheKeys,
|
||||
|
|
@ -1556,6 +1557,111 @@ describe('User Methods - Database Tests', () => {
|
|||
expect(found).toBeNull();
|
||||
});
|
||||
|
||||
test.each(['cancel', 'commit'] as const)(
|
||||
'rejects an old queued delete after an offline handoff and %s',
|
||||
async (outcome) => {
|
||||
const user = await User.create({
|
||||
email: `delayed-${outcome}@example.com`,
|
||||
provider: 'local',
|
||||
});
|
||||
const id = user._id.toString();
|
||||
const attemptId = randomUUID();
|
||||
const startedAt = new Date('2026-01-01');
|
||||
const recoveredAt = new Date(startedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
|
||||
await methods.beginAgentTriggerUserDeletion(id, startedAt, attemptId);
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const pendingWrite = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const original = User.collection.deleteOne.bind(User.collection);
|
||||
const write = jest
|
||||
.spyOn(User.collection, 'deleteOne')
|
||||
.mockImplementationOnce(async (filter, options) => {
|
||||
entered();
|
||||
await gate;
|
||||
return original(filter, options);
|
||||
});
|
||||
const pendingDelete = methods.deleteUserById(id, { startedAt, attemptId });
|
||||
try {
|
||||
await pendingWrite;
|
||||
expect(write.mock.calls[0][0]).toMatchObject({
|
||||
agentTriggerDeletionStartedAt: startedAt,
|
||||
agentTriggerDeletionAttemptId: attemptId,
|
||||
});
|
||||
expect(
|
||||
await methods.recoverStaleAgentTriggerUserDeletion(id, recoveredAt, randomUUID()),
|
||||
).toBe('acquired');
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, recoveredAt)).toBe(
|
||||
attemptId,
|
||||
);
|
||||
if (outcome === 'cancel') {
|
||||
expect(await methods.cancelAgentTriggerUserDeletion(id, recoveredAt, attemptId)).toBe(
|
||||
true,
|
||||
);
|
||||
} else {
|
||||
expect(
|
||||
await methods.deleteUserById(id, { startedAt: recoveredAt, attemptId }),
|
||||
).toMatchObject({ deletedCount: 1 });
|
||||
}
|
||||
release();
|
||||
expect(await pendingDelete).toMatchObject({ deletedCount: 0 });
|
||||
expect(await User.exists({ _id: id })).toEqual(
|
||||
outcome === 'cancel' ? { _id: user._id } : null,
|
||||
);
|
||||
} finally {
|
||||
release();
|
||||
await pendingDelete;
|
||||
write.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test('rejects the wrong UUID or Date and commits only the current owner', async () => {
|
||||
const user = await User.create({ email: 'fenced-delete@example.com', provider: 'local' });
|
||||
const id = user._id.toString();
|
||||
const attemptId = randomUUID();
|
||||
const startedAt = new Date();
|
||||
await methods.beginAgentTriggerUserDeletion(id, startedAt, attemptId);
|
||||
expect(
|
||||
await methods.deleteUserById(id, { startedAt, attemptId: randomUUID() }),
|
||||
).toMatchObject({ deletedCount: 0 });
|
||||
expect(
|
||||
await methods.deleteUserById(id, {
|
||||
startedAt: new Date(startedAt.getTime() + 1),
|
||||
attemptId,
|
||||
}),
|
||||
).toMatchObject({ deletedCount: 0 });
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(id, startedAt, attemptId)).toBe(true);
|
||||
expect(await methods.deleteUserById(id, { startedAt, attemptId })).toMatchObject({
|
||||
deletedCount: 1,
|
||||
});
|
||||
expect(
|
||||
await methods.recoverStaleAgentTriggerUserDeletion(
|
||||
id,
|
||||
new Date(startedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1),
|
||||
randomUUID(),
|
||||
),
|
||||
).toBe('missing');
|
||||
});
|
||||
|
||||
test('rejects invalid commit fences before any cleanup or delete', async () => {
|
||||
const id = new mongoose.Types.ObjectId().toString();
|
||||
const write = jest.spyOn(User, 'deleteOne');
|
||||
const grants = jest.spyOn(mongoose.models.ToolApprovalGrant, 'deleteMany');
|
||||
await expect(
|
||||
methods.deleteUserById(id, { startedAt: new Date(Number.NaN), attemptId: randomUUID() }),
|
||||
).rejects.toThrow('startedAt must be a valid Date');
|
||||
await expect(
|
||||
methods.deleteUserById(id, { startedAt: new Date(), attemptId: '' }),
|
||||
).rejects.toThrow('Invalid user deletion attempt ID');
|
||||
expect(write).not.toHaveBeenCalled();
|
||||
expect(grants).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should return zero count for non-existent user', async () => {
|
||||
const fakeId = new mongoose.Types.ObjectId();
|
||||
const result = await methods.deleteUserById(fakeId.toString());
|
||||
|
|
@ -1590,6 +1696,88 @@ describe('User Methods - Database Tests', () => {
|
|||
await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);
|
||||
});
|
||||
|
||||
test('proves the exact live owner from primary without releasing its fence', async () => {
|
||||
const user = await User.create({ email: 'deletion-owner@example.com', provider: 'local' });
|
||||
const userId = user._id.toString();
|
||||
const startedAt = new Date('2026-11-10T12:00:00.000Z');
|
||||
await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(false);
|
||||
await methods.beginAgentTriggerUserDeletion(userId, startedAt);
|
||||
const read = jest.spyOn(mongoose.Query.prototype, 'read');
|
||||
await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(true);
|
||||
expect(read).toHaveBeenCalledWith('primary');
|
||||
await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);
|
||||
await expect(
|
||||
methods.isAgentTriggerUserDeletionOwner(userId, new Date(startedAt.getTime() + 1)),
|
||||
).resolves.toBe(false);
|
||||
await methods.deleteUserById(userId);
|
||||
await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
test('propagates ownership-read errors instead of treating them as absent ownership', async () => {
|
||||
const failure = new Error('ownership read unavailable');
|
||||
jest.spyOn(User.collection, 'findOne').mockRejectedValueOnce(failure);
|
||||
await expect(
|
||||
methods.isAgentTriggerUserDeletionOwner(
|
||||
new mongoose.Types.ObjectId().toString(),
|
||||
new Date(),
|
||||
),
|
||||
).rejects.toThrow(failure);
|
||||
});
|
||||
|
||||
test('persists identity across repeated offline recoveries and fences stale Date and UUID owners', async () => {
|
||||
const user = await User.create({ email: 'durable-attempt@example.com', provider: 'local' });
|
||||
const id = user._id.toString();
|
||||
const attempt = randomUUID();
|
||||
const other = randomUUID();
|
||||
const first = new Date('2026-01-01');
|
||||
const second = new Date(first.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
|
||||
const third = new Date(second.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
|
||||
expect(await methods.beginAgentTriggerUserDeletion(id, first, attempt)).toBe('acquired');
|
||||
expect((await User.findById(id).lean())?.agentTriggerDeletionAttemptId).toBeUndefined();
|
||||
expect(await methods.recoverStaleAgentTriggerUserDeletion(id, second, other)).toBe(
|
||||
'acquired',
|
||||
);
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, first)).toBeNull();
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBe(attempt);
|
||||
expect(await methods.cancelAgentTriggerUserDeletion(id, first, attempt)).toBe(false);
|
||||
expect(await methods.cancelAgentTriggerUserDeletion(id, second, other)).toBe(false);
|
||||
expect(await methods.isAgentTriggerUserDeletionOwner(id, second, other)).toBe(false);
|
||||
// Recovery can itself crash before adopting the identity; the next process still finds it.
|
||||
expect(await methods.recoverStaleAgentTriggerUserDeletion(id, third, randomUUID())).toBe(
|
||||
'acquired',
|
||||
);
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, third)).toBe(attempt);
|
||||
expect(await methods.cancelAgentTriggerUserDeletion(id, third, attempt)).toBe(true);
|
||||
expect(
|
||||
await User.findById(id)
|
||||
.select('+agentTriggerDeletionAttemptId +agentTriggerDeletionStartedAt')
|
||||
.lean(),
|
||||
).toEqual(expect.not.objectContaining({ agentTriggerDeletionAttemptId: expect.anything() }));
|
||||
expect(await methods.beginAgentTriggerUserDeletion(id, third, other)).toBe('acquired');
|
||||
expect(await methods.cancelAgentTriggerUserDeletion(id, third, attempt)).toBe(false);
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, third)).toBe(other);
|
||||
});
|
||||
test('initializes only a legacy missing UUID and clears stray identity on a no-token acquisition', async () => {
|
||||
const user = await User.create({ email: 'legacy-attempt@example.com', provider: 'local' });
|
||||
const id = user._id.toString();
|
||||
const first = new Date('2026-01-01');
|
||||
const second = new Date(first.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
|
||||
const fallback = randomUUID();
|
||||
await methods.beginAgentTriggerUserDeletion(id, first);
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, first)).toBeNull();
|
||||
expect(await methods.recoverStaleAgentTriggerUserDeletion(id, second, fallback)).toBe(
|
||||
'acquired',
|
||||
);
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBe(fallback);
|
||||
await methods.cancelAgentTriggerUserDeletion(id, second, fallback);
|
||||
await User.updateOne({ _id: id }, { $set: { agentTriggerDeletionAttemptId: randomUUID() } });
|
||||
await methods.beginAgentTriggerUserDeletion(id, second);
|
||||
expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBeNull();
|
||||
await expect(methods.beginAgentTriggerUserDeletion(id, second, '')).rejects.toThrow(
|
||||
'attempt',
|
||||
);
|
||||
});
|
||||
|
||||
test('reports a missing principal without creating a deletion fence', async () => {
|
||||
const userId = new mongoose.Types.ObjectId().toString();
|
||||
|
||||
|
|
|
|||
|
|
@ -183,18 +183,33 @@ export function createUserMethods(
|
|||
beginAgentTriggerUserDeletion: (
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
) => Promise<'acquired' | 'in_progress' | 'missing'>;
|
||||
recoverStaleAgentTriggerUserDeletion: (
|
||||
userId: string,
|
||||
recoveredAt: Date,
|
||||
legacyAttemptId?: string,
|
||||
) => Promise<'acquired' | 'in_progress' | 'missing'>;
|
||||
cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;
|
||||
getAgentTriggerUserDeletionAttemptId: (userId: string, startedAt: Date) => Promise<string | null>;
|
||||
isAgentTriggerUserDeletionOwner: (
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
) => Promise<boolean>;
|
||||
cancelAgentTriggerUserDeletion: (
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
) => Promise<boolean>;
|
||||
isAgentTriggerPrincipalActive: (userId: string) => Promise<boolean>;
|
||||
fenceSubagentAdmission: (userId: string, token: string, fencedUntil: Date) => Promise<void>;
|
||||
renewSubagentAdmission: (userId: string, token: string, fencedUntil: Date) => Promise<boolean>;
|
||||
releaseSubagentAdmission: (userId: string, token: string) => Promise<void>;
|
||||
isSubagentOwnerAdmissible: (userId: string) => Promise<boolean>;
|
||||
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
|
||||
deleteUserById: (
|
||||
userId: string,
|
||||
fence?: { startedAt: Date; attemptId: string },
|
||||
) => Promise<UserDeleteResult>;
|
||||
updateUserPlugins: (
|
||||
userId: string,
|
||||
plugins: string[] | undefined,
|
||||
|
|
@ -645,7 +660,16 @@ export function createUserMethods(
|
|||
/**
|
||||
* Delete a user by their unique ID.
|
||||
*/
|
||||
async function deleteUserById(userId: string): Promise<UserDeleteResult> {
|
||||
async function deleteUserById(
|
||||
userId: string,
|
||||
fence?: { startedAt: Date; attemptId: string },
|
||||
): Promise<UserDeleteResult> {
|
||||
if (fence !== undefined) {
|
||||
if (!(fence?.startedAt instanceof Date) || !Number.isFinite(fence.startedAt.getTime())) {
|
||||
throw new TypeError('startedAt must be a valid Date');
|
||||
}
|
||||
validateDeletionAttemptId(fence.attemptId);
|
||||
}
|
||||
try {
|
||||
const User = mongoose.models.User;
|
||||
const owner = await User.findById(userId).select('tenantId').lean<Pick<IUser, 'tenantId'>>();
|
||||
|
|
@ -664,7 +688,15 @@ export function createUserMethods(
|
|||
);
|
||||
}
|
||||
await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId });
|
||||
const result = await User.deleteOne({ _id: userId });
|
||||
const result = await User.deleteOne({
|
||||
_id: userId,
|
||||
...(fence === undefined
|
||||
? {}
|
||||
: {
|
||||
agentTriggerDeletionStartedAt: fence.startedAt,
|
||||
agentTriggerDeletionAttemptId: fence.attemptId,
|
||||
}),
|
||||
});
|
||||
if (result.deletedCount === 0) {
|
||||
return { deletedCount: 0, message: 'No user found with that ID.' };
|
||||
}
|
||||
|
|
@ -676,18 +708,32 @@ export function createUserMethods(
|
|||
}
|
||||
}
|
||||
|
||||
function validateDeletionAttemptId(attemptId: string): void {
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(attemptId)) {
|
||||
throw new TypeError('Invalid user deletion attempt ID');
|
||||
}
|
||||
}
|
||||
|
||||
/** Establishes the durable admission fence used while account deletion drains triggers. */
|
||||
async function beginAgentTriggerUserDeletion(
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
): Promise<'acquired' | 'in_progress' | 'missing'> {
|
||||
if (!(startedAt instanceof Date) || !Number.isFinite(startedAt.getTime())) {
|
||||
throw new TypeError('startedAt must be a valid Date');
|
||||
}
|
||||
if (attemptId !== undefined) validateDeletionAttemptId(attemptId);
|
||||
const User = mongoose.models.User;
|
||||
const result = await User.updateOne(
|
||||
{ _id: userId, agentTriggerDeletionStartedAt: { $exists: false } },
|
||||
{ $set: { agentTriggerDeletionStartedAt: startedAt } },
|
||||
{
|
||||
$set: {
|
||||
agentTriggerDeletionStartedAt: startedAt,
|
||||
...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }),
|
||||
},
|
||||
...(attemptId === undefined ? { $unset: { agentTriggerDeletionAttemptId: 1 } } : {}),
|
||||
},
|
||||
);
|
||||
if (result.modifiedCount === 1) {
|
||||
await invalidateAuthUserDocCache(userId);
|
||||
|
|
@ -701,12 +747,34 @@ export function createUserMethods(
|
|||
async function recoverStaleAgentTriggerUserDeletion(
|
||||
userId: string,
|
||||
recoveredAt: Date,
|
||||
legacyAttemptId?: string,
|
||||
): Promise<'acquired' | 'in_progress' | 'missing'> {
|
||||
if (!(recoveredAt instanceof Date) || !Number.isFinite(recoveredAt.getTime())) {
|
||||
throw new TypeError('recoveredAt must be a valid Date');
|
||||
}
|
||||
const User = mongoose.models.User;
|
||||
if (legacyAttemptId !== undefined) validateDeletionAttemptId(legacyAttemptId);
|
||||
const staleBefore = new Date(recoveredAt.getTime() - USER_DELETION_FENCE_STALE_MS);
|
||||
// Two ordinary CAS updates keep DocumentDB compatibility and never overwrite a stored UUID.
|
||||
if (legacyAttemptId !== undefined) {
|
||||
const legacy = await User.updateOne(
|
||||
{
|
||||
_id: userId,
|
||||
agentTriggerDeletionStartedAt: { $lte: staleBefore },
|
||||
agentTriggerDeletionAttemptId: { $exists: false },
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
agentTriggerDeletionStartedAt: recoveredAt,
|
||||
agentTriggerDeletionAttemptId: legacyAttemptId,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (legacy.modifiedCount === 1) {
|
||||
await invalidateAuthUserDocCache(userId);
|
||||
return 'acquired';
|
||||
}
|
||||
}
|
||||
const result = await User.updateOne(
|
||||
{ _id: userId, agentTriggerDeletionStartedAt: { $lte: staleBefore } },
|
||||
{ $set: { agentTriggerDeletionStartedAt: recoveredAt } },
|
||||
|
|
@ -718,12 +786,55 @@ export function createUserMethods(
|
|||
return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress';
|
||||
}
|
||||
|
||||
async function getAgentTriggerUserDeletionAttemptId(
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
): Promise<string | null> {
|
||||
const user = await mongoose.models.User.findOne({
|
||||
_id: userId,
|
||||
agentTriggerDeletionStartedAt: startedAt,
|
||||
})
|
||||
.select({ agentTriggerDeletionAttemptId: 1, _id: 0 })
|
||||
.read('primary')
|
||||
.lean<Pick<IUser, 'agentTriggerDeletionAttemptId'>>();
|
||||
const attemptId = user?.agentTriggerDeletionAttemptId;
|
||||
if (attemptId == null) return null;
|
||||
validateDeletionAttemptId(attemptId);
|
||||
return attemptId;
|
||||
}
|
||||
|
||||
/** Reads current ownership without releasing the durable retry anchor. */
|
||||
async function isAgentTriggerUserDeletionOwner(
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
): Promise<boolean> {
|
||||
if (attemptId !== undefined) validateDeletionAttemptId(attemptId);
|
||||
const User = mongoose.models.User;
|
||||
return (
|
||||
(await User.exists({
|
||||
_id: userId,
|
||||
agentTriggerDeletionStartedAt: startedAt,
|
||||
...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }),
|
||||
}).read('primary')) != null
|
||||
);
|
||||
}
|
||||
|
||||
/** Releases only the account-deletion attempt that owns this exact fence. */
|
||||
async function cancelAgentTriggerUserDeletion(userId: string, startedAt: Date): Promise<boolean> {
|
||||
async function cancelAgentTriggerUserDeletion(
|
||||
userId: string,
|
||||
startedAt: Date,
|
||||
attemptId?: string,
|
||||
): Promise<boolean> {
|
||||
if (attemptId !== undefined) validateDeletionAttemptId(attemptId);
|
||||
const User = mongoose.models.User;
|
||||
const result = await User.updateOne(
|
||||
{ _id: userId, agentTriggerDeletionStartedAt: startedAt },
|
||||
{ $unset: { agentTriggerDeletionStartedAt: 1 } },
|
||||
{
|
||||
_id: userId,
|
||||
agentTriggerDeletionStartedAt: startedAt,
|
||||
...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }),
|
||||
},
|
||||
{ $unset: { agentTriggerDeletionStartedAt: 1, agentTriggerDeletionAttemptId: 1 } },
|
||||
);
|
||||
if (result.modifiedCount === 1) {
|
||||
await invalidateAuthUserDocCache(userId);
|
||||
|
|
@ -1088,6 +1199,8 @@ export function createUserMethods(
|
|||
generateToken,
|
||||
beginAgentTriggerUserDeletion,
|
||||
recoverStaleAgentTriggerUserDeletion,
|
||||
getAgentTriggerUserDeletionAttemptId,
|
||||
isAgentTriggerUserDeletionOwner,
|
||||
cancelAgentTriggerUserDeletion,
|
||||
isAgentTriggerPrincipalActive,
|
||||
fenceSubagentAdmission,
|
||||
|
|
|
|||
117
packages/data-schemas/src/postgres/backend.spec.ts
Normal file
117
packages/data-schemas/src/postgres/backend.spec.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import mongoose from 'mongoose';
|
||||
import type { PostgresDatabase, PostgresOptions } from './pool';
|
||||
import type { PostgresFavoriteBackend } from './backend';
|
||||
import { createPostgresTestDatabase } from './postgres.helper';
|
||||
import { createPostgresFavoriteBackend } from './backend';
|
||||
import { migratePostgres } from './migrations';
|
||||
import { importFavorites } from './import';
|
||||
|
||||
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
const userId = '0123456789abcdef01234567';
|
||||
|
||||
postgres('PostgreSQL favorite backend composition', () => {
|
||||
let db: PostgresDatabase | undefined;
|
||||
let backend: PostgresFavoriteBackend | undefined;
|
||||
let options: PostgresOptions;
|
||||
|
||||
beforeEach(async () => {
|
||||
({ db, options } = await createPostgresTestDatabase());
|
||||
await migratePostgres(db);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await backend?.close();
|
||||
} finally {
|
||||
backend = undefined;
|
||||
await db?.close();
|
||||
db = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
async function verifyEmpty(): Promise<void> {
|
||||
await importFavorites(
|
||||
db!,
|
||||
async function* () {
|
||||
/* verified empty source */
|
||||
},
|
||||
{ offline: true, batchSize: 1, verifiedEmpty: true },
|
||||
);
|
||||
}
|
||||
|
||||
async function readiness() {
|
||||
return (await db!.query('SELECT * FROM tool_favorite_readiness')).rows;
|
||||
}
|
||||
|
||||
it('does not activate verified data when constructed or read before initialization', async () => {
|
||||
await verifyEmpty();
|
||||
const before = await readiness();
|
||||
expect(before).toEqual([expect.objectContaining({ verified: true, activated: false })]);
|
||||
backend = createPostgresFavoriteBackend(options);
|
||||
expect(await backend.methods.getToolFavorites(userId)).toEqual([]);
|
||||
expect(await readiness()).toEqual(before);
|
||||
});
|
||||
|
||||
it('initializes verified-empty data by activating readiness without touching Mongo', async () => {
|
||||
await verifyEmpty();
|
||||
const before = await readiness();
|
||||
const connect = jest.spyOn(mongoose, 'connect');
|
||||
const collection = jest.spyOn(mongoose.connection, 'collection');
|
||||
const model = jest.spyOn(mongoose, 'model');
|
||||
backend = createPostgresFavoriteBackend(options);
|
||||
await backend.initialize();
|
||||
expect(await readiness()).toEqual([{ ...before[0], activated: true }]);
|
||||
expect(await backend.methods.getToolFavorites(userId)).toEqual([]);
|
||||
expect(connect).not.toHaveBeenCalled();
|
||||
expect(collection).not.toHaveBeenCalled();
|
||||
expect(model).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects unverified data without activating it', async () => {
|
||||
backend = createPostgresFavoriteBackend(options);
|
||||
await expect(backend.initialize()).rejects.toThrow('PostgreSQL favorites data not verified');
|
||||
expect(await readiness()).toEqual([
|
||||
expect.objectContaining({ verified: false, activated: false }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('exposes the native favorite add, read, and removal contract', async () => {
|
||||
await verifyEmpty();
|
||||
backend = createPostgresFavoriteBackend(options);
|
||||
await backend.initialize();
|
||||
const favorite = { userId, itemType: 'tool', itemId: 'backend-tool' } as const;
|
||||
await expect(backend.methods.addToolFavorite(favorite)).resolves.toEqual({
|
||||
ok: true,
|
||||
added: true,
|
||||
});
|
||||
await expect(backend.methods.getToolFavorites(userId)).resolves.toEqual([
|
||||
{ itemType: 'tool', itemId: 'backend-tool' },
|
||||
]);
|
||||
expect((await db!.query('SELECT count(*) FROM tool_favorites')).rows).toEqual([{ count: '1' }]);
|
||||
await expect(backend.methods.removeToolFavorite(favorite)).resolves.toEqual({
|
||||
ok: true,
|
||||
removed: true,
|
||||
});
|
||||
await expect(backend.methods.getToolFavorites(userId)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('closes idempotently and rejects methods and initialization after close', async () => {
|
||||
await verifyEmpty();
|
||||
backend = createPostgresFavoriteBackend(options);
|
||||
await backend.initialize();
|
||||
const closing = backend.close();
|
||||
expect(backend.close()).toBe(closing);
|
||||
await closing;
|
||||
await expect(backend.close()).resolves.toBeUndefined();
|
||||
await expect(backend.methods.getToolFavorites(userId)).rejects.toThrow('admission');
|
||||
await expect(backend.initialize()).rejects.toThrow('admission');
|
||||
});
|
||||
|
||||
it('can be closed idempotently after initialization fails', async () => {
|
||||
backend = createPostgresFavoriteBackend(options);
|
||||
await expect(backend.initialize()).rejects.toThrow('not verified');
|
||||
await expect(backend.close()).resolves.toBeUndefined();
|
||||
await expect(backend.close()).resolves.toBeUndefined();
|
||||
await expect(backend.methods.getToolFavorites(userId)).rejects.toThrow('admission');
|
||||
});
|
||||
});
|
||||
20
packages/data-schemas/src/postgres/backend.ts
Normal file
20
packages/data-schemas/src/postgres/backend.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import type { ToolFavoriteMethods } from '~/types/favorite';
|
||||
import type { PostgresOptions } from './pool';
|
||||
import { createPostgresFavoriteMethods } from './favorite';
|
||||
import { assertPostgresReady } from './migrations';
|
||||
import { createPostgresDatabase } from './pool';
|
||||
|
||||
export interface PostgresFavoriteBackend {
|
||||
methods: ToolFavoriteMethods;
|
||||
initialize(): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export function createPostgresFavoriteBackend(options: PostgresOptions): PostgresFavoriteBackend {
|
||||
const db = createPostgresDatabase(options);
|
||||
return {
|
||||
methods: createPostgresFavoriteMethods(db),
|
||||
initialize: () => assertPostgresReady(db),
|
||||
close: () => db.close(),
|
||||
};
|
||||
}
|
||||
303
packages/data-schemas/src/postgres/favorite.spec.ts
Normal file
303
packages/data-schemas/src/postgres/favorite.spec.ts
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { FavoriteHarness } from '~/contracts/favorite';
|
||||
import type { FavoriteSnapshotRow } from './import';
|
||||
import type { PostgresDatabase } from './pool';
|
||||
import type { PostgresOptions } from './pool';
|
||||
import { importFavorites, readPostgresFavorites, fingerprintFavorites } from './import';
|
||||
import { tenantStorage, SYSTEM_TENANT_ID } from '~/config/tenantContext';
|
||||
import { migratePostgres, assertPostgresReady } from './migrations';
|
||||
import { describeFavoriteConformance } from '~/contracts/favorite';
|
||||
import { createPostgresTestDatabase } from './postgres.helper';
|
||||
import { createPostgresFavoriteMethods } from './favorite';
|
||||
import { resetTenantStrictCache } from '~/tenant/policy';
|
||||
import { createPostgresDatabase } from './pool';
|
||||
|
||||
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
let db: PostgresDatabase;
|
||||
const users = ['0123456789abcdef01234567', '1123456789abcdef01234567'] as const;
|
||||
let options: PostgresOptions;
|
||||
const harness: FavoriteHarness = {
|
||||
name: 'PostgreSQL 16',
|
||||
users,
|
||||
async setup() {
|
||||
({ db, options } = await createPostgresTestDatabase());
|
||||
await migratePostgres(db);
|
||||
return createPostgresFavoriteMethods(db);
|
||||
},
|
||||
async teardown() {
|
||||
await db?.close();
|
||||
},
|
||||
async reset() {
|
||||
jest.restoreAllMocks();
|
||||
await db.query('TRUNCATE tool_favorites,tool_favorite_erasure');
|
||||
await db.query(
|
||||
'UPDATE tool_favorite_readiness SET checkpoint=NULL, snapshot=NULL, verified=false, activated=false, imported=0',
|
||||
);
|
||||
},
|
||||
async seed(rows) {
|
||||
for (const row of rows) {
|
||||
await db.query(
|
||||
`INSERT INTO tool_favorites (id,user_id,item_type,item_id,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$5)`,
|
||||
[
|
||||
new mongoose.Types.ObjectId().toString(),
|
||||
row.userId,
|
||||
row.itemType,
|
||||
row.itemId,
|
||||
row.createdAt ?? new Date(),
|
||||
],
|
||||
);
|
||||
}
|
||||
},
|
||||
inTenant: (tenantId, operation) => tenantStorage.run({ tenantId }, operation),
|
||||
failNext(operation, error) {
|
||||
if (operation === 'read' || operation === 'remove') {
|
||||
jest.spyOn(db, 'query').mockRejectedValueOnce(error);
|
||||
} else {
|
||||
jest.spyOn(db, 'transaction').mockRejectedValueOnce(error);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
postgres('real PostgreSQL favorites', () => {
|
||||
describeFavoriteConformance(harness);
|
||||
describe('adapter, migrations, import, and readiness', () => {
|
||||
beforeAll(() => harness.setup());
|
||||
afterAll(() => harness.teardown());
|
||||
beforeEach(() => harness.reset());
|
||||
const source = async function* () {
|
||||
yield {
|
||||
id: '2123456789abcdef01234567',
|
||||
userId: users[0],
|
||||
tenantId: 'tenant-a',
|
||||
itemType: 'mcp',
|
||||
itemId: 'server',
|
||||
createdAt: new Date('2026-01-01'),
|
||||
updatedAt: new Date('2026-01-02'),
|
||||
} satisfies FavoriteSnapshotRow;
|
||||
};
|
||||
it('keeps schema migration idempotent, checksum-checked, and separate from data readiness', async () => {
|
||||
await migratePostgres(db);
|
||||
await expect(assertPostgresReady(db)).rejects.toThrow('not verified');
|
||||
await db.query("UPDATE librechat_migrations SET checksum='changed'");
|
||||
await expect(migratePostgres(db)).rejects.toThrow('checksum');
|
||||
// Restore through the exact original checksum, not by rerunning altered SQL.
|
||||
await db.query('DELETE FROM librechat_migrations');
|
||||
await db.query('DROP TABLE tool_favorites,tool_favorite_erasure,tool_favorite_readiness');
|
||||
await migratePostgres(db);
|
||||
});
|
||||
it('requires verified-empty approval, then activates and permanently refuses import replay', async () => {
|
||||
const empty = async function* () {
|
||||
/* verified empty source */
|
||||
};
|
||||
await expect(importFavorites(db, empty, { offline: true, batchSize: 2 })).rejects.toThrow(
|
||||
'verified-empty',
|
||||
);
|
||||
await importFavorites(db, empty, { offline: true, batchSize: 2, verifiedEmpty: true });
|
||||
await assertPostgresReady(db);
|
||||
await expect(
|
||||
importFavorites(db, empty, { offline: true, batchSize: 2, verifiedEmpty: true }),
|
||||
).rejects.toThrow('activated');
|
||||
});
|
||||
it('preserves source identity, tenant, timestamps, and supports idempotent checkpoint replay', async () => {
|
||||
await importFavorites(db, source, { offline: true, batchSize: 1 });
|
||||
await importFavorites(db, source, { offline: true, batchSize: 1 });
|
||||
expect(await fingerprintFavorites(readPostgresFavorites(db, 1))).toEqual(
|
||||
await fingerprintFavorites(source),
|
||||
);
|
||||
await assertPostgresReady(db);
|
||||
});
|
||||
it('does not write in dry-run and refuses a false empty-source assertion', async () => {
|
||||
await importFavorites(db, source, { offline: true, batchSize: 1, dryRun: true });
|
||||
expect((await db.query('SELECT * FROM tool_favorites')).rowCount).toBe(0);
|
||||
await expect(
|
||||
importFavorites(db, source, { offline: true, batchSize: 1, verifiedEmpty: true }),
|
||||
).rejects.toThrow('not empty');
|
||||
});
|
||||
it('rolls back a failed transaction', async () => {
|
||||
await expect(
|
||||
db.transaction(async (sql) => {
|
||||
await sql.query("INSERT INTO tool_favorite_erasure VALUES ($1,'*')", [users[0]]);
|
||||
throw new Error('fault');
|
||||
}),
|
||||
).rejects.toThrow('fault');
|
||||
expect((await db.query('SELECT * FROM tool_favorite_erasure')).rowCount).toBe(0);
|
||||
});
|
||||
it('denies missing strict context and distinguishes system all-tenants from tenantless', async () => {
|
||||
const methods = createPostgresFavoriteMethods(db);
|
||||
await harness.inTenant('tenant-a', () =>
|
||||
methods.addToolFavorite({ userId: users[0], itemType: 'skill', itemId: 'a' }),
|
||||
);
|
||||
expect(
|
||||
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
|
||||
methods.getToolFavorites(users[0]),
|
||||
),
|
||||
).toHaveLength(1);
|
||||
const old = process.env.TENANT_ISOLATION_STRICT;
|
||||
process.env.TENANT_ISOLATION_STRICT = 'true';
|
||||
resetTenantStrictCache();
|
||||
try {
|
||||
await expect(methods.getToolFavorites(users[0])).rejects.toThrow('without tenant context');
|
||||
} finally {
|
||||
resetTenantStrictCache();
|
||||
if (old == null) {
|
||||
delete process.env.TENANT_ISOLATION_STRICT;
|
||||
} else {
|
||||
process.env.TENANT_ISOLATION_STRICT = old;
|
||||
}
|
||||
}
|
||||
});
|
||||
it('accepts ObjectId input and prevents late recreation after erasure', async () => {
|
||||
const methods = createPostgresFavoriteMethods(db);
|
||||
const params = {
|
||||
userId: new mongoose.Types.ObjectId(users[0]),
|
||||
itemType: 'tool',
|
||||
itemId: 'x',
|
||||
} as const;
|
||||
await methods.addToolFavorite(params);
|
||||
await methods.deleteUserToolFavorites(users[0]);
|
||||
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
|
||||
});
|
||||
it('cancels only its UUID and scope, never another attempt or a permanent marker', async () => {
|
||||
const methods = createPostgresFavoriteMethods(db);
|
||||
const first = randomUUID();
|
||||
const second = randomUUID();
|
||||
const params = { userId: users[0], itemType: 'tool', itemId: 'retry' } as const;
|
||||
await harness.inTenant('tenant-a', async () => {
|
||||
await methods.deleteUserToolFavorites(users[0], first);
|
||||
await methods.deleteUserToolFavorites(users[0], second);
|
||||
});
|
||||
await harness.inTenant('tenant-b', () =>
|
||||
methods.cancelUserToolFavoriteDeletion!(users[0], first),
|
||||
);
|
||||
await harness.inTenant('tenant-a', async () => {
|
||||
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
|
||||
await methods.cancelUserToolFavoriteDeletion!(users[0], first);
|
||||
await methods.cancelUserToolFavoriteDeletion!(users[0], first);
|
||||
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
|
||||
await methods.cancelUserToolFavoriteDeletion!(users[0], second);
|
||||
await expect(methods.addToolFavorite(params)).resolves.toEqual({ ok: true, added: true });
|
||||
await methods.deleteUserToolFavorites(users[0]);
|
||||
await methods.cancelUserToolFavoriteDeletion!(users[0], first);
|
||||
await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure');
|
||||
});
|
||||
await expect(methods.cancelUserToolFavoriteDeletion!(users[0], '')).rejects.toThrow(
|
||||
'attempt',
|
||||
);
|
||||
await expect(methods.deleteUserToolFavorites(users[0], '')).rejects.toThrow('attempt');
|
||||
});
|
||||
it('lets system recovery cancel its UUID across scopes without clearing successors or permanent markers', async () => {
|
||||
const methods = createPostgresFavoriteMethods(db);
|
||||
const owned = randomUUID();
|
||||
const successor = randomUUID();
|
||||
for (const tenantId of ['tenant-a', 'tenant-b', SYSTEM_TENANT_ID]) {
|
||||
await tenantStorage.run({ tenantId }, () =>
|
||||
methods.deleteUserToolFavorites(users[0], owned),
|
||||
);
|
||||
}
|
||||
await harness.inTenant('tenant-b', () =>
|
||||
methods.deleteUserToolFavorites(users[0], successor),
|
||||
);
|
||||
await methods.deleteUserToolFavorites(users[1]);
|
||||
await harness.inTenant('tenant-a', () =>
|
||||
methods.cancelUserToolFavoriteDeletion!(users[0], owned),
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await db.query(
|
||||
'SELECT scope FROM tool_favorite_erasure WHERE user_id=$1 AND attempt_id=$2 ORDER BY scope',
|
||||
[users[0], owned],
|
||||
)
|
||||
).rows,
|
||||
).toEqual([{ scope: '*' }, { scope: 'tenant:tenant-b' }]);
|
||||
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
|
||||
methods.cancelUserToolFavoriteDeletion!(users[0], owned),
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await db.query(
|
||||
'SELECT scope,attempt_id FROM tool_favorite_erasure WHERE user_id=$1 ORDER BY scope',
|
||||
[users[0]],
|
||||
)
|
||||
).rows,
|
||||
).toEqual([{ scope: 'tenant:tenant-b', attempt_id: successor }]);
|
||||
expect(
|
||||
(
|
||||
await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [
|
||||
users[1],
|
||||
])
|
||||
).rows,
|
||||
).toEqual([{ attempt_id: '' }]);
|
||||
await harness.inTenant('tenant-b', () =>
|
||||
expect(
|
||||
methods.addToolFavorite({ userId: users[0], itemType: 'tool', itemId: 'still-fenced' }),
|
||||
).rejects.toThrow('erasure'),
|
||||
);
|
||||
});
|
||||
it('keeps global markers fenced against tenant cancellation and across fresh adapters', async () => {
|
||||
const methods = createPostgresFavoriteMethods(db);
|
||||
const attempt = randomUUID();
|
||||
const params = { userId: users[0], itemType: 'tool', itemId: 'global' } as const;
|
||||
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
|
||||
methods.deleteUserToolFavorites(users[0], attempt),
|
||||
);
|
||||
await harness.inTenant('tenant-a', () =>
|
||||
methods.cancelUserToolFavoriteDeletion!(users[0], attempt),
|
||||
);
|
||||
await harness.inTenant('tenant-b', () =>
|
||||
expect(createPostgresFavoriteMethods(db).addToolFavorite(params)).rejects.toThrow(
|
||||
'erasure',
|
||||
),
|
||||
);
|
||||
await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () =>
|
||||
methods.cancelUserToolFavoriteDeletion!(users[0], attempt),
|
||||
);
|
||||
await expect(methods.addToolFavorite(params)).resolves.toEqual({ ok: true, added: true });
|
||||
});
|
||||
it('bounds concurrent cap admission and enforces uniqueness across tenants', async () => {
|
||||
await harness.seed(
|
||||
Array.from({ length: 99 }, (_, i) => ({
|
||||
userId: users[0],
|
||||
itemType: 'tool',
|
||||
itemId: `${i}`,
|
||||
})),
|
||||
);
|
||||
const methods = createPostgresFavoriteMethods(db);
|
||||
await Promise.allSettled(
|
||||
Array.from({ length: 8 }, (_, i) =>
|
||||
methods.addToolFavorite({ userId: users[0], itemType: 'tool', itemId: `race-${i}` }),
|
||||
),
|
||||
);
|
||||
expect(await methods.getToolFavorites(users[0])).toHaveLength(100);
|
||||
});
|
||||
it('bounds query duration, releases failed transactions, and rejects work after close', async () => {
|
||||
const bounded = createPostgresDatabase({ ...options, queryTimeoutMs: 50 });
|
||||
try {
|
||||
await expect(
|
||||
bounded.transaction((sql) => sql.query('SELECT pg_sleep(1)')),
|
||||
).rejects.toThrow();
|
||||
expect((await bounded.query('SELECT 1 AS value')).rows).toEqual([{ value: 1 }]);
|
||||
} finally {
|
||||
await bounded.close();
|
||||
}
|
||||
await expect(bounded.query('SELECT 1')).rejects.toThrow('admission');
|
||||
});
|
||||
it('refuses concurrent import writers using the pinned session lock', async () => {
|
||||
const other = createPostgresDatabase(options);
|
||||
try {
|
||||
await other.session(async (sql) => {
|
||||
await sql.query('SELECT pg_advisory_lock(17490261, hashtext(current_schema()))');
|
||||
try {
|
||||
await expect(
|
||||
importFavorites(db, source, { offline: true, batchSize: 1 }),
|
||||
).rejects.toThrow('Another');
|
||||
} finally {
|
||||
await sql.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))');
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await other.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
121
packages/data-schemas/src/postgres/favorite.ts
Normal file
121
packages/data-schemas/src/postgres/favorite.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import { randomBytes } from 'node:crypto';
|
||||
import type { ToolFavoriteMethods, ToolFavoriteParams } from '~/types/favorite';
|
||||
import type { PostgresDatabase, SqlExecutor } from './pool';
|
||||
import { resolveTenantScope, tenantFilter, stampTenantOnDocument } from '~/tenant/policy';
|
||||
import { MAX_TOOL_FAVORITES } from '~/methods/favorite';
|
||||
|
||||
function userKey(value: ToolFavoriteParams['userId']): string {
|
||||
const user = String(value).toLowerCase();
|
||||
if (!/^[0-9a-f]{24}$/.test(user)) {
|
||||
throw new Error('Invalid favorite user ID');
|
||||
}
|
||||
return user;
|
||||
}
|
||||
|
||||
function validateAttemptId(attemptId: string): void {
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(attemptId)) {
|
||||
throw new Error('Invalid favorite deletion attempt ID');
|
||||
}
|
||||
}
|
||||
|
||||
export function createPostgresFavoriteMethods(db: PostgresDatabase): ToolFavoriteMethods {
|
||||
function scope(): string | null {
|
||||
return tenantFilter(resolveTenantScope('Query'))?.tenantId ?? null;
|
||||
}
|
||||
async function lock(sql: SqlExecutor, user: string): Promise<void> {
|
||||
await sql.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [user]);
|
||||
}
|
||||
return {
|
||||
async getToolFavorites(userId) {
|
||||
const result = await db.query<{ itemType: ToolFavoriteParams['itemType']; itemId: string }>(
|
||||
`SELECT item_type AS "itemType", item_id AS "itemId" FROM tool_favorites
|
||||
WHERE user_id = $1 AND ($2::text IS NULL OR tenant_id = $2) ORDER BY created_at, id`,
|
||||
[userKey(userId), scope()],
|
||||
);
|
||||
return result.rows;
|
||||
},
|
||||
async addToolFavorite({ userId, itemType, itemId }) {
|
||||
const user = userKey(userId);
|
||||
const tenant = scope();
|
||||
const document: { tenantId?: string } = {};
|
||||
stampTenantOnDocument(resolveTenantScope('Query'), document);
|
||||
if (
|
||||
!['builtin', 'tool', 'mcp', 'skill'].includes(itemType) ||
|
||||
typeof itemId !== 'string' ||
|
||||
itemId.length < 1 ||
|
||||
itemId.length > 256
|
||||
) {
|
||||
throw new Error('Invalid favorite item');
|
||||
}
|
||||
return db.transaction(async (sql) => {
|
||||
await lock(sql, user);
|
||||
const erased = await sql.query(
|
||||
'SELECT 1 FROM tool_favorite_erasure WHERE user_id = $1 AND (scope = $2 OR scope = $3) LIMIT 1',
|
||||
[user, '*', tenant == null ? '*' : `tenant:${tenant}`],
|
||||
);
|
||||
if (erased.rowCount) {
|
||||
throw new Error('Favorite account erasure in progress');
|
||||
}
|
||||
const exists = await sql.query(
|
||||
'SELECT 1 FROM tool_favorites WHERE user_id=$1 AND item_type=$2 AND item_id=$3 AND ($4::text IS NULL OR tenant_id=$4)',
|
||||
[user, itemType, itemId, tenant],
|
||||
);
|
||||
if (exists.rowCount) {
|
||||
return { ok: true, added: false };
|
||||
}
|
||||
const count = await sql.query<{ count: string }>(
|
||||
'SELECT count(*) FROM tool_favorites WHERE user_id=$1 AND ($2::text IS NULL OR tenant_id=$2)',
|
||||
[user, tenant],
|
||||
);
|
||||
if (Number(count.rows[0].count) >= MAX_TOOL_FAVORITES) {
|
||||
throw Object.assign(new Error(`Maximum of ${MAX_TOOL_FAVORITES} favorites reached`), {
|
||||
code: 'MAX_FAVORITES_EXCEEDED',
|
||||
limit: MAX_TOOL_FAVORITES,
|
||||
});
|
||||
}
|
||||
await sql.query(
|
||||
`INSERT INTO tool_favorites (id,user_id,tenant_id,item_type,item_id,created_at,updated_at)
|
||||
VALUES ($1,$2,$3,$4,$5,now(),now()) ON CONFLICT (user_id,item_type,item_id) DO NOTHING`,
|
||||
[randomBytes(12).toString('hex'), user, document.tenantId ?? null, itemType, itemId],
|
||||
);
|
||||
return { ok: true, added: true };
|
||||
});
|
||||
},
|
||||
async removeToolFavorite({ userId, itemType, itemId }) {
|
||||
const result = await db.query(
|
||||
'DELETE FROM tool_favorites WHERE user_id=$1 AND item_type=$2 AND item_id=$3 AND ($4::text IS NULL OR tenant_id=$4)',
|
||||
[userKey(userId), itemType, itemId, scope()],
|
||||
);
|
||||
return { ok: true, removed: (result.rowCount ?? 0) > 0 };
|
||||
},
|
||||
async cancelUserToolFavoriteDeletion(userId, deletionAttemptId) {
|
||||
validateAttemptId(deletionAttemptId);
|
||||
const user = userKey(userId);
|
||||
const tenant = scope();
|
||||
await db.transaction(async (sql) => {
|
||||
await lock(sql, user);
|
||||
await sql.query(
|
||||
'DELETE FROM tool_favorite_erasure WHERE user_id=$1 AND ($2::text IS NULL OR scope=$2) AND attempt_id=$3',
|
||||
[user, tenant == null ? null : `tenant:${tenant}`, deletionAttemptId],
|
||||
);
|
||||
});
|
||||
},
|
||||
async deleteUserToolFavorites(userId, deletionAttemptId) {
|
||||
if (deletionAttemptId !== undefined) validateAttemptId(deletionAttemptId);
|
||||
const user = userKey(userId);
|
||||
const tenant = scope();
|
||||
return db.transaction(async (sql) => {
|
||||
await lock(sql, user);
|
||||
await sql.query(
|
||||
'INSERT INTO tool_favorite_erasure (user_id,scope,attempt_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING',
|
||||
[user, tenant == null ? '*' : `tenant:${tenant}`, deletionAttemptId ?? ''],
|
||||
);
|
||||
const result = await sql.query(
|
||||
'DELETE FROM tool_favorites WHERE user_id=$1 AND ($2::text IS NULL OR tenant_id=$2)',
|
||||
[user, tenant],
|
||||
);
|
||||
return result.rowCount ?? 0;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
369
packages/data-schemas/src/postgres/import.spec.ts
Normal file
369
packages/data-schemas/src/postgres/import.spec.ts
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
import type { QueryResultRow } from 'pg';
|
||||
import type { FavoriteSnapshotRow, FavoriteSnapshotSource } from './import';
|
||||
import type { PostgresDatabase, SqlExecutor } from './pool';
|
||||
import { fingerprintFavorites, importFavorites, readPostgresFavorites } from './import';
|
||||
import { createPostgresTestDatabase } from './postgres.helper';
|
||||
import { migratePostgres } from './migrations';
|
||||
|
||||
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
const userId = '0123456789abcdef01234567';
|
||||
const options = { offline: true, batchSize: 2 };
|
||||
const rows: FavoriteSnapshotRow[] = Array.from({ length: 5 }, (_, i) => ({
|
||||
id: (i + 1).toString(16).padStart(24, '0'),
|
||||
userId,
|
||||
tenantId: i % 2 ? null : 'tenant-a',
|
||||
itemType: 'mcp',
|
||||
itemId: `server-${i}`,
|
||||
createdAt: new Date('2026-01-01T01:02:03.123Z'),
|
||||
updatedAt: new Date('2026-01-02T04:05:06.789Z'),
|
||||
}));
|
||||
const source = (data = rows): FavoriteSnapshotSource =>
|
||||
async function* () {
|
||||
yield* data;
|
||||
};
|
||||
|
||||
postgres('real PostgreSQL snapshot recovery', () => {
|
||||
let db: PostgresDatabase;
|
||||
beforeAll(async () => {
|
||||
({ db } = await createPostgresTestDatabase());
|
||||
await migratePostgres(db);
|
||||
});
|
||||
afterAll(async () => db?.close());
|
||||
beforeEach(async () => {
|
||||
await db.query('TRUNCATE tool_favorites,tool_favorite_erasure');
|
||||
await db.query(
|
||||
'UPDATE tool_favorite_readiness SET checkpoint=NULL,snapshot=NULL,verified=false,activated=false,imported=0',
|
||||
);
|
||||
});
|
||||
|
||||
function interruptAfter(prefix: string, occurrence: number): PostgresDatabase {
|
||||
let remaining = occurrence;
|
||||
return {
|
||||
...db,
|
||||
session: (operation) =>
|
||||
db.session((session) =>
|
||||
operation({
|
||||
...session,
|
||||
transaction: (work) =>
|
||||
session.transaction((sql) =>
|
||||
work({
|
||||
async query<R extends QueryResultRow>(text: string, values?: unknown[]) {
|
||||
const result = await sql.query<R>(text, values);
|
||||
if (text.startsWith(prefix) && --remaining === 0) {
|
||||
throw new Error('Injected interruption');
|
||||
}
|
||||
return result;
|
||||
},
|
||||
} satisfies SqlExecutor),
|
||||
),
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async function state() {
|
||||
return (
|
||||
await db.query(
|
||||
'SELECT checkpoint,snapshot,verified,activated,imported FROM tool_favorite_readiness',
|
||||
)
|
||||
).rows[0];
|
||||
}
|
||||
|
||||
it.each([
|
||||
['INSERT INTO tool_favorites', 1, 0],
|
||||
['INSERT INTO tool_favorites', 2, 2],
|
||||
['UPDATE tool_favorite_readiness SET checkpoint', 2, 2],
|
||||
])(
|
||||
'rolls back rows and checkpoint after %s interruption %i, then resumes',
|
||||
async (prefix, occurrence, committed) => {
|
||||
await expect(
|
||||
importFavorites(interruptAfter(prefix, occurrence), source(), options),
|
||||
).rejects.toThrow('Injected');
|
||||
expect((await db.query('SELECT id FROM tool_favorites ORDER BY id')).rows).toEqual(
|
||||
rows.slice(0, committed).map(({ id }) => ({ id })),
|
||||
);
|
||||
expect(await state()).toMatchObject({
|
||||
checkpoint: committed ? rows[committed - 1].id : null,
|
||||
verified: false,
|
||||
imported: '0',
|
||||
});
|
||||
await importFavorites(db, source(), options);
|
||||
await importFavorites(db, source(), { ...options, batchSize: 3 });
|
||||
expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual(
|
||||
await fingerprintFavorites(source()),
|
||||
);
|
||||
expect(await state()).toMatchObject({
|
||||
checkpoint: rows[4].id,
|
||||
verified: true,
|
||||
imported: '5',
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('resumes when the source iterator fails after a committed batch', async () => {
|
||||
let pass = 0;
|
||||
const interrupted: FavoriteSnapshotSource = async function* () {
|
||||
pass++;
|
||||
for (const row of rows) {
|
||||
yield row;
|
||||
if (pass === 2 && row.id === rows[1].id) throw new Error('Source interrupted');
|
||||
}
|
||||
};
|
||||
await expect(importFavorites(db, interrupted, options)).rejects.toThrow('Source interrupted');
|
||||
expect(await state()).toMatchObject({ checkpoint: rows[1].id, verified: false });
|
||||
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2);
|
||||
await importFavorites(db, source(), options);
|
||||
expect(await state()).toMatchObject({ verified: true, imported: '5' });
|
||||
});
|
||||
|
||||
it('keeps verification false when the source changes during the final verification pass', async () => {
|
||||
let pass = 0;
|
||||
const changing: FavoriteSnapshotSource = () =>
|
||||
source(++pass === 3 ? rows.map((row) => ({ ...row, tenantId: 'changed' })) : rows)();
|
||||
await expect(importFavorites(db, changing, options)).rejects.toThrow('verification failed');
|
||||
expect(await state()).toMatchObject({ verified: false });
|
||||
expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual(
|
||||
await fingerprintFavorites(source()),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'survives a final source scan beyond the idle transaction timeout (changed: %s)',
|
||||
async (changed) => {
|
||||
let pass = 0;
|
||||
const slow: FavoriteSnapshotSource = async function* () {
|
||||
if (++pass === 3) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
yield* changed ? rows.map((row) => ({ ...row, tenantId: 'changed' })) : rows;
|
||||
return;
|
||||
}
|
||||
yield* rows;
|
||||
};
|
||||
const shortTimeout: PostgresDatabase = {
|
||||
...db,
|
||||
session: (operation) =>
|
||||
db.session(async (session) => {
|
||||
await session.query("SET idle_in_transaction_session_timeout = '100ms'");
|
||||
try {
|
||||
return await operation(session);
|
||||
} finally {
|
||||
await session.query('RESET idle_in_transaction_session_timeout');
|
||||
}
|
||||
}),
|
||||
};
|
||||
if (changed) {
|
||||
await expect(importFavorites(shortTimeout, slow, options)).rejects.toThrow(
|
||||
'verification failed',
|
||||
);
|
||||
} else {
|
||||
await expect(importFavorites(shortTimeout, slow, options)).resolves.toEqual(
|
||||
await fingerprintFavorites(source()),
|
||||
);
|
||||
}
|
||||
expect(pass).toBe(3);
|
||||
expect(await state()).toMatchObject({ verified: !changed, imported: changed ? '0' : '5' });
|
||||
expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual(
|
||||
await fingerprintFavorites(source()),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('holds the pinned migration lock without a transaction during every source pass', async () => {
|
||||
let pid = 0;
|
||||
let passes = 0;
|
||||
const pinned: PostgresDatabase = {
|
||||
...db,
|
||||
session: (operation) =>
|
||||
db.session(async (session) => {
|
||||
pid = (await session.query<{ pid: number }>('SELECT pg_backend_pid() AS pid')).rows[0]
|
||||
.pid;
|
||||
return operation(session);
|
||||
}),
|
||||
};
|
||||
const checking: FavoriteSnapshotSource = async function* () {
|
||||
passes++;
|
||||
expect(
|
||||
(await db.query('SELECT state FROM pg_stat_activity WHERE pid=$1', [pid])).rows,
|
||||
).toEqual([{ state: 'idle' }]);
|
||||
await expect(importFavorites(db, source(), options)).rejects.toThrow(
|
||||
'Another favorite migration or import is running',
|
||||
);
|
||||
yield* rows;
|
||||
};
|
||||
await importFavorites(pinned, checking, options);
|
||||
expect(passes).toBe(3);
|
||||
expect(await state()).toMatchObject({ verified: true, imported: '5' });
|
||||
await importFavorites(db, source(), options);
|
||||
});
|
||||
|
||||
it.each(['activated', 'snapshot'])(
|
||||
'refuses %s authority change between batches',
|
||||
async (change) => {
|
||||
let pass = 0;
|
||||
const changing: FavoriteSnapshotSource = async function* () {
|
||||
pass++;
|
||||
for (const row of rows) {
|
||||
yield row;
|
||||
if (pass !== 2 || row.id !== rows[1].id) continue;
|
||||
if (change === 'activated')
|
||||
await db.query('UPDATE tool_favorite_readiness SET activated=true');
|
||||
else await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'");
|
||||
}
|
||||
};
|
||||
await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed');
|
||||
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2);
|
||||
expect(await state()).toMatchObject({ checkpoint: rows[1].id, verified: false });
|
||||
},
|
||||
);
|
||||
|
||||
it('preserves canonical identity, tenant and millisecond timestamps exactly', async () => {
|
||||
await importFavorites(db, source(), options);
|
||||
const actual: FavoriteSnapshotRow[] = [];
|
||||
for await (const row of readPostgresFavorites(db, 2)()) actual.push(row);
|
||||
expect(actual).toEqual(rows);
|
||||
});
|
||||
|
||||
it('refuses a changed fingerprint on resume without modifying committed rows', async () => {
|
||||
await expect(
|
||||
importFavorites(interruptAfter('INSERT INTO tool_favorites', 2), source(), options),
|
||||
).rejects.toThrow('Injected');
|
||||
const before = await state();
|
||||
const changed = rows.map((row, i) => (i ? row : { ...row, tenantId: 'changed' }));
|
||||
await expect(importFavorites(db, source(changed), options)).rejects.toThrow(
|
||||
'different snapshot',
|
||||
);
|
||||
expect(await state()).toEqual(before);
|
||||
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2);
|
||||
});
|
||||
|
||||
it('refuses duplicate logical favorites instead of silently checkpointing them', async () => {
|
||||
const duplicate = { ...rows[1], itemId: rows[0].itemId };
|
||||
await expect(importFavorites(db, source([rows[0], duplicate]), options)).rejects.toThrow(
|
||||
'conflicts',
|
||||
);
|
||||
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0);
|
||||
expect(await state()).toMatchObject({ checkpoint: null, verified: false });
|
||||
});
|
||||
|
||||
it('refuses a destination mapping collision and preserves unrelated rows', async () => {
|
||||
await db.query('INSERT INTO tool_favorites VALUES ($1,$2,$3,$4,$5,$6,$7)', [
|
||||
rows[0].id,
|
||||
userId,
|
||||
'other-tenant',
|
||||
'tool',
|
||||
'unrelated',
|
||||
rows[0].createdAt,
|
||||
rows[0].updatedAt,
|
||||
]);
|
||||
await expect(importFavorites(db, source(), options)).rejects.toThrow('conflicts');
|
||||
expect((await db.query('SELECT tenant_id,item_id FROM tool_favorites')).rows).toEqual([
|
||||
{ tenant_id: 'other-tenant', item_id: 'unrelated' },
|
||||
]);
|
||||
expect(await state()).toMatchObject({ checkpoint: null, verified: false });
|
||||
});
|
||||
|
||||
it('detects a corrupted checkpoint prefix without overwriting it', async () => {
|
||||
await expect(
|
||||
importFavorites(interruptAfter('INSERT INTO tool_favorites', 2), source(), options),
|
||||
).rejects.toThrow('Injected');
|
||||
await db.query('UPDATE tool_favorites SET tenant_id=$1 WHERE id=$2', ['wrong', rows[0].id]);
|
||||
await expect(importFavorites(db, source(), options)).rejects.toThrow('verification failed');
|
||||
expect(await state()).toMatchObject({ verified: false });
|
||||
expect(
|
||||
(await db.query('SELECT tenant_id FROM tool_favorites WHERE id=$1', [rows[0].id])).rows[0]
|
||||
.tenant_id,
|
||||
).toBe('wrong');
|
||||
});
|
||||
|
||||
it.each(['activated', 'snapshot', 'missing'])(
|
||||
'refuses %s authority change after the last batch',
|
||||
async (change) => {
|
||||
let pass = 0;
|
||||
const changing: FavoriteSnapshotSource = async function* () {
|
||||
pass++;
|
||||
yield* rows.slice(0, 2);
|
||||
if (pass !== 2) return;
|
||||
if (change === 'activated')
|
||||
await db.query('UPDATE tool_favorite_readiness SET activated=true');
|
||||
if (change === 'snapshot')
|
||||
await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'");
|
||||
if (change === 'missing') await db.query('DELETE FROM tool_favorite_readiness');
|
||||
};
|
||||
try {
|
||||
await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed');
|
||||
if (change !== 'missing') expect(await state()).toMatchObject({ verified: false });
|
||||
} finally {
|
||||
if (change === 'missing')
|
||||
await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['activated', 'snapshot', 'missing'])(
|
||||
'refuses %s authority change during the final source pass',
|
||||
async (change) => {
|
||||
let pass = 0;
|
||||
const changing: FavoriteSnapshotSource = async function* () {
|
||||
if (++pass === 3) {
|
||||
if (change === 'activated')
|
||||
await db.query('UPDATE tool_favorite_readiness SET activated=true');
|
||||
if (change === 'snapshot')
|
||||
await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'");
|
||||
if (change === 'missing') await db.query('DELETE FROM tool_favorite_readiness');
|
||||
}
|
||||
yield* rows;
|
||||
};
|
||||
try {
|
||||
await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed');
|
||||
if (change !== 'missing') expect(await state()).toMatchObject({ verified: false });
|
||||
} finally {
|
||||
if (change === 'missing')
|
||||
await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('refuses a changed middle source pass even when the final source would match', async () => {
|
||||
let pass = 0;
|
||||
const changing: FavoriteSnapshotSource = () => source(++pass === 2 ? rows.slice(0, 2) : rows)();
|
||||
await expect(importFavorites(db, changing, options)).rejects.toThrow('source changed');
|
||||
expect(await state()).toMatchObject({ verified: false });
|
||||
});
|
||||
|
||||
it('refuses unsorted rows in the write pass', async () => {
|
||||
let pass = 0;
|
||||
const changing: FavoriteSnapshotSource = () =>
|
||||
source(++pass === 2 ? [rows[1], rows[0]] : rows)();
|
||||
await expect(importFavorites(db, changing, options)).rejects.toThrow('uniquely ordered');
|
||||
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ userId: 'not-canonical' },
|
||||
{ id: 'bad-id' },
|
||||
{ itemType: 'unknown' },
|
||||
{ itemId: 42 },
|
||||
{ itemId: '' },
|
||||
{ tenantId: 12 },
|
||||
{ createdAt: new Date('invalid') },
|
||||
{ updatedAt: 'not-a-date' },
|
||||
])('refuses malformed input before claiming a snapshot: %p', async (invalid) => {
|
||||
const malformed = { ...rows[0], ...invalid } as FavoriteSnapshotRow;
|
||||
await expect(importFavorites(db, source([malformed]), options)).rejects.toThrow(
|
||||
'Invalid favorite snapshot record',
|
||||
);
|
||||
expect(await state()).toMatchObject({ snapshot: null, checkpoint: null, verified: false });
|
||||
expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0);
|
||||
});
|
||||
|
||||
it('refuses duplicate IDs before claiming a snapshot', async () => {
|
||||
await expect(importFavorites(db, source([rows[0], rows[0]]), options)).rejects.toThrow(
|
||||
'uniquely ordered',
|
||||
);
|
||||
expect(await state()).toMatchObject({ snapshot: null });
|
||||
});
|
||||
|
||||
it('rejects a zero-sized PostgreSQL reader rather than fingerprinting a false empty source', () => {
|
||||
expect(() => readPostgresFavorites(db, 0)).toThrow('batch size');
|
||||
});
|
||||
});
|
||||
243
packages/data-schemas/src/postgres/import.ts
Normal file
243
packages/data-schemas/src/postgres/import.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type { PostgresDatabase, PostgresSession, SqlExecutor } from './pool';
|
||||
import type { FavoriteItemType } from '~/types/favorite';
|
||||
import { lockFavoriteMigration } from './migrations';
|
||||
|
||||
export interface FavoriteSnapshotRow {
|
||||
id: string;
|
||||
userId: string;
|
||||
tenantId: string | null;
|
||||
itemType: FavoriteItemType;
|
||||
itemId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
export type FavoriteSnapshotSource = () => AsyncIterable<FavoriteSnapshotRow>;
|
||||
|
||||
function encoded(row: FavoriteSnapshotRow): string {
|
||||
if (
|
||||
typeof row.id !== 'string' ||
|
||||
!/^[0-9a-f]{24}$/.test(row.id) ||
|
||||
typeof row.userId !== 'string' ||
|
||||
!/^[0-9a-f]{24}$/.test(row.userId) ||
|
||||
!['builtin', 'tool', 'mcp', 'skill'].includes(row.itemType) ||
|
||||
typeof row.itemId !== 'string' ||
|
||||
!row.itemId ||
|
||||
row.itemId.length > 256 ||
|
||||
(row.tenantId !== null && typeof row.tenantId !== 'string') ||
|
||||
!(row.createdAt instanceof Date) ||
|
||||
!Number.isFinite(row.createdAt.getTime()) ||
|
||||
!(row.updatedAt instanceof Date) ||
|
||||
!Number.isFinite(row.updatedAt.getTime())
|
||||
) {
|
||||
throw new Error('Invalid favorite snapshot record');
|
||||
}
|
||||
return JSON.stringify([
|
||||
row.id,
|
||||
row.userId,
|
||||
row.tenantId,
|
||||
row.itemType,
|
||||
row.itemId,
|
||||
row.createdAt.toISOString(),
|
||||
row.updatedAt.toISOString(),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function fingerprintFavorites(
|
||||
source: FavoriteSnapshotSource,
|
||||
): Promise<{ count: number; digest: string }> {
|
||||
const hash = createHash('sha256');
|
||||
let count = 0;
|
||||
let previous = '';
|
||||
for await (const row of source()) {
|
||||
if (row.id <= previous) {
|
||||
throw new Error('Favorite snapshot must be uniquely ordered by ID');
|
||||
}
|
||||
hash.update(encoded(row)).update('\n');
|
||||
previous = row.id;
|
||||
count++;
|
||||
}
|
||||
return { count, digest: hash.digest('hex') };
|
||||
}
|
||||
|
||||
export function readPostgresFavorites(db: SqlExecutor, batchSize: number): FavoriteSnapshotSource {
|
||||
if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1000) {
|
||||
throw new Error('Favorite snapshot batch size must be between 1 and 1000');
|
||||
}
|
||||
return async function* () {
|
||||
let after = '';
|
||||
for (;;) {
|
||||
const result = await db.query<FavoriteSnapshotRow>(
|
||||
`SELECT id, user_id AS "userId", tenant_id AS "tenantId",
|
||||
item_type AS "itemType", item_id AS "itemId", created_at AS "createdAt", updated_at AS "updatedAt"
|
||||
FROM tool_favorites WHERE id > $1 ORDER BY id LIMIT $2`,
|
||||
[after, batchSize],
|
||||
);
|
||||
if (!result.rows.length) {
|
||||
return;
|
||||
}
|
||||
for (const row of result.rows) {
|
||||
yield row;
|
||||
after = row.id;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Source and destination writers must be stopped. Activation permanently blocks replay. */
|
||||
interface ImportOptions {
|
||||
offline: boolean;
|
||||
batchSize: number;
|
||||
dryRun?: boolean;
|
||||
verifiedEmpty?: boolean;
|
||||
}
|
||||
|
||||
export async function importFavorites(
|
||||
db: PostgresDatabase,
|
||||
source: FavoriteSnapshotSource,
|
||||
options: ImportOptions,
|
||||
): Promise<{ count: number; digest: string }> {
|
||||
return db.session(async (session) => {
|
||||
const lock = await session.query<{ acquired: boolean }>(
|
||||
'SELECT pg_try_advisory_lock(17490261, hashtext(current_schema())) AS acquired',
|
||||
);
|
||||
if (!lock.rows[0].acquired) {
|
||||
throw new Error('Another favorite migration or import is running');
|
||||
}
|
||||
try {
|
||||
return await importSnapshot(session, source, options);
|
||||
} finally {
|
||||
await session.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function importSnapshot(
|
||||
db: PostgresSession,
|
||||
source: FavoriteSnapshotSource,
|
||||
options: ImportOptions,
|
||||
): Promise<{ count: number; digest: string }> {
|
||||
if (
|
||||
!options.offline ||
|
||||
!Number.isInteger(options.batchSize) ||
|
||||
options.batchSize < 1 ||
|
||||
options.batchSize > 1000
|
||||
) {
|
||||
throw new Error('Offline assertion and batch size between 1 and 1000 are required');
|
||||
}
|
||||
const snapshot = await fingerprintFavorites(source);
|
||||
if (snapshot.count === 0 && !options.verifiedEmpty) {
|
||||
throw new Error('Empty source requires explicit verified-empty approval');
|
||||
}
|
||||
if (options.verifiedEmpty && snapshot.count !== 0) {
|
||||
throw new Error('Verified-empty refused: Mongo favorites are not empty');
|
||||
}
|
||||
if (options.dryRun) {
|
||||
return snapshot;
|
||||
}
|
||||
await db.transaction(async (sql) => {
|
||||
await lockFavoriteMigration(sql);
|
||||
const result = await sql.query<{ activated: boolean; snapshot: string | null }>(
|
||||
'SELECT activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE',
|
||||
);
|
||||
if (
|
||||
!result.rows[0] ||
|
||||
result.rows[0].activated ||
|
||||
(result.rows[0].snapshot && result.rows[0].snapshot !== snapshot.digest)
|
||||
) {
|
||||
throw new Error('Favorite import cannot replace an activated or different snapshot');
|
||||
}
|
||||
await sql.query(
|
||||
'UPDATE tool_favorite_readiness SET snapshot=$1, verified=false WHERE singleton',
|
||||
[snapshot.digest],
|
||||
);
|
||||
});
|
||||
let batch: FavoriteSnapshotRow[] = [];
|
||||
const write = async () => {
|
||||
await db.transaction(async (sql) => {
|
||||
await lockFavoriteMigration(sql);
|
||||
const state = await sql.query<{
|
||||
checkpoint: string | null;
|
||||
activated: boolean;
|
||||
snapshot: string;
|
||||
}>(
|
||||
'SELECT checkpoint, activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE',
|
||||
);
|
||||
if (!state.rows[0] || state.rows[0].activated || state.rows[0].snapshot !== snapshot.digest) {
|
||||
throw new Error('Favorite import authority changed');
|
||||
}
|
||||
const pending = batch.filter((row) => row.id > (state.rows[0].checkpoint ?? ''));
|
||||
if (pending.length) {
|
||||
const inserted = await sql.query(
|
||||
`INSERT INTO tool_favorites (id,user_id,tenant_id,item_type,item_id,created_at,updated_at)
|
||||
SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::text[],$5::text[],$6::timestamptz[],$7::timestamptz[])
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[
|
||||
pending.map((row) => row.id),
|
||||
pending.map((row) => row.userId),
|
||||
pending.map((row) => row.tenantId),
|
||||
pending.map((row) => row.itemType),
|
||||
pending.map((row) => row.itemId),
|
||||
pending.map((row) => row.createdAt),
|
||||
pending.map((row) => row.updatedAt),
|
||||
],
|
||||
);
|
||||
if (inserted.rowCount !== pending.length) {
|
||||
throw new Error('Favorite snapshot conflicts with destination mapping');
|
||||
}
|
||||
}
|
||||
await sql.query(
|
||||
"UPDATE tool_favorite_readiness SET checkpoint=GREATEST(COALESCE(checkpoint,''),$1) WHERE singleton",
|
||||
[batch[batch.length - 1].id],
|
||||
);
|
||||
});
|
||||
batch = [];
|
||||
};
|
||||
const written = createHash('sha256');
|
||||
let count = 0;
|
||||
let previous = '';
|
||||
for await (const row of source()) {
|
||||
if (row.id <= previous) {
|
||||
throw new Error('Favorite snapshot must be uniquely ordered by ID');
|
||||
}
|
||||
written.update(encoded(row)).update('\n');
|
||||
previous = row.id;
|
||||
count++;
|
||||
batch.push(row);
|
||||
if (batch.length === options.batchSize) {
|
||||
await write();
|
||||
}
|
||||
}
|
||||
if (batch.length) {
|
||||
await write();
|
||||
}
|
||||
if (count !== snapshot.count || written.digest('hex') !== snapshot.digest) {
|
||||
throw new Error('Favorite source changed during import');
|
||||
}
|
||||
const current = await fingerprintFavorites(source);
|
||||
await db.transaction(async (sql) => {
|
||||
await lockFavoriteMigration(sql);
|
||||
const state = await sql.query<{ activated: boolean; snapshot: string | null }>(
|
||||
'SELECT activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE',
|
||||
);
|
||||
if (!state.rows[0] || state.rows[0].activated || state.rows[0].snapshot !== snapshot.digest) {
|
||||
throw new Error('Favorite import authority changed');
|
||||
}
|
||||
const destination = await fingerprintFavorites(readPostgresFavorites(sql, options.batchSize));
|
||||
if (
|
||||
current.digest !== snapshot.digest ||
|
||||
destination.digest !== snapshot.digest ||
|
||||
destination.count !== snapshot.count
|
||||
) {
|
||||
throw new Error('Favorite snapshot verification failed');
|
||||
}
|
||||
const verified = await sql.query(
|
||||
'UPDATE tool_favorite_readiness SET verified=true, imported=$1 WHERE singleton AND NOT activated AND snapshot=$2',
|
||||
[snapshot.count, snapshot.digest],
|
||||
);
|
||||
if (verified.rowCount !== 1) {
|
||||
throw new Error('Favorite import authority changed');
|
||||
}
|
||||
});
|
||||
return snapshot;
|
||||
}
|
||||
98
packages/data-schemas/src/postgres/migrations.spec.ts
Normal file
98
packages/data-schemas/src/postgres/migrations.spec.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import { randomBytes } from 'node:crypto';
|
||||
import type { PostgresDatabase } from './pool';
|
||||
import { migratePostgres, assertPostgresReady } from './migrations';
|
||||
import { createPostgresTestDatabase } from './postgres.helper';
|
||||
import { importFavorites } from './import';
|
||||
|
||||
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
postgres('PostgreSQL catalog readiness', () => {
|
||||
let db: PostgresDatabase;
|
||||
let schema: string;
|
||||
beforeAll(async () => {
|
||||
({ db, schema } = await createPostgresTestDatabase());
|
||||
});
|
||||
afterAll(() => db?.close());
|
||||
beforeEach(async () => {
|
||||
await db.query(
|
||||
'DROP TABLE IF EXISTS tool_favorites,tool_favorite_erasure,tool_favorite_readiness,librechat_migrations',
|
||||
);
|
||||
await migratePostgres(db);
|
||||
await importFavorites(db, async function* () {}, {
|
||||
offline: true,
|
||||
batchSize: 10,
|
||||
verifiedEmpty: true,
|
||||
});
|
||||
});
|
||||
it.each([
|
||||
'DROP INDEX tool_favorites_user_created',
|
||||
'ALTER TABLE tool_favorites DROP CONSTRAINT tool_favorites_user_id_item_type_item_id_key',
|
||||
'ALTER TABLE tool_favorites DROP CONSTRAINT tool_favorites_item_id_check',
|
||||
'ALTER TABLE tool_favorites ALTER COLUMN item_id DROP NOT NULL',
|
||||
'ALTER TABLE tool_favorites ENABLE ROW LEVEL SECURITY',
|
||||
'DROP TABLE tool_favorite_erasure',
|
||||
])('refuses schema drift before activation: %s', async (statement) => {
|
||||
await db.query(statement);
|
||||
await expect(assertPostgresReady(db)).rejects.toThrow('schema');
|
||||
expect((await db.query('SELECT activated FROM tool_favorite_readiness')).rows).toEqual([
|
||||
{ activated: false },
|
||||
]);
|
||||
});
|
||||
it('supports concurrent idempotent migration runners', async () => {
|
||||
await Promise.all([migratePostgres(db), migratePostgres(db), migratePostgres(db)]);
|
||||
expect((await db.query('SELECT version FROM librechat_migrations')).rows).toEqual([
|
||||
{ version: 1 },
|
||||
]);
|
||||
await assertPostgresReady(db);
|
||||
});
|
||||
it('does not turn nested work into a premature commit', async () => {
|
||||
await expect(
|
||||
db.session(async (session) =>
|
||||
session.transaction(async (sql) => {
|
||||
await sql.query("INSERT INTO tool_favorite_erasure VALUES ('nested','*')");
|
||||
await session.transaction(async () => {});
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow('Nested');
|
||||
expect((await db.query('SELECT * FROM tool_favorite_erasure')).rowCount).toBe(0);
|
||||
});
|
||||
it('validates runtime DML privileges rather than requiring migration-owner permissions', async () => {
|
||||
const role = `favorites_role_${randomBytes(8).toString('hex')}`;
|
||||
await db.query(`DROP ROLE IF EXISTS ${role}`);
|
||||
await db.query(`CREATE ROLE ${role}`);
|
||||
try {
|
||||
await db.query(`GRANT USAGE ON SCHEMA ${schema} TO ${role}`);
|
||||
await db.query(`GRANT SELECT ON librechat_migrations TO ${role}`);
|
||||
await db.query(
|
||||
`GRANT SELECT, INSERT, DELETE ON tool_favorites,tool_favorite_erasure,tool_favorite_readiness TO ${role}`,
|
||||
);
|
||||
await db.query(`GRANT UPDATE ON tool_favorite_readiness TO ${role}`);
|
||||
const runAsRole = (operation: (database: PostgresDatabase) => Promise<void>) =>
|
||||
db.session(async (session) => {
|
||||
await session.query(`SET ROLE ${role}`);
|
||||
try {
|
||||
const selected: PostgresDatabase = {
|
||||
...db,
|
||||
query: session.query,
|
||||
transaction: session.transaction,
|
||||
};
|
||||
await operation(selected);
|
||||
} finally {
|
||||
await session.query('RESET ROLE');
|
||||
}
|
||||
});
|
||||
await runAsRole((database) => assertPostgresReady(database));
|
||||
await db.query(`REVOKE DELETE ON tool_favorite_erasure FROM ${role}`);
|
||||
await runAsRole(async (database) => {
|
||||
await expect(assertPostgresReady(database)).rejects.toThrow('permissions');
|
||||
});
|
||||
await db.query(`GRANT DELETE ON tool_favorite_erasure TO ${role}`);
|
||||
await db.query(`REVOKE INSERT ON tool_favorites FROM ${role}`);
|
||||
await runAsRole(async (database) => {
|
||||
await expect(assertPostgresReady(database)).rejects.toThrow('permissions');
|
||||
});
|
||||
} finally {
|
||||
await db.query(`DROP OWNED BY ${role}`);
|
||||
await db.query(`DROP ROLE ${role}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
158
packages/data-schemas/src/postgres/migrations.ts
Normal file
158
packages/data-schemas/src/postgres/migrations.ts
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type { PostgresDatabase, SqlExecutor } from './pool';
|
||||
|
||||
const migration = {
|
||||
version: 1,
|
||||
sql: `CREATE TABLE tool_favorites (
|
||||
id text PRIMARY KEY,
|
||||
user_id text NOT NULL CHECK (user_id ~ '^[0-9a-f]{24}$'),
|
||||
tenant_id text,
|
||||
item_type text NOT NULL CHECK (item_type IN ('builtin', 'tool', 'mcp', 'skill')),
|
||||
item_id text NOT NULL CHECK (char_length(item_id) BETWEEN 1 AND 256),
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
UNIQUE (user_id, item_type, item_id)
|
||||
);
|
||||
CREATE INDEX tool_favorites_user_created ON tool_favorites (user_id, created_at, id);
|
||||
CREATE INDEX tool_favorites_tenant_user ON tool_favorites (tenant_id, user_id);
|
||||
CREATE TABLE tool_favorite_erasure (
|
||||
user_id text NOT NULL,
|
||||
scope text NOT NULL,
|
||||
attempt_id text NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (user_id, scope, attempt_id)
|
||||
);
|
||||
CREATE TABLE tool_favorite_readiness (
|
||||
singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
|
||||
checkpoint text,
|
||||
verified boolean NOT NULL DEFAULT false,
|
||||
activated boolean NOT NULL DEFAULT false,
|
||||
snapshot text,
|
||||
imported bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT INTO tool_favorite_readiness (singleton) VALUES (true);`,
|
||||
};
|
||||
const checksum = createHash('sha256').update(migration.sql).digest('hex');
|
||||
|
||||
export async function lockFavoriteMigration(sql: SqlExecutor): Promise<void> {
|
||||
await sql.query('SELECT pg_advisory_xact_lock(17490261, hashtext(current_schema()))');
|
||||
}
|
||||
|
||||
async function assertVersion(sql: SqlExecutor): Promise<void> {
|
||||
const result = await sql.query<{ version: string }>(
|
||||
"SELECT current_setting('server_version_num') AS version",
|
||||
);
|
||||
if (Number(result.rows[0].version) < 160000 || Number(result.rows[0].version) >= 170000) {
|
||||
throw new Error('PostgreSQL 16 is required');
|
||||
}
|
||||
}
|
||||
|
||||
const tables = ['tool_favorites', 'tool_favorite_erasure', 'tool_favorite_readiness'];
|
||||
|
||||
/** Compare deployed columns, constraints and indexes, not only the applied-version marker. */
|
||||
async function schemaChecksum(sql: SqlExecutor): Promise<string> {
|
||||
const catalog = await sql.query<{ definition: unknown }>(
|
||||
`SELECT jsonb_build_object(
|
||||
'name', expected.name, 'kind', relation.relkind,
|
||||
'rowSecurity', relation.relrowsecurity, 'forceRowSecurity', relation.relforcerowsecurity,
|
||||
'columns', (SELECT jsonb_agg(jsonb_build_array(attribute.attname,
|
||||
format_type(attribute.atttypid, attribute.atttypmod), attribute.attnotnull,
|
||||
pg_get_expr(defaults.adbin, defaults.adrelid), attribute.attidentity, attribute.attgenerated)
|
||||
ORDER BY attribute.attnum)
|
||||
FROM pg_attribute attribute LEFT JOIN pg_attrdef defaults
|
||||
ON defaults.adrelid=attribute.attrelid AND defaults.adnum=attribute.attnum
|
||||
WHERE attribute.attrelid=relation.oid AND attribute.attnum>0 AND NOT attribute.attisdropped),
|
||||
'constraints', (SELECT jsonb_agg(jsonb_build_array(constraint_name.conname,
|
||||
pg_get_constraintdef(constraint_name.oid), constraint_name.convalidated)
|
||||
ORDER BY constraint_name.conname)
|
||||
FROM pg_constraint constraint_name WHERE constraint_name.conrelid=relation.oid),
|
||||
'indexes', (SELECT jsonb_agg(jsonb_build_array(pg_get_indexdef(index_definition.indexrelid),
|
||||
index_definition.indisvalid, index_definition.indisready) ORDER BY pg_get_indexdef(index_definition.indexrelid))
|
||||
FROM pg_index index_definition WHERE index_definition.indrelid=relation.oid),
|
||||
'triggers', (SELECT jsonb_agg(pg_get_triggerdef(trigger_definition.oid)
|
||||
ORDER BY trigger_definition.tgname) FROM pg_trigger trigger_definition
|
||||
WHERE trigger_definition.tgrelid=relation.oid AND NOT trigger_definition.tgisinternal),
|
||||
'policies', (SELECT jsonb_agg(jsonb_build_array(policy.polname, policy.polcmd,
|
||||
policy.polpermissive, policy.polroles, pg_get_expr(policy.polqual, policy.polrelid),
|
||||
pg_get_expr(policy.polwithcheck, policy.polrelid)) ORDER BY policy.polname)
|
||||
FROM pg_policy policy WHERE policy.polrelid=relation.oid)
|
||||
) AS definition FROM unnest($1::text[]) AS expected(name)
|
||||
LEFT JOIN pg_class relation ON relation.oid=to_regclass(expected.name)
|
||||
ORDER BY expected.name`,
|
||||
[tables],
|
||||
);
|
||||
return createHash('sha256').update(JSON.stringify(catalog.rows)).digest('hex');
|
||||
}
|
||||
|
||||
async function assertSchema(sql: SqlExecutor): Promise<void> {
|
||||
const applied = await sql.query<{ version: number; checksum: string; schema_checksum: string }>(
|
||||
'SELECT version, checksum, schema_checksum FROM librechat_migrations ORDER BY version',
|
||||
);
|
||||
if (
|
||||
applied.rows.length !== 1 ||
|
||||
applied.rows[0].version !== migration.version ||
|
||||
applied.rows[0].checksum !== checksum ||
|
||||
applied.rows[0].schema_checksum !== (await schemaChecksum(sql))
|
||||
) {
|
||||
throw new Error('PostgreSQL schema checksum or version mismatch');
|
||||
}
|
||||
const privileges = await sql.query<{ allowed: boolean }>(
|
||||
`SELECT has_table_privilege(current_user, 'tool_favorites', 'SELECT')
|
||||
AND has_table_privilege(current_user, 'tool_favorites', 'INSERT')
|
||||
AND has_table_privilege(current_user, 'tool_favorites', 'DELETE')
|
||||
AND has_table_privilege(current_user, 'tool_favorite_erasure', 'SELECT')
|
||||
AND has_table_privilege(current_user, 'tool_favorite_erasure', 'INSERT')
|
||||
AND has_table_privilege(current_user, 'tool_favorite_erasure', 'DELETE')
|
||||
AND has_table_privilege(current_user, 'tool_favorite_readiness', 'SELECT')
|
||||
AND has_table_privilege(current_user, 'tool_favorite_readiness', 'UPDATE') AS allowed`,
|
||||
);
|
||||
if (!privileges.rows[0]?.allowed) throw new Error('PostgreSQL runtime permissions not ready');
|
||||
}
|
||||
|
||||
export async function migratePostgres(db: PostgresDatabase): Promise<void> {
|
||||
await db.transaction(async (sql) => {
|
||||
await assertVersion(sql);
|
||||
await lockFavoriteMigration(sql);
|
||||
await sql.query(`CREATE TABLE IF NOT EXISTS librechat_migrations (
|
||||
version integer PRIMARY KEY, checksum text NOT NULL, schema_checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)`);
|
||||
const result = await sql.query<{ version: number; checksum: string }>(
|
||||
'SELECT version, checksum FROM librechat_migrations ORDER BY version',
|
||||
);
|
||||
if (result.rows.length) {
|
||||
if (
|
||||
result.rows.length !== 1 ||
|
||||
result.rows[0].version !== migration.version ||
|
||||
result.rows[0].checksum !== checksum
|
||||
) {
|
||||
throw new Error('PostgreSQL migration checksum or version mismatch');
|
||||
}
|
||||
await assertSchema(sql);
|
||||
return;
|
||||
}
|
||||
await sql.query(migration.sql);
|
||||
await sql.query(
|
||||
'INSERT INTO librechat_migrations (version, checksum, schema_checksum) VALUES ($1, $2, $3)',
|
||||
[migration.version, checksum, await schemaChecksum(sql)],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Readiness check without selecting an authority or changing its activation marker. */
|
||||
export async function assertPostgresDataReady(sql: SqlExecutor): Promise<void> {
|
||||
await assertVersion(sql);
|
||||
await assertSchema(sql);
|
||||
const ready = await sql.query<{ verified: boolean; snapshot: string | null }>(
|
||||
'SELECT verified, snapshot FROM tool_favorite_readiness WHERE singleton',
|
||||
);
|
||||
if (ready.rows[0]?.verified !== true || !/^[a-f0-9]{64}$/.test(ready.rows[0].snapshot ?? '')) {
|
||||
throw new Error('PostgreSQL favorites data not verified');
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertPostgresReady(db: PostgresDatabase): Promise<void> {
|
||||
await db.transaction(async (sql) => {
|
||||
await lockFavoriteMigration(sql);
|
||||
await assertPostgresDataReady(sql);
|
||||
await sql.query('UPDATE tool_favorite_readiness SET activated = true WHERE singleton');
|
||||
});
|
||||
}
|
||||
102
packages/data-schemas/src/postgres/mongo.spec.ts
Normal file
102
packages/data-schemas/src/postgres/mongo.spec.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import mongoose from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { createToolFavoriteModel } from '~/models/favorite';
|
||||
import toolFavoriteSchema from '~/schema/favorite';
|
||||
import { createMongoFavoriteStore } from './mongo';
|
||||
|
||||
async function rows(source: AsyncIterable<unknown>): Promise<unknown[]> {
|
||||
const result: unknown[] = [];
|
||||
for await (const row of source) result.push(row);
|
||||
return result;
|
||||
}
|
||||
|
||||
describe('CLI Mongo favorite snapshots', () => {
|
||||
let server: MongoMemoryServer;
|
||||
const database = new mongoose.Mongoose();
|
||||
beforeAll(async () => {
|
||||
server = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
|
||||
}, 90000);
|
||||
beforeEach(async () => {
|
||||
await database.connect(server.getUri(), { autoCreate: false, autoIndex: false });
|
||||
await database.connection.dropDatabase();
|
||||
});
|
||||
afterEach(async () => {
|
||||
await database.disconnect();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await server.stop();
|
||||
});
|
||||
it('reads missing collections without DDL, model registration, or inherited schema options', async () => {
|
||||
const previous = {
|
||||
autoCreate: toolFavoriteSchema.get('autoCreate'),
|
||||
autoIndex: toolFavoriteSchema.get('autoIndex'),
|
||||
autoSearchIndex: toolFavoriteSchema.get('autoSearchIndex'),
|
||||
};
|
||||
toolFavoriteSchema.set('autoCreate', true);
|
||||
toolFavoriteSchema.set('autoIndex', true);
|
||||
toolFavoriteSchema.set('autoSearchIndex', true);
|
||||
try {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
const store = createMongoFavoriteStore(database);
|
||||
expect(await rows(store.source())).toEqual([]);
|
||||
expect(database.modelNames()).toEqual([]);
|
||||
expect(await database.connection.db!.listCollections().toArray()).toEqual([]);
|
||||
await database.disconnect();
|
||||
await database.connect(server.getUri(), { autoCreate: false, autoIndex: false });
|
||||
}
|
||||
} finally {
|
||||
toolFavoriteSchema.set('autoCreate', previous.autoCreate);
|
||||
toolFavoriteSchema.set('autoIndex', previous.autoIndex);
|
||||
toolFavoriteSchema.set('autoSearchIndex', previous.autoSearchIndex);
|
||||
}
|
||||
});
|
||||
it('uses the data-schema collection name and preserves raw documents and indexes', async () => {
|
||||
const metadata = new mongoose.Mongoose();
|
||||
metadata.set('autoCreate', false);
|
||||
metadata.set('autoIndex', false);
|
||||
const name = createToolFavoriteModel(metadata).collection.name;
|
||||
const collection = database.connection.collection(name);
|
||||
const row = {
|
||||
_id: new mongoose.Types.ObjectId(),
|
||||
user: new mongoose.Types.ObjectId(),
|
||||
tenantId: 'tenant-one',
|
||||
itemType: 'tool',
|
||||
itemId: 'favorite',
|
||||
createdAt: new Date('2025-01-01'),
|
||||
updatedAt: new Date('2025-01-02'),
|
||||
};
|
||||
await collection.insertOne(row);
|
||||
const before = await database.connection.db!.listCollections().toArray();
|
||||
const indexes = await collection.indexes();
|
||||
const store = createMongoFavoriteStore(database);
|
||||
expect(await rows(store.source())).toEqual([
|
||||
{
|
||||
id: row._id.toHexString(),
|
||||
userId: row.user.toHexString(),
|
||||
tenantId: row.tenantId,
|
||||
itemType: row.itemType,
|
||||
itemId: row.itemId,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
},
|
||||
]);
|
||||
expect(await collection.find().toArray()).toEqual([row]);
|
||||
expect(await collection.indexes()).toEqual(indexes);
|
||||
expect(await database.connection.db!.listCollections().toArray()).toEqual(before);
|
||||
expect(database.modelNames()).toEqual([]);
|
||||
});
|
||||
it('can replace a missing collection with an empty snapshot using only explicit staging DDL', async () => {
|
||||
const store = createMongoFavoriteStore(database);
|
||||
const staging = await store.stage();
|
||||
expect(await rows(staging.source())).toEqual([]);
|
||||
const staged = await database.connection.db!.listCollections().toArray();
|
||||
expect(staged).toHaveLength(1);
|
||||
expect(staged[0].name).toContain('_reconcile_');
|
||||
await staging.replace();
|
||||
const replaced = await database.connection.db!.listCollections().toArray();
|
||||
expect(replaced).toHaveLength(1);
|
||||
expect(replaced[0].name).toBe(staged[0].name.split('_reconcile_')[0]);
|
||||
expect(await rows(store.source())).toEqual([]);
|
||||
expect(database.modelNames()).toEqual([]);
|
||||
});
|
||||
});
|
||||
95
packages/data-schemas/src/postgres/mongo.ts
Normal file
95
packages/data-schemas/src/postgres/mongo.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { randomBytes } from 'node:crypto';
|
||||
import type mongoose from 'mongoose';
|
||||
import type { FavoriteSnapshotRow, FavoriteSnapshotSource } from './import';
|
||||
import { buildIndexWithRetry } from '~/utils/retry';
|
||||
import toolFavoriteSchema from '~/schema/favorite';
|
||||
|
||||
export interface MongoFavoriteStaging {
|
||||
source: FavoriteSnapshotSource;
|
||||
insert(rows: FavoriteSnapshotRow[]): Promise<void>;
|
||||
replace(): Promise<void>;
|
||||
discard(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface MongoFavoriteStore {
|
||||
source: FavoriteSnapshotSource;
|
||||
stage(): Promise<MongoFavoriteStaging>;
|
||||
}
|
||||
|
||||
/** Never register application models on the connected CLI instance. */
|
||||
export function createMongoFavoriteStore(database: typeof mongoose): MongoFavoriteStore {
|
||||
const metadata = new database.Mongoose();
|
||||
const schema = toolFavoriteSchema.clone();
|
||||
schema.set('autoCreate', false);
|
||||
schema.set('autoIndex', false);
|
||||
schema.set('autoSearchIndex', false);
|
||||
// eslint-disable-next-line no-restricted-syntax -- Collection name metadata only, no raw driver operations.
|
||||
const name = metadata.model('ToolFavorite', schema).collection.name;
|
||||
const collection = database.connection.collection(name);
|
||||
const snapshot = (target: typeof collection): FavoriteSnapshotSource =>
|
||||
async function* () {
|
||||
const cursor = target.find({}).sort({ _id: 1 }).batchSize(250);
|
||||
try {
|
||||
for await (const row of cursor) {
|
||||
yield {
|
||||
id: String(row._id),
|
||||
userId: String(row.user),
|
||||
tenantId: row.tenantId ?? null,
|
||||
itemType: row.itemType,
|
||||
itemId: row.itemId,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
} finally {
|
||||
await cursor.close();
|
||||
}
|
||||
};
|
||||
return {
|
||||
source: snapshot(collection),
|
||||
async stage() {
|
||||
const stagingName = `${name}_reconcile_${randomBytes(6).toString('hex')}`;
|
||||
const staging = database.connection.collection(stagingName);
|
||||
const discard = async () => {
|
||||
await staging.drop().catch(() => undefined);
|
||||
};
|
||||
try {
|
||||
await database.connection.createCollection(stagingName);
|
||||
const exists = await database.connection.db!.listCollections({ name }).hasNext();
|
||||
for (const index of exists ? await collection.indexes() : []) {
|
||||
if (index.name === '_id_') continue;
|
||||
const options = Object.fromEntries(
|
||||
Object.entries(index).filter(([key]) => !['key', 'v', 'ns'].includes(key)),
|
||||
);
|
||||
await buildIndexWithRetry(
|
||||
() => staging.createIndex(index.key, options),
|
||||
'favorites_reconciliation_index',
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
await discard();
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
source: snapshot(staging),
|
||||
async insert(rows) {
|
||||
await staging.insertMany(
|
||||
rows.map((row) => ({
|
||||
_id: new database.Types.ObjectId(row.id),
|
||||
user: new database.Types.ObjectId(row.userId),
|
||||
tenantId: row.tenantId,
|
||||
itemType: row.itemType,
|
||||
itemId: row.itemId,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
})),
|
||||
);
|
||||
},
|
||||
async replace() {
|
||||
await staging.rename(name, { dropTarget: true });
|
||||
},
|
||||
discard,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
237
packages/data-schemas/src/postgres/pool.spec.ts
Normal file
237
packages/data-schemas/src/postgres/pool.spec.ts
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
import pg from 'pg';
|
||||
import path from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { execFile } from 'node:child_process';
|
||||
import type { PostgresOptions } from './pool';
|
||||
import { createPostgresDatabase } from './pool';
|
||||
|
||||
const postgres = process.env.PG_TEST_URL ? describe : describe.skip;
|
||||
const options = (): PostgresOptions => ({
|
||||
connectionString: process.env.PG_TEST_URL ?? '',
|
||||
max: 1,
|
||||
maxWaiting: 10,
|
||||
acquireTimeoutMs: 2000,
|
||||
queryTimeoutMs: 10000,
|
||||
idleTimeoutMs: 1000,
|
||||
});
|
||||
const safeMessage = 'PostgreSQL connection unavailable';
|
||||
const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
postgres('PostgreSQL borrowed connections', () => {
|
||||
it.each(['session', 'transaction'] as const)(
|
||||
'survives an idle %s disconnect in a disposable child process',
|
||||
async (kind) => {
|
||||
const source = `
|
||||
import pg from 'pg';
|
||||
import { createPostgresDatabase } from ${JSON.stringify(path.resolve(__dirname, 'pool.ts'))};
|
||||
const db = createPostgresDatabase({ ...${JSON.stringify({ ...options(), connectionString: undefined })}, connectionString: process.env.PG_TEST_URL });
|
||||
const killer = new pg.Client({ connectionString: process.env.PG_TEST_URL });
|
||||
await killer.connect();
|
||||
let openGate;
|
||||
const gate = new Promise(resolve => { openGate = resolve; });
|
||||
let session;
|
||||
let entered;
|
||||
const ready = new Promise(resolve => { entered = resolve; });
|
||||
const operation = db[${JSON.stringify(kind)}](async sql => {
|
||||
session = sql;
|
||||
const { rows } = await sql.query('SELECT pg_backend_pid() AS pid');
|
||||
entered(rows[0].pid);
|
||||
await gate;
|
||||
return 'must not succeed';
|
||||
});
|
||||
const outcome = operation.then(() => 'success', error => error.message);
|
||||
await killer.query('SELECT pg_terminate_backend($1)', [await ready]);
|
||||
const observed = await Promise.race([outcome, new Promise(resolve => setTimeout(() => resolve('not reported'), 1000))]);
|
||||
if (observed !== ${JSON.stringify(safeMessage)}) throw new Error(observed);
|
||||
for (const attempt of [
|
||||
() => session.query('SELECT 1'),
|
||||
() => session.transaction(async () => 'must not succeed'),
|
||||
]) {
|
||||
const message = await attempt().then(() => 'success', error => error.message);
|
||||
if (message !== ${JSON.stringify(safeMessage)}) throw new Error(message);
|
||||
}
|
||||
let closed = false;
|
||||
const closing = db.close().then(() => { closed = true; });
|
||||
await new Promise(resolve => setTimeout(resolve, 30));
|
||||
if (closed) throw new Error('close released an unfinished callback');
|
||||
openGate();
|
||||
await outcome;
|
||||
await closing;
|
||||
await killer.end();
|
||||
if (observed !== ${JSON.stringify(safeMessage)}) throw new Error(observed);
|
||||
console.log('borrowed-disconnect-safe');
|
||||
`;
|
||||
const { stdout } = await promisify(execFile)(
|
||||
process.execPath,
|
||||
['--experimental-strip-types', '--input-type=module', '--eval', source],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
expect(stdout).toContain('borrowed-disconnect-safe');
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['query', 'transaction'] as const)(
|
||||
'sanitizes an active %s disconnect and replaces the failed backend',
|
||||
async (kind) => {
|
||||
const db = createPostgresDatabase(options());
|
||||
const killer = new pg.Client({ connectionString: process.env.PG_TEST_URL });
|
||||
await killer.connect();
|
||||
try {
|
||||
const pid = (await db.query('SELECT pg_backend_pid() AS pid')).rows[0].pid;
|
||||
const operation =
|
||||
kind === 'query'
|
||||
? db.query('SELECT pg_sleep(5)')
|
||||
: db.transaction(async (sql) => {
|
||||
await sql.query('SELECT pg_sleep(5)');
|
||||
});
|
||||
const outcome = operation.catch((error: Error) => error);
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const { rows } = await killer.query('SELECT state FROM pg_stat_activity WHERE pid=$1', [
|
||||
pid,
|
||||
]);
|
||||
if (rows[0]?.state === 'active') break;
|
||||
await delay(10);
|
||||
}
|
||||
await killer.query('SELECT pg_terminate_backend($1)', [pid]);
|
||||
const error = await outcome;
|
||||
expect(error).toEqual(new Error(safeMessage));
|
||||
expect(error).not.toHaveProperty('cause');
|
||||
const next = await db.query('SELECT pg_backend_pid() AS pid');
|
||||
expect(next.rows[0].pid).not.toBe(pid);
|
||||
} finally {
|
||||
await db.close();
|
||||
await killer.end();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('PostgreSQL lease lifecycle', () => {
|
||||
function setup() {
|
||||
const client = new EventEmitter();
|
||||
const idle = () => undefined;
|
||||
client.on('error', idle);
|
||||
const query = jest.fn(async (_text: string) => ({ rows: [], rowCount: 0 }));
|
||||
const release = jest.fn((_discard: boolean) => {
|
||||
client.on('error', idle);
|
||||
});
|
||||
Object.assign(client, { query, release });
|
||||
const pool = new EventEmitter();
|
||||
let onHandoff: () => void = () => undefined;
|
||||
const connect = jest.fn((callback: (error: Error | undefined, client: unknown) => void) => {
|
||||
client.removeListener('error', idle);
|
||||
callback(undefined, client);
|
||||
onHandoff();
|
||||
});
|
||||
const end = jest.fn(async () => undefined);
|
||||
Object.assign(pool, { connect, end, waitingCount: 0 });
|
||||
jest.spyOn(pg, 'Pool').mockImplementation(() => pool as pg.Pool);
|
||||
const db = createPostgresDatabase(options());
|
||||
return {
|
||||
db,
|
||||
connect,
|
||||
client,
|
||||
query,
|
||||
release,
|
||||
onHandoff: (callback: () => void) => {
|
||||
onHandoff = callback;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it.each(['callback', 'throw'] as const)('sanitizes a driver connect %s failure', async (kind) => {
|
||||
const { db, connect, release } = setup();
|
||||
connect.mockImplementation((callback) => {
|
||||
const original = new Error('postgresql://private:secret@host/private');
|
||||
if (kind === 'throw') throw original;
|
||||
callback(original, undefined);
|
||||
});
|
||||
const error = await db.query('SELECT 1').catch((error: Error) => error);
|
||||
expect(error).toEqual(new Error(safeMessage));
|
||||
expect(error).not.toHaveProperty('cause');
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('protects the synchronous connect handoff and releases exactly once', async () => {
|
||||
const fixture = setup();
|
||||
fixture.onHandoff(() => fixture.client.emit('error', new Error('private DSN')));
|
||||
await expect(fixture.db.query('private query')).rejects.toThrow(safeMessage);
|
||||
expect(fixture.query).not.toHaveBeenCalled();
|
||||
expect(fixture.release).toHaveBeenCalledTimes(1);
|
||||
expect(fixture.release).toHaveBeenCalledWith(true);
|
||||
expect(fixture.client.listenerCount('error')).toBe(1);
|
||||
await fixture.db.close();
|
||||
});
|
||||
|
||||
it('leaves only the pool idle listener across successful leases', async () => {
|
||||
const { db, client, release } = setup();
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await db.session(async (sql) => {
|
||||
expect(client.listenerCount('error')).toBe(1);
|
||||
await sql.query('SELECT 1');
|
||||
});
|
||||
expect(client.listenerCount('error')).toBe(1);
|
||||
}
|
||||
expect(release).toHaveBeenCalledTimes(20);
|
||||
expect(release).toHaveBeenLastCalledWith(false);
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('handles an error racing release without returning success or leaking listeners', async () => {
|
||||
const { db, client, release } = setup();
|
||||
release.mockImplementation(() => {
|
||||
client.on('error', () => undefined);
|
||||
client.emit('error', new Error('private release fault'));
|
||||
});
|
||||
await expect(db.query('SELECT 1')).rejects.toThrow(safeMessage);
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
expect(client.listenerCount('error')).toBe(1);
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('retains local SQL and callback exception identity', async () => {
|
||||
const { db, query, release } = setup();
|
||||
const sqlError = Object.assign(new Error('local constraint'), { code: '23505' });
|
||||
query.mockRejectedValueOnce(sqlError);
|
||||
await expect(db.query('INSERT')).rejects.toBe(sqlError);
|
||||
const original = new Error('local callback');
|
||||
await expect(
|
||||
db.transaction(async () => {
|
||||
throw original;
|
||||
}),
|
||||
).rejects.toBe(original);
|
||||
expect(query.mock.calls.map(([text]) => text)).toEqual(['INSERT', 'BEGIN', 'ROLLBACK']);
|
||||
expect(release).toHaveBeenLastCalledWith(false);
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('discards on rollback failure while preserving the original callback exception', async () => {
|
||||
const { db, query, release } = setup();
|
||||
const original = new Error('local callback');
|
||||
query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
|
||||
query.mockRejectedValueOnce(Object.assign(new Error('rollback fault'), { code: 'XX000' }));
|
||||
await expect(
|
||||
db.transaction(async () => {
|
||||
throw original;
|
||||
}),
|
||||
).rejects.toBe(original);
|
||||
expect(release).toHaveBeenCalledWith(true);
|
||||
await db.close();
|
||||
});
|
||||
|
||||
it('reports a lost COMMIT acknowledgement without issuing a misleading ROLLBACK', async () => {
|
||||
const { db, query, release } = setup();
|
||||
query.mockImplementation(async (text) => {
|
||||
if (text === 'COMMIT')
|
||||
throw Object.assign(new Error('private commit detail'), { code: 'ECONNRESET' });
|
||||
return { rows: [], rowCount: 0 };
|
||||
});
|
||||
await expect(db.transaction(async () => 'value')).rejects.toThrow(safeMessage);
|
||||
expect(query.mock.calls.map(([text]) => text)).toEqual(['BEGIN', 'COMMIT']);
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
expect(release).toHaveBeenCalledWith(true);
|
||||
await db.close();
|
||||
});
|
||||
});
|
||||
190
packages/data-schemas/src/postgres/pool.ts
Normal file
190
packages/data-schemas/src/postgres/pool.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import pg from 'pg';
|
||||
import type { QueryResult, QueryResultRow } from 'pg';
|
||||
|
||||
export interface PostgresOptions {
|
||||
connectionString: string;
|
||||
max: number;
|
||||
maxWaiting: number;
|
||||
acquireTimeoutMs: number;
|
||||
queryTimeoutMs: number;
|
||||
idleTimeoutMs: number;
|
||||
}
|
||||
|
||||
export interface SqlExecutor {
|
||||
query<R extends QueryResultRow = QueryResultRow>(
|
||||
text: string,
|
||||
values?: unknown[],
|
||||
): Promise<QueryResult<R>>;
|
||||
}
|
||||
|
||||
export interface PostgresSession extends SqlExecutor {
|
||||
transaction<T>(operation: (sql: SqlExecutor) => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
export interface PostgresDatabase extends PostgresSession {
|
||||
session<T>(operation: (sql: PostgresSession) => Promise<T>): Promise<T>;
|
||||
transaction<T>(operation: (sql: SqlExecutor) => Promise<T>): Promise<T>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export function createPostgresDatabase(options: PostgresOptions): PostgresDatabase {
|
||||
const pool = new pg.Pool({
|
||||
connectionString: options.connectionString,
|
||||
max: options.max,
|
||||
connectionTimeoutMillis: options.acquireTimeoutMs,
|
||||
idleTimeoutMillis: options.idleTimeoutMs,
|
||||
statement_timeout: options.queryTimeoutMs,
|
||||
query_timeout: options.queryTimeoutMs,
|
||||
idle_in_transaction_session_timeout: options.queryTimeoutMs,
|
||||
application_name: 'librechat',
|
||||
});
|
||||
// Idle-client errors must not become uncaught exceptions or expose connection details.
|
||||
pool.on('error', () => undefined);
|
||||
let closing = false;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
function connectionError(): Error {
|
||||
return new Error('PostgreSQL connection unavailable');
|
||||
}
|
||||
function isConnectionError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) return false;
|
||||
const code = 'code' in error && typeof error.code === 'string' ? error.code : '';
|
||||
const severity = 'severity' in error ? error.severity : undefined;
|
||||
return (
|
||||
severity === 'FATAL' ||
|
||||
severity === 'PANIC' ||
|
||||
code?.startsWith('08') === true ||
|
||||
['57P01', '57P02', '57P03', 'ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'ECONNREFUSED'].includes(
|
||||
code ?? '',
|
||||
) ||
|
||||
[
|
||||
'Connection terminated unexpectedly',
|
||||
'Connection terminated',
|
||||
'Client has encountered a connection error and is not queryable',
|
||||
'Client was closed and is not queryable',
|
||||
'Query read timeout',
|
||||
].includes(error.message)
|
||||
);
|
||||
}
|
||||
function connect() {
|
||||
if (closing || pool.waitingCount >= options.maxWaiting) {
|
||||
return Promise.reject(new Error('PostgreSQL connection admission unavailable'));
|
||||
}
|
||||
// The callback installs protection before pg-pool's synchronous handoff returns.
|
||||
return new Promise<{
|
||||
query: SqlExecutor['query'];
|
||||
failure: Promise<never>;
|
||||
assertHealthy(): void;
|
||||
discard(): void;
|
||||
release(): void;
|
||||
}>((resolve, reject) => {
|
||||
pool.connect((error, client) => {
|
||||
if (error || !client) {
|
||||
reject(connectionError());
|
||||
return;
|
||||
}
|
||||
let failed: Error | undefined;
|
||||
let released = false;
|
||||
let discard = false;
|
||||
let rejectFailure: (error: Error) => void;
|
||||
const failure = new Promise<never>((_, reject) => {
|
||||
rejectFailure = reject;
|
||||
});
|
||||
void failure.catch(() => undefined);
|
||||
const onError = () => {
|
||||
if (failed) return;
|
||||
failed = connectionError();
|
||||
rejectFailure(failed);
|
||||
};
|
||||
client.on('error', onError);
|
||||
const assertHealthy = () => {
|
||||
if (failed) throw failed;
|
||||
if (released) throw connectionError();
|
||||
};
|
||||
resolve({
|
||||
failure,
|
||||
assertHealthy,
|
||||
discard() {
|
||||
discard = true;
|
||||
},
|
||||
async query<R extends QueryResultRow>(text: string, values?: unknown[]) {
|
||||
assertHealthy();
|
||||
try {
|
||||
const result = await client.query<R>(text, values);
|
||||
assertHealthy();
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (isConnectionError(error)) onError();
|
||||
throw failed ?? error;
|
||||
}
|
||||
},
|
||||
release() {
|
||||
if (released) return;
|
||||
released = true;
|
||||
try {
|
||||
client.release(discard || Boolean(failed));
|
||||
} finally {
|
||||
// release restores pg-pool's idle listener before ours is removed.
|
||||
client.removeListener('error', onError);
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
}).catch(() => {
|
||||
throw connectionError();
|
||||
});
|
||||
}
|
||||
const database: PostgresDatabase = {
|
||||
query: (text, values) => database.session((session) => session.query(text, values)),
|
||||
async transaction<T>(operation: (sql: SqlExecutor) => Promise<T>): Promise<T> {
|
||||
return database.session((session) => session.transaction(operation));
|
||||
},
|
||||
async session<T>(operation: (sql: PostgresSession) => Promise<T>): Promise<T> {
|
||||
const lease = await connect();
|
||||
let inTransaction = false;
|
||||
const session: PostgresSession = {
|
||||
query: lease.query,
|
||||
async transaction<U>(work: (sql: SqlExecutor) => Promise<U>): Promise<U> {
|
||||
lease.assertHealthy();
|
||||
if (inTransaction) throw new Error('Nested PostgreSQL transactions are not supported');
|
||||
inTransaction = true;
|
||||
try {
|
||||
await lease.query('BEGIN');
|
||||
const value = await work(session);
|
||||
await lease.query('COMMIT');
|
||||
return value;
|
||||
} catch (error) {
|
||||
// A lost COMMIT acknowledgement has an unknown outcome. Do not claim rollback.
|
||||
lease.assertHealthy();
|
||||
try {
|
||||
await lease.query('ROLLBACK');
|
||||
} catch {
|
||||
lease.discard();
|
||||
}
|
||||
lease.assertHealthy();
|
||||
throw error;
|
||||
} finally {
|
||||
inTransaction = false;
|
||||
}
|
||||
},
|
||||
};
|
||||
const work = Promise.resolve()
|
||||
.then(() => {
|
||||
lease.assertHealthy();
|
||||
return operation(session);
|
||||
})
|
||||
.then((value) => {
|
||||
lease.assertHealthy();
|
||||
return value;
|
||||
})
|
||||
.finally(() => lease.release());
|
||||
// Report disconnects promptly, but retain ownership until the callback settles.
|
||||
return Promise.race([work, lease.failure]);
|
||||
},
|
||||
close() {
|
||||
closing = true;
|
||||
closePromise ??= pool.end();
|
||||
return closePromise;
|
||||
},
|
||||
};
|
||||
return database;
|
||||
}
|
||||
51
packages/data-schemas/src/postgres/postgres.helper.ts
Normal file
51
packages/data-schemas/src/postgres/postgres.helper.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { randomBytes } from 'node:crypto';
|
||||
import type { PostgresDatabase, PostgresOptions } from './pool';
|
||||
import { createPostgresDatabase } from './pool';
|
||||
|
||||
export const postgresTestOptions: PostgresOptions = {
|
||||
connectionString: process.env.PG_TEST_URL ?? '',
|
||||
max: 4,
|
||||
maxWaiting: 20,
|
||||
acquireTimeoutMs: 1000,
|
||||
queryTimeoutMs: 5000,
|
||||
idleTimeoutMs: 1000,
|
||||
};
|
||||
|
||||
/** Each spec/Stryker worker gets its own schema; network and constraints remain real. */
|
||||
export async function createPostgresTestDatabase(): Promise<{
|
||||
db: PostgresDatabase;
|
||||
options: PostgresOptions;
|
||||
schema: string;
|
||||
}> {
|
||||
const admin = createPostgresDatabase(postgresTestOptions);
|
||||
const schema = `favorites_test_${randomBytes(8).toString('hex')}`;
|
||||
try {
|
||||
await admin.query(`CREATE SCHEMA ${schema}`);
|
||||
const url = new URL(postgresTestOptions.connectionString);
|
||||
url.searchParams.set('options', `-c search_path=${schema}`);
|
||||
const options = { ...postgresTestOptions, connectionString: url.toString() };
|
||||
const database = createPostgresDatabase(options);
|
||||
let closing: Promise<void> | undefined;
|
||||
const db: PostgresDatabase = {
|
||||
...database,
|
||||
close() {
|
||||
closing ??= (async () => {
|
||||
try {
|
||||
await database.close();
|
||||
} finally {
|
||||
try {
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`);
|
||||
} finally {
|
||||
await admin.close();
|
||||
}
|
||||
}
|
||||
})();
|
||||
return closing;
|
||||
},
|
||||
};
|
||||
return { db, options, schema };
|
||||
} catch (error) {
|
||||
await admin.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
|
@ -170,6 +170,10 @@ const userSchema: Schema<IUser> = new Schema<IUser>(
|
|||
type: Date,
|
||||
select: false,
|
||||
},
|
||||
agentTriggerDeletionAttemptId: {
|
||||
type: String,
|
||||
select: false,
|
||||
},
|
||||
subagentAdmissionFences: {
|
||||
type: [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -33,3 +33,11 @@ export interface RemoveToolFavoriteResult {
|
|||
ok: boolean;
|
||||
removed: boolean;
|
||||
}
|
||||
|
||||
export interface ToolFavoriteMethods {
|
||||
getToolFavorites(userId: string): Promise<IToolFavoriteLean[]>;
|
||||
addToolFavorite(params: ToolFavoriteParams): Promise<AddToolFavoriteResult>;
|
||||
removeToolFavorite(params: ToolFavoriteParams): Promise<RemoveToolFavoriteResult>;
|
||||
deleteUserToolFavorites(userId: string, deletionAttemptId?: string): Promise<number>;
|
||||
cancelUserToolFavoriteDeletion?(userId: string, deletionAttemptId: string): Promise<void>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ export interface IUser extends Document {
|
|||
termsAcceptedSource?: 'migration';
|
||||
/** Internal fence that prevents agent-trigger admission during account deletion. */
|
||||
agentTriggerDeletionStartedAt?: Date;
|
||||
/** Durable identity shared by every offline recovery of this erasure attempt. */
|
||||
agentTriggerDeletionAttemptId?: string;
|
||||
/** Expiring fences closing subagent admission while bulk deletions drain. */
|
||||
subagentAdmissionFences?: Array<{
|
||||
token: string;
|
||||
|
|
|
|||
331
scripts/quality/aggregate.mts
Normal file
331
scripts/quality/aggregate.mts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { parseArgs, isDeepStrictEqual } from 'node:util';
|
||||
import {
|
||||
appendFileSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
|
||||
import { shardTargets } from './shard.mts';
|
||||
|
||||
interface Options {
|
||||
head: string;
|
||||
workspace: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
type ObjectValue = Record<string, unknown>;
|
||||
const statuses = [
|
||||
'Killed',
|
||||
'Survived',
|
||||
'Timeout',
|
||||
'NoCoverage',
|
||||
'CompileError',
|
||||
'RuntimeError',
|
||||
'Ignored',
|
||||
'Pending',
|
||||
] as const;
|
||||
type Status = (typeof statuses)[number];
|
||||
|
||||
function requireValue(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
function object(value: unknown): ObjectValue {
|
||||
requireValue(
|
||||
value !== null && typeof value === 'object' && !Array.isArray(value),
|
||||
'Expected JSON object',
|
||||
);
|
||||
return value as ObjectValue;
|
||||
}
|
||||
function array(value: unknown): unknown[] {
|
||||
requireValue(Array.isArray(value), 'Expected JSON array');
|
||||
return value;
|
||||
}
|
||||
function text(value: unknown): string {
|
||||
requireValue(typeof value === 'string' && value.length > 0, 'Expected nonempty string');
|
||||
return value;
|
||||
}
|
||||
function integer(value: unknown, minimum = 0): number {
|
||||
requireValue(
|
||||
Number.isSafeInteger(value) && typeof value === 'number' && value >= minimum,
|
||||
'Expected nonnegative integer',
|
||||
);
|
||||
return value;
|
||||
}
|
||||
function path(value: unknown): string {
|
||||
const file = text(value);
|
||||
requireValue(
|
||||
!file.includes('\\') &&
|
||||
!file.startsWith('/') &&
|
||||
file.split('/').every((part) => part && part !== '.' && part !== '..'),
|
||||
'Expected repository-relative file',
|
||||
);
|
||||
return file;
|
||||
}
|
||||
function location(value: unknown): ObjectValue {
|
||||
const span = object(value);
|
||||
const start = object(span.start);
|
||||
const end = object(span.end);
|
||||
const line = integer(start.line, 1);
|
||||
const column = integer(start.column, 1);
|
||||
const endLine = integer(end.line, 1);
|
||||
const endColumn = integer(end.column, 1);
|
||||
requireValue(endLine > line || (endLine === line && endColumn >= column), 'Invalid location');
|
||||
return span;
|
||||
}
|
||||
function targets(value: unknown) {
|
||||
const files = new Set<string>();
|
||||
return array(value).map((entry) => {
|
||||
const target = object(entry);
|
||||
const file = path(target.file);
|
||||
requireValue(!files.has(file), 'Duplicate target file');
|
||||
files.add(file);
|
||||
const functions = array(target.functions);
|
||||
requireValue(functions.length > 0 && array(target.mutate).length > 0, 'Empty mutation target');
|
||||
for (const fn of functions) {
|
||||
const row = object(fn);
|
||||
text(row.function);
|
||||
integer(row.line, 1);
|
||||
integer(row.column, 1);
|
||||
location(row.location);
|
||||
}
|
||||
for (const pattern of array(target.mutate)) text(pattern);
|
||||
return { file, functions, original: target };
|
||||
});
|
||||
}
|
||||
|
||||
/** Validate complete file shards without changing their reports or local test/mutant IDs. */
|
||||
export function aggregate(inputs: readonly unknown[], options: Options) {
|
||||
integer(options.count, 1);
|
||||
text(options.head);
|
||||
text(options.workspace);
|
||||
requireValue(inputs.length === options.count, 'Wrong shard directory count');
|
||||
const shards = inputs
|
||||
.map((input) => {
|
||||
const row = object(input);
|
||||
const selection = object(row.selection);
|
||||
const shard = object(selection.shard);
|
||||
const index = integer(shard.index, 1);
|
||||
requireValue(
|
||||
index <= options.count && shard.total === options.count,
|
||||
'Wrong shard index or count',
|
||||
);
|
||||
requireValue(selection.head === options.head, 'Wrong selection head');
|
||||
requireValue(selection.workspace === options.workspace, 'Wrong selection workspace');
|
||||
text(selection.base);
|
||||
const files = array(selection.files).map((entry) => path(object(entry).file));
|
||||
requireValue(new Set(files).size === files.length, 'Duplicate selected file');
|
||||
const all = targets(row.allFunctions);
|
||||
requireValue(
|
||||
all.every(({ file }) => files.includes(file)),
|
||||
'Target outside selected files',
|
||||
);
|
||||
targets(row.functions);
|
||||
const expected = shardTargets(all, { index, total: options.count });
|
||||
requireValue(
|
||||
isDeepStrictEqual(
|
||||
row.functions,
|
||||
expected.map(({ original }) => original),
|
||||
),
|
||||
'Wrong file/function shard partition',
|
||||
);
|
||||
return {
|
||||
index,
|
||||
selection,
|
||||
all,
|
||||
expected,
|
||||
report: object(row.report),
|
||||
progress: object(row.progress),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.index - b.index);
|
||||
requireValue(
|
||||
new Set(shards.map(({ index }) => index)).size === options.count,
|
||||
'Duplicate shard index',
|
||||
);
|
||||
const first = shards[0]!;
|
||||
const counts: Record<Status, number> = {
|
||||
Killed: 0,
|
||||
Survived: 0,
|
||||
Timeout: 0,
|
||||
NoCoverage: 0,
|
||||
CompileError: 0,
|
||||
RuntimeError: 0,
|
||||
Ignored: 0,
|
||||
Pending: 0,
|
||||
};
|
||||
const identities = new Set<string>();
|
||||
let staticTotal = 0;
|
||||
for (const shard of shards) {
|
||||
requireValue(shard.selection.base === first.selection.base, 'Inconsistent selection base');
|
||||
requireValue(
|
||||
isDeepStrictEqual(shard.selection.files, first.selection.files),
|
||||
'Inconsistent original selected files',
|
||||
);
|
||||
requireValue(
|
||||
isDeepStrictEqual(
|
||||
shard.all.map(({ original }) => original),
|
||||
first.all.map(({ original }) => original),
|
||||
),
|
||||
'Inconsistent all-functions manifest',
|
||||
);
|
||||
requireValue(shard.report.schemaVersion === '1.0', 'Unsupported mutation report schema');
|
||||
let total = 0;
|
||||
let staticCount = 0;
|
||||
const ids = new Set<string>();
|
||||
for (const [file, entry] of Object.entries(object(shard.report.files))) {
|
||||
path(file);
|
||||
requireValue(
|
||||
shard.expected.some((target) => target.file === file),
|
||||
'Report file outside shard targets',
|
||||
);
|
||||
for (const value of array(object(entry).mutants)) {
|
||||
const mutant = object(value);
|
||||
const id = text(mutant.id);
|
||||
requireValue(!ids.has(id), 'Duplicate local mutant ID');
|
||||
ids.add(id);
|
||||
const span = location(mutant.location);
|
||||
const mutator = text(mutant.mutatorName);
|
||||
requireValue(typeof mutant.replacement === 'string', 'Invalid mutant replacement');
|
||||
requireValue(
|
||||
mutant.static === undefined || typeof mutant.static === 'boolean',
|
||||
'Invalid static flag',
|
||||
);
|
||||
const status = statuses.find((name) => name === mutant.status);
|
||||
requireValue(status, 'Unknown mutant status');
|
||||
requireValue(
|
||||
status !== 'RuntimeError' && status !== 'Pending',
|
||||
'Incomplete mutant outcome',
|
||||
);
|
||||
const identity = JSON.stringify([
|
||||
file,
|
||||
object(span.start).line,
|
||||
object(span.start).column,
|
||||
object(span.end).line,
|
||||
object(span.end).column,
|
||||
mutator,
|
||||
mutant.replacement,
|
||||
]);
|
||||
requireValue(!identities.has(identity), 'Duplicate mutation identity');
|
||||
identities.add(identity);
|
||||
counts[status]++;
|
||||
total++;
|
||||
if (mutant.static === true) staticCount++;
|
||||
}
|
||||
}
|
||||
const progress = shard.progress;
|
||||
requireValue(
|
||||
progress.complete === true && progress.completed === total && progress.total === total,
|
||||
'Incomplete mutation progress',
|
||||
);
|
||||
requireValue(
|
||||
progress.staticCompleted === staticCount && progress.staticTotal === staticCount,
|
||||
'Incomplete static mutation progress',
|
||||
);
|
||||
staticTotal += staticCount;
|
||||
}
|
||||
const detected = counts.Killed + counts.Timeout;
|
||||
const tested = detected + counts.Survived + counts.NoCoverage;
|
||||
const total = Object.values(counts).reduce((sum, count) => sum + count, 0);
|
||||
return {
|
||||
head: options.head,
|
||||
base: first.selection.base,
|
||||
workspace: options.workspace,
|
||||
shardCount: options.count,
|
||||
complete: true,
|
||||
files: first.selection.files,
|
||||
functions: first.all.map(({ original }) => original),
|
||||
total,
|
||||
completed: total,
|
||||
staticTotal,
|
||||
staticCompleted: staticTotal,
|
||||
counts,
|
||||
score: tested ? (100 * detected) / tested : null,
|
||||
shards: shards.map(({ index, report, progress }) => ({ index, report, progress })),
|
||||
};
|
||||
}
|
||||
|
||||
export function aggregateMarkdown(result: ReturnType<typeof aggregate>): string {
|
||||
const score = result.score === null ? 'N/A (no valid mutants)' : `${result.score.toFixed(2)}%`;
|
||||
return [
|
||||
`## ${result.workspace}: mutation testing (advisory)`,
|
||||
'',
|
||||
`Complete: ${result.shardCount} shards, ${result.completed}/${result.total} mutants. Static mutations: ${result.staticCompleted}/${result.staticTotal}.`,
|
||||
'',
|
||||
`Mutation score: ${score}. Compile errors and ignored mutants are excluded from the score, not the report.`,
|
||||
'',
|
||||
'| Result | Mutants |',
|
||||
'| --- | ---: |',
|
||||
...statuses.map((status) => `| ${status} | ${result.counts[status]} |`),
|
||||
'',
|
||||
'Aggregate JSON retains each shard report, including source, test metadata, and unchanged local IDs.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function main() {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
directory: { type: 'string' },
|
||||
head: { type: 'string' },
|
||||
workspace: { type: 'string' },
|
||||
count: { type: 'string' },
|
||||
output: { type: 'string' },
|
||||
},
|
||||
});
|
||||
requireValue(
|
||||
values.directory && values.head && values.workspace && values.count && values.output,
|
||||
'Required: --directory --head --workspace --count --output',
|
||||
);
|
||||
for (const name of ['aggregate.json', 'summary.md']) {
|
||||
rmSync(join(values.output, name), { force: true });
|
||||
}
|
||||
requireValue(/^[1-9]\d*$/u.test(values.count), 'Invalid shard count');
|
||||
const entries = readdirSync(values.directory, { withFileTypes: true });
|
||||
requireValue(
|
||||
entries.length === Number(values.count) && entries.every((entry) => entry.isDirectory()),
|
||||
'Wrong shard directory count',
|
||||
);
|
||||
const inputs = entries.map((entry) => {
|
||||
const read = (name: string): unknown => {
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(values.directory!, entry.name, name), 'utf8'));
|
||||
} catch {
|
||||
throw new Error(`Missing or malformed shard ${name}`);
|
||||
}
|
||||
};
|
||||
return {
|
||||
selection: read('selection.json'),
|
||||
allFunctions: read('all-functions.json'),
|
||||
functions: read('functions.json'),
|
||||
report: read('mutation.json'),
|
||||
progress: read('progress.json'),
|
||||
};
|
||||
});
|
||||
const result = aggregate(inputs, {
|
||||
head: values.head,
|
||||
workspace: values.workspace,
|
||||
count: Number(values.count),
|
||||
});
|
||||
mkdirSync(values.output, { recursive: true });
|
||||
writeFileSync(join(values.output, 'aggregate.json'), JSON.stringify(result, null, 2));
|
||||
const markdown = aggregateMarkdown(result);
|
||||
writeFileSync(join(values.output, 'summary.md'), markdown);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown);
|
||||
console.log(markdown);
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
try {
|
||||
main();
|
||||
} catch {
|
||||
console.error(
|
||||
'Mutation aggregation failed: shard evidence is missing, inconsistent, malformed, or incomplete.',
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
351
scripts/quality/aggregate.test.mts
Normal file
351
scripts/quality/aggregate.test.mts
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
import { test } from 'node:test';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
|
||||
import { aggregate, aggregateMarkdown } from './aggregate.mts';
|
||||
import { shardTargets } from './shard.mts';
|
||||
|
||||
const options = { head: 'head-sha', workspace: 'api', count: 3 };
|
||||
const point = (line: number) => ({ line, column: 1 });
|
||||
const span = (line: number) => ({ start: point(line), end: point(line + 1) });
|
||||
const functions = Array.from({ length: 5 }, (_, index) => ({
|
||||
file: `packages/api/src/file${index}.ts`,
|
||||
functions: Array.from({ length: [2, 4, 1, 3, 2][index] }, (_, fn) => fn * 2 + 1).map((line) => ({
|
||||
function: `f${line}`,
|
||||
line,
|
||||
column: 1,
|
||||
location: span(line),
|
||||
})),
|
||||
mutate: [`packages/api/src/file${index}.ts:1:0-4:0`],
|
||||
}));
|
||||
const files = functions.map(({ file }) => ({ file, ranges: [{ start: 1, count: 4 }] }));
|
||||
const outcomes = ['Killed', 'Survived', 'Timeout', 'NoCoverage', 'CompileError', 'Ignored'];
|
||||
function fixtures() {
|
||||
return Array.from({ length: 3 }, (_, shard) => {
|
||||
const chosen = shardTargets(functions, { index: shard + 1, total: 3 });
|
||||
const reportFiles = Object.fromEntries(
|
||||
chosen.map(({ file }, index) => [
|
||||
file,
|
||||
{
|
||||
source: 'function source() {}',
|
||||
language: 'typescript',
|
||||
mutants: [0, 1].map((position) => ({
|
||||
id: String(index * 2 + position),
|
||||
location: span(position + 1),
|
||||
mutatorName: 'BooleanLiteral',
|
||||
replacement: 'false',
|
||||
status: outcomes[(shard * 2 + position) % outcomes.length],
|
||||
static: position === 1,
|
||||
coveredBy: ['0'],
|
||||
killedBy: position === 0 ? ['0'] : [],
|
||||
statusReason: 'preserve reason',
|
||||
})),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const total = chosen.length * 2;
|
||||
return {
|
||||
selection: {
|
||||
head: options.head,
|
||||
base: 'base-sha',
|
||||
workspace: options.workspace,
|
||||
files: structuredClone(files),
|
||||
shard: { index: shard + 1, total: 3 },
|
||||
},
|
||||
allFunctions: structuredClone(functions),
|
||||
functions: structuredClone([...chosen]),
|
||||
report: {
|
||||
schemaVersion: '1.0',
|
||||
files: reportFiles,
|
||||
testFiles: { 'tests/file.ts': { tests: [{ id: '0', name: 'test' }] } },
|
||||
framework: { name: 'Stryker' },
|
||||
},
|
||||
progress: {
|
||||
complete: true,
|
||||
completed: total,
|
||||
total,
|
||||
staticCompleted: chosen.length,
|
||||
staticTotal: chosen.length,
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
test('aggregates every file, function, outcome, static mutant, and unchanged raw report', () => {
|
||||
const inputs = fixtures();
|
||||
const snapshot = structuredClone(inputs);
|
||||
const result = aggregate(inputs, options);
|
||||
assert.equal(result.complete, true);
|
||||
assert.equal(result.total, 10);
|
||||
assert.equal(result.completed, 10);
|
||||
assert.equal(result.staticTotal, 5);
|
||||
assert.equal(result.staticCompleted, 5);
|
||||
assert.deepEqual(result.functions, functions);
|
||||
assert.deepEqual(result.files, files);
|
||||
assert.deepEqual(result.counts, {
|
||||
Killed: 2,
|
||||
Survived: 2,
|
||||
Timeout: 2,
|
||||
NoCoverage: 2,
|
||||
CompileError: 1,
|
||||
RuntimeError: 0,
|
||||
Ignored: 1,
|
||||
Pending: 0,
|
||||
});
|
||||
assert.equal(result.score, 50);
|
||||
assert.deepEqual(
|
||||
result.shards.map(({ report }) => report),
|
||||
inputs.map(({ report }) => report),
|
||||
);
|
||||
assert.deepEqual(inputs, snapshot);
|
||||
assert.match(aggregateMarkdown(result), /CompileError \| 1/);
|
||||
assert.match(aggregateMarkdown(result), /Static mutations: 5\/5/);
|
||||
});
|
||||
|
||||
test('shard directory ordering does not affect results or identities', () => {
|
||||
const inputs = fixtures();
|
||||
assert.deepEqual(
|
||||
aggregate([inputs[2], inputs[0], inputs[1]], options),
|
||||
aggregate(inputs, options),
|
||||
);
|
||||
});
|
||||
|
||||
const invalid: [string, (inputs: ReturnType<typeof fixtures>) => void][] = [
|
||||
[
|
||||
'missing shard',
|
||||
(inputs) => {
|
||||
inputs.pop();
|
||||
},
|
||||
],
|
||||
[
|
||||
'extra shard',
|
||||
(inputs) => {
|
||||
inputs.push(structuredClone(inputs[0]!));
|
||||
},
|
||||
],
|
||||
[
|
||||
'duplicate shard index',
|
||||
(inputs) => {
|
||||
inputs[1]!.selection.shard.index = 1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong shard count',
|
||||
(inputs) => {
|
||||
inputs[0]!.selection.shard.total = 4;
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong head',
|
||||
(inputs) => {
|
||||
inputs[0]!.selection.head = 'other';
|
||||
},
|
||||
],
|
||||
[
|
||||
'inconsistent base',
|
||||
(inputs) => {
|
||||
inputs[1]!.selection.base = 'other';
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong workspace',
|
||||
(inputs) => {
|
||||
inputs[0]!.selection.workspace = 'client';
|
||||
},
|
||||
],
|
||||
[
|
||||
'inconsistent original files',
|
||||
(inputs) => {
|
||||
inputs[0]!.selection.files.pop();
|
||||
},
|
||||
],
|
||||
[
|
||||
'inconsistent full manifest',
|
||||
(inputs) => {
|
||||
inputs[1]!.allFunctions[0]!.functions[0]!.function = 'other';
|
||||
},
|
||||
],
|
||||
[
|
||||
'dropped target file',
|
||||
(inputs) => {
|
||||
inputs[0]!.functions.pop();
|
||||
},
|
||||
],
|
||||
[
|
||||
'dropped target function',
|
||||
(inputs) => {
|
||||
inputs[0]!.functions[0]!.functions.pop();
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong round-robin partition',
|
||||
(inputs) => {
|
||||
inputs[0]!.functions.reverse();
|
||||
},
|
||||
],
|
||||
[
|
||||
'out-of-shard report file',
|
||||
(inputs) => {
|
||||
inputs[0]!.report.files['packages/api/src/other.ts'] = structuredClone(
|
||||
Object.values(inputs[0]!.report.files)[0]!,
|
||||
);
|
||||
},
|
||||
],
|
||||
[
|
||||
'runtime error',
|
||||
(inputs) => {
|
||||
Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'RuntimeError';
|
||||
},
|
||||
],
|
||||
[
|
||||
'pending mutant',
|
||||
(inputs) => {
|
||||
Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Pending';
|
||||
},
|
||||
],
|
||||
[
|
||||
'unknown status',
|
||||
(inputs) => {
|
||||
Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Unknown';
|
||||
},
|
||||
],
|
||||
[
|
||||
'incomplete progress',
|
||||
(inputs) => {
|
||||
inputs[0]!.progress.complete = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong completed count',
|
||||
(inputs) => {
|
||||
inputs[0]!.progress.completed--;
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong planned count',
|
||||
(inputs) => {
|
||||
inputs[0]!.progress.total++;
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong static completed count',
|
||||
(inputs) => {
|
||||
inputs[0]!.progress.staticCompleted--;
|
||||
},
|
||||
],
|
||||
[
|
||||
'wrong static total count',
|
||||
(inputs) => {
|
||||
inputs[0]!.progress.staticTotal++;
|
||||
},
|
||||
],
|
||||
[
|
||||
'duplicate mutant identity',
|
||||
(inputs) => {
|
||||
const mutants = Object.values(inputs[0]!.report.files)[0]!.mutants;
|
||||
mutants[1] = { ...mutants[0]!, id: 'new' };
|
||||
},
|
||||
],
|
||||
[
|
||||
'duplicate local ID',
|
||||
(inputs) => {
|
||||
Object.values(inputs[0]!.report.files)[0]!.mutants[1]!.id = '0';
|
||||
},
|
||||
],
|
||||
];
|
||||
for (const [name, change] of invalid) {
|
||||
test(`fails closed: ${name}`, () => {
|
||||
const inputs = fixtures();
|
||||
change(inputs);
|
||||
assert.throws(() => aggregate(inputs, options));
|
||||
});
|
||||
}
|
||||
|
||||
test('requires explicit complete evidence for zero-mutant shards', () => {
|
||||
const inputs = fixtures();
|
||||
for (const input of inputs) {
|
||||
input.allFunctions = [];
|
||||
input.functions = [];
|
||||
input.report.files = {};
|
||||
input.progress = { complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 };
|
||||
}
|
||||
const result = aggregate(inputs, options);
|
||||
assert.equal(result.total, 0);
|
||||
assert.equal(result.score, null);
|
||||
assert.deepEqual(result.functions, []);
|
||||
const missing: unknown[] = structuredClone(inputs);
|
||||
missing[0] = { ...inputs[0], progress: undefined };
|
||||
assert.throws(() => aggregate(missing, options));
|
||||
});
|
||||
|
||||
test('chosen function files may legitimately produce no mutants', () => {
|
||||
const inputs = fixtures();
|
||||
inputs[0]!.report.files = {};
|
||||
inputs[0]!.progress = {
|
||||
complete: true,
|
||||
completed: 0,
|
||||
total: 0,
|
||||
staticCompleted: 0,
|
||||
staticTotal: 0,
|
||||
};
|
||||
assert.equal(aggregate(inputs, options).total, 6);
|
||||
});
|
||||
|
||||
test('malformed evidence is rejected without trusting TypeScript casts', () => {
|
||||
for (const bad of [null, [], {}, { selection: null }]) {
|
||||
assert.throws(() => aggregate([bad, bad, bad], options));
|
||||
}
|
||||
const inputs = fixtures();
|
||||
assert.throws(() => aggregate(inputs, { ...options, count: 0 }));
|
||||
assert.throws(() => aggregate(inputs, { ...options, head: '' }));
|
||||
});
|
||||
|
||||
test('CLI uses only built-ins, reads unmerged directories, and fails without a green output', (t) => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'mutation-aggregate-'));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const input = join(directory, 'reports');
|
||||
const output = join(directory, 'output');
|
||||
for (const [index, shard] of fixtures().entries()) {
|
||||
const folder = join(input, `artifact-${2 - index}`);
|
||||
mkdirSync(folder, { recursive: true });
|
||||
for (const [name, value] of Object.entries({
|
||||
selection: shard.selection,
|
||||
'all-functions': shard.allFunctions,
|
||||
functions: shard.functions,
|
||||
mutation: shard.report,
|
||||
progress: shard.progress,
|
||||
})) {
|
||||
writeFileSync(join(folder, `${name}.json`), JSON.stringify(value));
|
||||
}
|
||||
}
|
||||
const args = [
|
||||
'scripts/quality/aggregate.mts',
|
||||
'--directory',
|
||||
input,
|
||||
'--head',
|
||||
options.head,
|
||||
'--workspace',
|
||||
options.workspace,
|
||||
'--count',
|
||||
'3',
|
||||
'--output',
|
||||
output,
|
||||
];
|
||||
const stepSummary = join(directory, 'step-summary.md');
|
||||
const result = spawnSync(process.execPath, args, {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, GITHUB_STEP_SUMMARY: stepSummary },
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(JSON.parse(readFileSync(join(output, 'aggregate.json'), 'utf8')).total, 10);
|
||||
assert.match(readFileSync(join(output, 'summary.md'), 'utf8'), /50.00%/);
|
||||
assert.equal(readFileSync(stepSummary, 'utf8'), readFileSync(join(output, 'summary.md'), 'utf8'));
|
||||
writeFileSync(join(input, 'artifact-0', 'progress.json'), 'not-json-secret-payload');
|
||||
const failed = spawnSync(process.execPath, args, { encoding: 'utf8' });
|
||||
assert.equal(failed.status, 1);
|
||||
assert.doesNotMatch(failed.stderr, /secret-payload/);
|
||||
assert.throws(() => readFileSync(join(output, 'summary.md')));
|
||||
});
|
||||
|
|
@ -14,6 +14,7 @@ import {
|
|||
|
||||
import { mutationSelection, riskMarkdown, riskReport } from './risk.mts';
|
||||
import { changes, git, WORKSPACES, workspaceFor } from './changes.mts';
|
||||
import { parseShard, shardTargets } from './shard.mts';
|
||||
import { coverageArgs } from './jest.mts';
|
||||
|
||||
const { values } = parseArgs({
|
||||
|
|
@ -22,14 +23,17 @@ const { values } = parseArgs({
|
|||
head: { type: 'string', default: 'HEAD' },
|
||||
workspace: { type: 'string' },
|
||||
check: { type: 'string' },
|
||||
shard: { type: 'string' },
|
||||
},
|
||||
});
|
||||
const workspace = WORKSPACES.find(({ name }) => name === values.workspace);
|
||||
if (!workspace || !['risk', 'mutation'].includes(values.check ?? '')) {
|
||||
throw new Error(
|
||||
'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation',
|
||||
'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation [--shard INDEX/COUNT]',
|
||||
);
|
||||
}
|
||||
const shard = parseShard(values.shard, values.check);
|
||||
const shardLabel = shard ? ` (shard ${shard.index}/${shard.total})` : '';
|
||||
const selection = changes(values.base, values.head);
|
||||
if (selection.head !== git(['rev-parse', 'HEAD']).trim()) {
|
||||
throw new Error('Check out the selected head before measuring its source');
|
||||
|
|
@ -38,14 +42,42 @@ const files = selection.files.filter(({ file }) => workspaceFor(file)?.name ===
|
|||
const output = resolve('coverage/changes', workspace.name, values.check!);
|
||||
rmSync(output, { recursive: true, force: true });
|
||||
mkdirSync(output, { recursive: true });
|
||||
writeFileSync(`${output}/selection.json`, JSON.stringify({ ...selection, files }, null, 2));
|
||||
writeFileSync(
|
||||
`${output}/selection.json`,
|
||||
JSON.stringify(
|
||||
{ ...selection, workspace: workspace.name, files, ...(shard ? { shard } : {}) },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
const summary = (markdown: string) => {
|
||||
console.log(markdown);
|
||||
writeFileSync(`${output}/summary.md`, markdown);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown);
|
||||
};
|
||||
const emptyMutationReports = () => {
|
||||
writeFileSync(
|
||||
`${output}/mutation.json`,
|
||||
JSON.stringify({ schemaVersion: '1.0', files: {} }, null, 2),
|
||||
);
|
||||
writeFileSync(
|
||||
`${output}/progress.json`,
|
||||
JSON.stringify(
|
||||
{ complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
};
|
||||
if (!files.length) {
|
||||
summary(`## ${workspace.name}: ${values.check}\n\nNo changed source files.\n`);
|
||||
if (shard) {
|
||||
writeFileSync(`${output}/all-functions.json`, '[]');
|
||||
writeFileSync(`${output}/functions.json`, '[]');
|
||||
emptyMutationReports();
|
||||
}
|
||||
summary(
|
||||
`## ${workspace.name}: ${values.check}${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No changed source files.'}\n`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
// Test binaries and configuration belong to the measured head, not the tooling checkout.
|
||||
|
|
@ -106,10 +138,17 @@ if (values.check === 'risk') {
|
|||
} else {
|
||||
process.env.NODE_ENV = env.NODE_ENV;
|
||||
process.env.BABEL_ENV = env.BABEL_ENV;
|
||||
const targets = mutationSelection(files, selection.base);
|
||||
const allTargets = mutationSelection(files, selection.base);
|
||||
const targets = shardTargets(allTargets, shard);
|
||||
if (shard) {
|
||||
writeFileSync(`${output}/all-functions.json`, JSON.stringify(allTargets, null, 2));
|
||||
}
|
||||
writeFileSync(`${output}/functions.json`, JSON.stringify(targets, null, 2));
|
||||
if (!targets.length) {
|
||||
summary(`## ${workspace.name}: mutation testing\n\nNo added or edited functions.\n`);
|
||||
if (shard) emptyMutationReports();
|
||||
summary(
|
||||
`## ${workspace.name}: mutation testing${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No added or edited functions.'}\n`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const started = Date.now();
|
||||
|
|
@ -176,7 +215,7 @@ if (values.check === 'risk') {
|
|||
const score = tested ? `${((100 * detected) / tested).toFixed(2)}%` : 'N/A (no valid mutants)';
|
||||
summary(
|
||||
[
|
||||
`## ${workspace.name}: mutation testing (advisory)`,
|
||||
`## ${workspace.name}: mutation testing (advisory)${shardLabel}`,
|
||||
'',
|
||||
`Changed functions: ${targets.reduce((total, target) => total + target.functions.length, 0)} in ${targets.length} files. Static mutations are included. Duration: ${((Date.now() - started) / 60000).toFixed(1)} minutes.`,
|
||||
'',
|
||||
|
|
@ -198,7 +237,7 @@ if (values.check === 'risk') {
|
|||
if (runtimeErrors) process.exitCode = 1;
|
||||
} catch (error) {
|
||||
summary(
|
||||
`## ${workspace.name}: mutation testing\n\nStryker failed. This is not a mutation score. See the test log.\n`,
|
||||
`## ${workspace.name}: mutation testing${shardLabel}\n\nStryker failed. This is not a mutation score. See the test log.\n`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
|
|
|
|||
30
scripts/quality/shard.mts
Normal file
30
scripts/quality/shard.mts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
export interface Shard {
|
||||
index: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export function parseShard(
|
||||
value: string | undefined,
|
||||
check: string | undefined,
|
||||
): Shard | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (check !== 'mutation') throw new Error('--shard is only allowed for mutation checks');
|
||||
if (!/^[1-9]\d*\/[1-9]\d*$/u.test(value)) {
|
||||
throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT');
|
||||
}
|
||||
const [index, total] = value.split('/').map(Number);
|
||||
if (!Number.isSafeInteger(index) || !Number.isSafeInteger(total) || index > total) {
|
||||
throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT');
|
||||
}
|
||||
return { index, total };
|
||||
}
|
||||
|
||||
export function shardTargets<T extends { file: string; functions: readonly unknown[] }>(
|
||||
targets: readonly T[],
|
||||
shard?: Shard,
|
||||
): readonly T[] {
|
||||
if (!shard) return targets;
|
||||
return [...targets]
|
||||
.sort((a, b) => b.functions.length - a.functions.length || a.file.localeCompare(b.file))
|
||||
.filter((_target, index) => index % shard.total === shard.index - 1);
|
||||
}
|
||||
234
scripts/quality/shard.test.mts
Normal file
234
scripts/quality/shard.test.mts
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
import { test } from 'node:test';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
|
||||
import { parseShard, shardTargets } from './shard.mts';
|
||||
import { mutationTarget } from './risk.mts';
|
||||
import { git } from './changes.mts';
|
||||
|
||||
const root = fileURLToPath(new URL('../../', import.meta.url));
|
||||
const runner = fileURLToPath(new URL('./run.mts', import.meta.url));
|
||||
|
||||
const targets = Object.freeze(
|
||||
Array.from({ length: 8 }, (_value, index) =>
|
||||
Object.freeze(
|
||||
mutationTarget(
|
||||
`file-${index}.ts`,
|
||||
'const first = (ok) => ok ? true : false;\nconst second = () => false;',
|
||||
[{ start: 1, count: 2 }],
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
for (const [index, expected] of [
|
||||
[1, [0, 3, 6]],
|
||||
[2, [1, 4, 7]],
|
||||
[3, [2, 5]],
|
||||
] as const) {
|
||||
test(`shard ${index}/3 retains whole files and their full function and mutation ranges`, () => {
|
||||
const shard = parseShard(`${index}/3`, 'mutation');
|
||||
const chosen = shardTargets(targets, shard);
|
||||
assert.deepEqual(
|
||||
chosen,
|
||||
expected.map((offset) => targets[offset]),
|
||||
);
|
||||
for (const [position, offset] of expected.entries()) {
|
||||
assert.equal(chosen[position], targets[offset]);
|
||||
assert.equal(chosen[position].functions.length, 2);
|
||||
assert.equal(chosen[position].mutate, targets[offset].mutate);
|
||||
}
|
||||
assert.deepEqual(shardTargets(targets, shard), chosen);
|
||||
});
|
||||
}
|
||||
|
||||
test('three shards contain every full target exactly once without mutating input order', () => {
|
||||
const before = JSON.stringify(targets);
|
||||
const union = [1, 2, 3].flatMap((index) => shardTargets(targets, { index, total: 3 }));
|
||||
assert.equal(union.length, targets.length);
|
||||
assert.equal(new Set(union).size, targets.length);
|
||||
assert.deepEqual(new Set(union), new Set(targets));
|
||||
assert.deepEqual(
|
||||
union.flatMap(({ mutate }) => mutate).sort(),
|
||||
targets.flatMap(({ mutate }) => mutate).sort(),
|
||||
);
|
||||
assert.equal(JSON.stringify(targets), before);
|
||||
});
|
||||
|
||||
test('omitting --shard retains all original targets and allows risk checks', () => {
|
||||
assert.equal(parseShard(undefined, 'mutation'), undefined);
|
||||
assert.equal(parseShard(undefined, 'risk'), undefined);
|
||||
assert.equal(shardTargets(targets), targets);
|
||||
assert.deepEqual(shardTargets(targets, parseShard('1/1', 'mutation')), targets);
|
||||
});
|
||||
|
||||
test('equal function counts use file order regardless of input order without mutating input', () => {
|
||||
const reversed = Object.freeze([...targets].reverse());
|
||||
const before = JSON.stringify(reversed);
|
||||
for (const index of [1, 2, 3]) {
|
||||
const shard = parseShard(`${index}/3`, 'mutation');
|
||||
assert.deepEqual(shardTargets(reversed, shard), shardTargets(targets, shard));
|
||||
}
|
||||
assert.equal(JSON.stringify(reversed), before);
|
||||
assert.equal(shardTargets(reversed), reversed);
|
||||
assert.deepEqual(shardTargets(reversed, parseShard('1/1', 'mutation')), targets);
|
||||
});
|
||||
|
||||
test('function-weighted round robin spreads the known target file workload across three shards', () => {
|
||||
const workload = Object.freeze(
|
||||
(
|
||||
[
|
||||
['pool.ts', 25],
|
||||
['import.ts', 19],
|
||||
['nativefavorite.ts', 13],
|
||||
['mongo.ts', 11],
|
||||
['migrations.ts', 9],
|
||||
['user.ts', 8],
|
||||
['backend.ts', 3],
|
||||
['favorites/mongo.ts', 1],
|
||||
['methods/createMethods.ts', 1],
|
||||
] as const
|
||||
).map(([file, count]) =>
|
||||
Object.freeze({
|
||||
file,
|
||||
functions: Object.freeze(Array.from({ length: count }, (_value, index) => index)),
|
||||
}),
|
||||
),
|
||||
);
|
||||
const before = JSON.stringify(workload);
|
||||
const expected = [
|
||||
['pool.ts', 'mongo.ts', 'backend.ts'],
|
||||
['import.ts', 'migrations.ts', 'favorites/mongo.ts'],
|
||||
['nativefavorite.ts', 'user.ts', 'methods/createMethods.ts'],
|
||||
];
|
||||
const shuffled = Object.freeze([...workload].reverse());
|
||||
const shards = [1, 2, 3].map((index) => {
|
||||
const shard = parseShard(`${index}/3`, 'mutation');
|
||||
const chosen = shardTargets(shuffled, shard);
|
||||
assert.deepEqual(
|
||||
chosen.map(({ file }) => file),
|
||||
expected[index - 1],
|
||||
);
|
||||
assert.deepEqual(chosen, shardTargets(workload, shard));
|
||||
return chosen;
|
||||
});
|
||||
const union = shards.flat();
|
||||
assert.equal(union.length, workload.length);
|
||||
assert.equal(new Set(union).size, workload.length);
|
||||
assert.deepEqual(new Set(union), new Set(workload));
|
||||
assert.deepEqual(
|
||||
shards.map((chosen) => chosen.reduce((sum, target) => sum + target.functions.length, 0)),
|
||||
[39, 29, 22],
|
||||
);
|
||||
assert.equal(JSON.stringify(workload), before);
|
||||
assert.equal(shardTargets(shuffled), shuffled);
|
||||
});
|
||||
|
||||
test('a shard beyond the number of selected target files is empty', () => {
|
||||
assert.deepEqual(shardTargets(targets.slice(0, 1), parseShard('2/3', 'mutation')), []);
|
||||
assert.deepEqual(shardTargets([], parseShard('1/3', 'mutation')), []);
|
||||
});
|
||||
|
||||
for (const value of [
|
||||
'0/2',
|
||||
'4/3',
|
||||
'foo',
|
||||
'1/0',
|
||||
'1.1/2',
|
||||
'1/2/3',
|
||||
'',
|
||||
'-1/2',
|
||||
'1/-2',
|
||||
'01/2',
|
||||
'1/02',
|
||||
'+1/2',
|
||||
' 1/2',
|
||||
'1/2 ',
|
||||
'1e1/20',
|
||||
'1/9007199254740992',
|
||||
]) {
|
||||
test(`invalid shard ${JSON.stringify(value)} is rejected`, () => {
|
||||
assert.throws(() => parseShard(value, 'mutation'), /--shard must be INDEX\/COUNT/u);
|
||||
});
|
||||
}
|
||||
|
||||
test('risk checks reject --shard before attempting source selection', () => {
|
||||
assert.throws(() => parseShard('1/3', 'risk'), /only allowed for mutation/u);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
runner,
|
||||
'--workspace',
|
||||
'data-schemas',
|
||||
'--check',
|
||||
'risk',
|
||||
'--shard',
|
||||
'1/3',
|
||||
'--base',
|
||||
'nonexistent-shard-test-ref',
|
||||
],
|
||||
{ cwd: root, encoding: 'utf8', timeout: 15000 },
|
||||
);
|
||||
assert.notEqual(result.status, 0);
|
||||
assert.match(result.stderr, /--shard is only allowed for mutation checks/u);
|
||||
assert.doesNotMatch(result.stderr, /Not a valid object name/u);
|
||||
});
|
||||
|
||||
test('empty sharded CLI runs pass with truthful metadata; unsharded output stays unchanged', (t) => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'quality-shard-'));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const env = {
|
||||
...process.env,
|
||||
GIT_DIR: git(['rev-parse', '--absolute-git-dir'], root).trim(),
|
||||
GIT_WORK_TREE: directory,
|
||||
GITHUB_STEP_SUMMARY: '',
|
||||
};
|
||||
const output = join(directory, 'coverage/changes/data-schemas/mutation');
|
||||
for (const shard of ['2/3', undefined]) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
runner,
|
||||
'--workspace',
|
||||
'data-schemas',
|
||||
'--check',
|
||||
'mutation',
|
||||
'--base',
|
||||
'HEAD',
|
||||
...(shard ? ['--shard', shard] : []),
|
||||
],
|
||||
{ cwd: directory, env, encoding: 'utf8', timeout: 15000 },
|
||||
);
|
||||
assert.equal(result.status, 0, `${result.error ?? ''}\n${result.stdout}\n${result.stderr}`);
|
||||
const selection = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8'));
|
||||
assert.equal(selection.workspace, 'data-schemas');
|
||||
assert.deepEqual(selection.files, []);
|
||||
if (shard) {
|
||||
assert.deepEqual(selection.shard, { index: 2, total: 3 });
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')), {
|
||||
schemaVersion: '1.0',
|
||||
files: {},
|
||||
});
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')), {
|
||||
complete: true,
|
||||
completed: 0,
|
||||
total: 0,
|
||||
staticCompleted: 0,
|
||||
staticTotal: 0,
|
||||
});
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(output, 'all-functions.json'), 'utf8')), []);
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(output, 'functions.json'), 'utf8')), []);
|
||||
assert.match(result.stdout, /mutation \(shard 2\/3\)/u);
|
||||
assert.match(result.stdout, /No changed functions for shard\./u);
|
||||
} else {
|
||||
assert.equal(selection.shard, undefined);
|
||||
assert.equal(existsSync(join(output, 'all-functions.json')), false);
|
||||
assert.equal(existsSync(join(output, 'functions.json')), false);
|
||||
assert.equal(result.stdout, '## data-schemas: mutation\n\nNo changed source files.\n\n');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -26,8 +26,15 @@ interface Step {
|
|||
run?: string;
|
||||
with?: Record<string, string | number | boolean>;
|
||||
'working-directory'?: string;
|
||||
if?: string;
|
||||
env?: Record<string, string>;
|
||||
}
|
||||
interface Job {
|
||||
name?: string;
|
||||
needs?: string[];
|
||||
if?: string;
|
||||
'timeout-minutes'?: number | string;
|
||||
strategy?: { matrix: { report?: { check: string; shard: number }[]; workspace: string } };
|
||||
defaults?: { run: { 'working-directory': string } };
|
||||
steps: Step[];
|
||||
}
|
||||
|
|
@ -84,13 +91,60 @@ test('workflow separates tooling, source, installs and artifacts without changin
|
|||
],
|
||||
);
|
||||
assert.equal(runner['working-directory'], undefined);
|
||||
assert.equal(
|
||||
runner.run,
|
||||
'node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK"',
|
||||
assert.match(runner.run!, /args=\(\)/);
|
||||
assert.match(runner.run!, /--shard "\$QUALITY_SHARD"/);
|
||||
assert.match(runner.run!, /--base "\$QUALITY_BASE" --head "\$QUALITY_HEAD"/);
|
||||
assert.match(
|
||||
runner.run!,
|
||||
/--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/,
|
||||
);
|
||||
assert.equal(
|
||||
report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path,
|
||||
'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.check }}/',
|
||||
'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/',
|
||||
);
|
||||
});
|
||||
|
||||
test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => {
|
||||
assert.equal(report['timeout-minutes'], "${{ matrix.report.check == 'mutation' && 240 || 45 }}");
|
||||
assert.deepEqual(report.strategy?.matrix.report, [
|
||||
{ check: 'risk', shard: 0 },
|
||||
{ check: 'mutation', shard: 1 },
|
||||
{ check: 'mutation', shard: 2 },
|
||||
{ check: 'mutation', shard: 3 },
|
||||
]);
|
||||
assert.equal(runner.env?.QUALITY_CHECK, '${{ matrix.report.check }}');
|
||||
assert.equal(
|
||||
runner.env?.QUALITY_SHARD,
|
||||
"${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}",
|
||||
);
|
||||
const aggregate = workflow.jobs.mutation;
|
||||
assert.equal(aggregate.name, 'Mutation testing (${{ matrix.workspace }})');
|
||||
assert.deepEqual(aggregate.needs, ['select', 'reports']);
|
||||
assert.match(aggregate.if!, /^always\(\)/);
|
||||
assert.equal(checkouts(aggregate)[0].with?.ref, '${{ github.sha }}');
|
||||
assert.equal(
|
||||
aggregate.steps.some(({ run }) => run?.includes('npm ci')),
|
||||
false,
|
||||
);
|
||||
const download = aggregate.steps.find(({ uses }) => uses === 'actions/download-artifact@v7');
|
||||
assert.equal(
|
||||
download?.with?.pattern,
|
||||
'change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}',
|
||||
);
|
||||
assert.equal(download?.with?.['merge-multiple'], undefined);
|
||||
const verify = aggregate.steps.findIndex(({ name }) => name === 'Verify all mutation shards');
|
||||
const requireSuccess = aggregate.steps.findIndex(
|
||||
({ name }) => name === 'Require successful report jobs',
|
||||
);
|
||||
assert.ok(verify >= 0 && requireSuccess > verify);
|
||||
assert.match(aggregate.steps[verify].run!, /--head "\$QUALITY_HEAD"/);
|
||||
assert.match(aggregate.steps[verify].run!, /--count 3/);
|
||||
assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success');
|
||||
const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6');
|
||||
assert.equal(upload?.if, 'always()');
|
||||
assert.equal(
|
||||
upload?.with?.name,
|
||||
'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -189,6 +243,7 @@ for (const olderHead of [true, false]) {
|
|||
...process.env,
|
||||
QUALITY_HEAD: head,
|
||||
QUALITY_WORKSPACE: 'data-schemas',
|
||||
QUALITY_SHARD: '',
|
||||
GITHUB_OUTPUT: '',
|
||||
GITHUB_STEP_SUMMARY: '',
|
||||
};
|
||||
|
|
@ -249,6 +304,51 @@ for (const olderHead of [true, false]) {
|
|||
const progress = JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8'));
|
||||
assert.equal(progress.complete, true);
|
||||
assert.ok(progress.completed > 0);
|
||||
const unsharded = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8'));
|
||||
const identify = (report: { files: Record<string, { mutants: unknown[] }> }) =>
|
||||
Object.entries(report.files)
|
||||
.flatMap(([file, { mutants }]) =>
|
||||
mutants.map((value) => {
|
||||
const mutant = value as {
|
||||
location: unknown;
|
||||
mutatorName: string;
|
||||
replacement: string;
|
||||
};
|
||||
return JSON.stringify([
|
||||
file,
|
||||
mutant.location,
|
||||
mutant.mutatorName,
|
||||
mutant.replacement,
|
||||
]);
|
||||
}),
|
||||
)
|
||||
.sort();
|
||||
const shardReports = join(directory, 'mutation-reports');
|
||||
const selectedMutants: string[] = [];
|
||||
for (let index = 1; index <= 3; index++) {
|
||||
run(runner.run!, source, {
|
||||
QUALITY_BASE: base,
|
||||
QUALITY_CHECK: 'mutation',
|
||||
QUALITY_SHARD: `${index}/3`,
|
||||
});
|
||||
const shard = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8'));
|
||||
assert.deepEqual(shard.shard, { index, total: 3 });
|
||||
assert.equal(shard.workspace, 'data-schemas');
|
||||
const mutation = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8'));
|
||||
selectedMutants.push(...identify(mutation));
|
||||
cpSync(output, join(shardReports, `shard-${index}`), { recursive: true });
|
||||
}
|
||||
assert.deepEqual(selectedMutants.sort(), identify(unsharded));
|
||||
const aggregateOutput = join(directory, 'aggregate');
|
||||
run(
|
||||
`node ../tooling/scripts/quality/aggregate.mts --directory "${shardReports}" --head "${head}" --workspace data-schemas --count 3 --output "${aggregateOutput}"`,
|
||||
source,
|
||||
{},
|
||||
);
|
||||
const aggregate = JSON.parse(readFileSync(join(aggregateOutput, 'aggregate.json'), 'utf8'));
|
||||
assert.equal(aggregate.complete, true);
|
||||
assert.equal(aggregate.head, head);
|
||||
assert.equal(aggregate.total, selectedMutants.length);
|
||||
}
|
||||
assert.equal(git(['rev-parse', 'HEAD'], source).trim(), head);
|
||||
assert.equal(git(['diff', 'HEAD', '--', file], source), '');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue