From 42219ec8e95302076ca6812abd2a056283526eae Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:07:07 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=9C=20feat:=20Support=20Native=20Postg?= =?UTF-8?q?reSQL=20Tool=20Favorites=20(#18032)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚜 feat: Support Native PostgreSQL Tool Favorites * fix: Guard PostgreSQL Reconciliation and Connection Leases * fix: Scope Favorites Erasure Fences to Deletion Attempts * fix: Preserve Account Erasure Ownership Until Cancellation Completes * fix: Reject Unsupported PostgreSQL Command Flags Before Connecting * fix: Keep Command Validation Compatible With the TypeScript Target * fix: Keep Migration Previews Read-Only and Preserve Erasure Recovery Identity * fix: Fence Erasure Recovery Across Tenant and Ownership Changes * fix: Isolate Migration Dependencies and Keep Mongo Scans Outside Transactions * fix: Inject PostgreSQL Backend Factories at Composition Roots * ci: Shard Complete Mutation Work Without Relaxing the Aggregate Gate --------- Co-authored-by: Lia --- .github/workflows/change-quality.yml | 121 +++- api/models/index.js | 17 +- api/models/index.spec.js | 55 +- api/server/controllers/UserController.js | 25 +- api/server/controllers/UserController.spec.js | 84 ++- .../controllers/__tests__/deleteUser.spec.js | 10 +- api/server/experimental.js | 9 +- api/server/index.js | 9 +- api/server/persistence.spec.js | 137 ++++ api/server/routes/admin/users.js | 1 + config/__tests__/delete-user.recovery.spec.js | 259 +++++++ config/__tests__/delete-user.spec.js | 115 +++- .../migrate-shared-link-permissions.spec.js | 1 + config/delete-user.js | 53 +- config/postgres.js | 21 + package-lock.json | 154 ++++- package.json | 3 +- packages/api/src/admin/users.spec.ts | 64 +- packages/api/src/admin/users.ts | 66 +- packages/api/src/app/index.ts | 4 + packages/api/src/app/loader.ts | 31 + packages/api/src/app/persistence.spec.ts | 319 +++++++++ packages/api/src/app/persistence.ts | 100 +++ packages/api/src/app/postgres.spec.ts | 648 ++++++++++++++++++ packages/api/src/app/postgres.ts | 164 +++++ packages/api/src/user/erasure.spec.ts | 343 ++++++++- packages/api/src/user/erasure.ts | 72 +- packages/api/src/user/resolver.spec.ts | 6 +- packages/api/src/user/resolver.ts | 10 + packages/data-provider/src/config.ts | 29 + .../data-provider/src/persistence.spec.ts | 33 + .../data-schemas/jest.postgres.config.mjs | 9 + packages/data-schemas/package.json | 7 +- packages/data-schemas/src/index.ts | 8 + packages/data-schemas/src/methods/favorite.ts | 11 +- packages/data-schemas/src/methods/index.ts | 5 +- .../src/methods/user.methods.spec.ts | 188 +++++ packages/data-schemas/src/methods/user.ts | 129 +++- .../data-schemas/src/postgres/backend.spec.ts | 117 ++++ packages/data-schemas/src/postgres/backend.ts | 20 + .../src/postgres/favorite.spec.ts | 303 ++++++++ .../data-schemas/src/postgres/favorite.ts | 121 ++++ .../data-schemas/src/postgres/import.spec.ts | 369 ++++++++++ packages/data-schemas/src/postgres/import.ts | 243 +++++++ .../src/postgres/migrations.spec.ts | 98 +++ .../data-schemas/src/postgres/migrations.ts | 158 +++++ .../data-schemas/src/postgres/mongo.spec.ts | 102 +++ packages/data-schemas/src/postgres/mongo.ts | 95 +++ .../data-schemas/src/postgres/pool.spec.ts | 237 +++++++ packages/data-schemas/src/postgres/pool.ts | 190 +++++ .../src/postgres/postgres.helper.ts | 51 ++ packages/data-schemas/src/schema/user.ts | 4 + packages/data-schemas/src/types/favorite.ts | 8 + packages/data-schemas/src/types/user.ts | 2 + scripts/quality/aggregate.mts | 331 +++++++++ scripts/quality/aggregate.test.mts | 351 ++++++++++ scripts/quality/run.mts | 53 +- scripts/quality/shard.mts | 30 + scripts/quality/shard.test.mts | 234 +++++++ scripts/quality/workflow.test.mts | 108 ++- 60 files changed, 6433 insertions(+), 112 deletions(-) create mode 100644 api/server/persistence.spec.js create mode 100644 config/__tests__/delete-user.recovery.spec.js create mode 100644 config/postgres.js create mode 100644 packages/api/src/app/persistence.spec.ts create mode 100644 packages/api/src/app/persistence.ts create mode 100644 packages/api/src/app/postgres.spec.ts create mode 100644 packages/api/src/app/postgres.ts create mode 100644 packages/data-provider/src/persistence.spec.ts create mode 100644 packages/data-schemas/jest.postgres.config.mjs create mode 100644 packages/data-schemas/src/postgres/backend.spec.ts create mode 100644 packages/data-schemas/src/postgres/backend.ts create mode 100644 packages/data-schemas/src/postgres/favorite.spec.ts create mode 100644 packages/data-schemas/src/postgres/favorite.ts create mode 100644 packages/data-schemas/src/postgres/import.spec.ts create mode 100644 packages/data-schemas/src/postgres/import.ts create mode 100644 packages/data-schemas/src/postgres/migrations.spec.ts create mode 100644 packages/data-schemas/src/postgres/migrations.ts create mode 100644 packages/data-schemas/src/postgres/mongo.spec.ts create mode 100644 packages/data-schemas/src/postgres/mongo.ts create mode 100644 packages/data-schemas/src/postgres/pool.spec.ts create mode 100644 packages/data-schemas/src/postgres/pool.ts create mode 100644 packages/data-schemas/src/postgres/postgres.helper.ts create mode 100644 scripts/quality/aggregate.mts create mode 100644 scripts/quality/aggregate.test.mts create mode 100644 scripts/quality/shard.mts create mode 100644 scripts/quality/shard.test.mts diff --git a/.github/workflows/change-quality.yml b/.github/workflows/change-quality.yml index a15505418e..56dec2321c 100644 --- a/.github/workflows/change-quality.yml +++ b/.github/workflows/change-quality.yml @@ -54,6 +54,7 @@ jobs: outputs: workspaces: ${{ steps.select.outputs.workspaces }} hasChanges: ${{ steps.select.outputs.hasChanges }} + postgresChanged: ${{ steps.postgres.outputs.changed }} steps: - uses: actions/checkout@v5 with: @@ -69,13 +70,56 @@ jobs: QUALITY_SCOPE: ${{ inputs.scope || 'data-schemas' }} run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE" + - name: Select PostgreSQL schema and lifecycle changes + id: postgres + run: | + if git diff --name-only "$QUALITY_BASE...$QUALITY_HEAD" | grep -Eq '^(packages/data-schemas/src/(postgres/|types/favorite|methods/(favorite|index))|packages/api/src/app/(persistence|postgres)|packages/data-provider/src/config|api/(models/index|server/(index|experimental))|config/(postgres|delete-user)|packages/data-schemas/jest.postgres|package-lock.json|.github/workflows/change-quality.yml)'; then + echo 'changed=true' >> "$GITHUB_OUTPUT" + fi + + postgres: + name: PostgreSQL 16 contracts and readiness + needs: [select] + if: needs.select.outputs.postgresChanged == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 5 + env: + PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres + NODE_OPTIONS: --max-old-space-size=6144 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ env.QUALITY_HEAD }} + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: '24.22.1' + cache: npm + - run: npm ci + - run: npm run build:data-provider && npm run build:data-schemas && npm run build:api + - run: npm run test:postgres --workspace packages/data-schemas + - run: npx --no-install jest src/app/persistence.spec.ts src/app/postgres.spec.ts src/favorites/handlers.spec.ts src/user/erasure.spec.ts --coverage=false --runInBand + working-directory: packages/api + - run: npx --no-install jest --config config/jest.config.js --runTestsByPath config/__tests__/delete-user.recovery.spec.js --coverage=false --runInBand + reports: - name: ${{ matrix.check == 'risk' && 'Change risk' || 'Mutation testing' }} (${{ matrix.workspace }}) + name: ${{ matrix.report.check == 'risk' && format('Change risk ({0})', matrix.workspace) || format('Mutation shard {0}/3 ({1})', matrix.report.shard, matrix.workspace) }} needs: [select, verify] if: needs.select.outputs.hasChanges == 'true' runs-on: ubuntu-latest # Whole edited database functions can produce thousands of mutation cases. - timeout-minutes: ${{ matrix.check == 'mutation' && 240 || 45 }} + timeout-minutes: ${{ matrix.report.check == 'mutation' && 240 || 45 }} defaults: run: working-directory: source @@ -83,7 +127,11 @@ jobs: fail-fast: false matrix: workspace: ${{ fromJSON(needs.select.outputs.workspaces) }} - check: [risk, mutation] + report: + - { check: risk, shard: 0 } + - { check: mutation, shard: 1 } + - { check: mutation, shard: 2 } + - { check: mutation, shard: 3 } services: redis: image: redis:7-alpine @@ -93,7 +141,18 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 5 + postgres: + image: postgres:16-alpine + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 5 env: + PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres REDIS_URI: redis://127.0.0.1:6379 NODE_OPTIONS: --max-old-space-size=6144 steps: @@ -138,15 +197,59 @@ jobs: - name: Generate advisory report env: QUALITY_WORKSPACE: ${{ matrix.workspace }} - QUALITY_CHECK: ${{ matrix.check }} - run: >- - node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" - --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" + QUALITY_CHECK: ${{ matrix.report.check }} + QUALITY_SHARD: ${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }} + run: | + args=() + if [ -n "$QUALITY_SHARD" ]; then args+=(--shard "$QUALITY_SHARD"); fi + node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \ + --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" "${args[@]}" - name: Upload reports if: always() uses: actions/upload-artifact@v6 with: - name: change-${{ matrix.check }}-${{ matrix.workspace }}-${{ github.event.pull_request.head.sha || github.sha }} - path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.check }}/ + name: change-${{ matrix.report.check }}-${{ matrix.workspace }}${{ matrix.report.shard && format('-shard-{0}', matrix.report.shard) || '' }}-${{ env.QUALITY_HEAD }} + path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/ + if-no-files-found: warn + retention-days: 14 + + mutation: + name: Mutation testing (${{ matrix.workspace }}) + needs: [select, reports] + if: always() && needs.select.result == 'success' && needs.select.outputs.hasChanges == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + workspace: ${{ fromJSON(needs.select.outputs.workspaces) }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: '24.22.1' + - uses: actions/download-artifact@v7 + with: + pattern: change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }} + path: mutation-reports + - name: Verify all mutation shards + env: + QUALITY_WORKSPACE: ${{ matrix.workspace }} + run: >- + node scripts/quality/aggregate.mts --directory mutation-reports --head "$QUALITY_HEAD" + --workspace "$QUALITY_WORKSPACE" --count 3 + --output "coverage/changes/$QUALITY_WORKSPACE/mutation" + - name: Require successful report jobs + env: + REPORT_RESULT: ${{ needs.reports.result }} + run: test "$REPORT_RESULT" = success + - uses: actions/upload-artifact@v6 + if: always() + with: + name: change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }} + path: coverage/changes/${{ matrix.workspace }}/mutation/ if-no-files-found: warn retention-days: 14 diff --git a/api/models/index.js b/api/models/index.js index 49a550e786..4e63df3a30 100644 --- a/api/models/index.js +++ b/api/models/index.js @@ -1,10 +1,16 @@ const mongoose = require('mongoose'); -const { createMethods } = require('@librechat/data-schemas'); +const { + createMethods, + createToolFavoriteMethods, + createPostgresFavoriteBackend, +} = require('@librechat/data-schemas'); const { matchModelName, findMatchingPattern, isDeploymentSkillId, createMessageBudgetReader, + createFavoriteRuntime, + getPersistenceConfig, createChannelStorageConfigReader, } = require('@librechat/api'); const getLogStores = require('~/cache/getLogStores'); @@ -12,7 +18,14 @@ const getLogStores = require('~/cache/getLogStores'); const messageBudget = createMessageBudgetReader(); const channelStorageConfig = createChannelStorageConfigReader(); +const favorites = createFavoriteRuntime( + createToolFavoriteMethods(mongoose), + process.env, + createPostgresFavoriteBackend, +); + const methods = createMethods(mongoose, { + toolFavorites: favorites.methods, matchModelName, findMatchingPattern, isExternalSkillId: isDeploymentSkillId, @@ -30,6 +43,8 @@ const seedDatabase = async () => { module.exports = { ...methods, + initializeFavorites: (config) => favorites.initialize(getPersistenceConfig(config)), + closeFavorites: favorites.close, initializeMessageBudget: messageBudget.initialize, initializeChannelStorageConfig: channelStorageConfig.initialize, seedDatabase, diff --git a/api/models/index.spec.js b/api/models/index.spec.js index d4f0aa369b..9bffb00742 100644 --- a/api/models/index.spec.js +++ b/api/models/index.spec.js @@ -1,5 +1,18 @@ -jest.mock('@librechat/data-schemas', () => ({ createMethods: jest.fn(() => ({})) })); +jest.mock('@librechat/data-schemas', () => ({ + createPostgresFavoriteBackend: jest.fn(), + createMethods: jest.fn((_mongoose, deps) => ({ ...deps.toolFavorites })), + createToolFavoriteMethods: jest.fn(() => ({ + getToolFavorites: jest.fn().mockResolvedValue([]), + addToolFavorite: jest.fn().mockResolvedValue({ ok: true, added: true }), + removeToolFavorite: jest.fn().mockResolvedValue({ ok: true, removed: true }), + deleteUserToolFavorites: jest.fn().mockResolvedValue(2), + })), +})); jest.mock('@librechat/api', () => ({ + ...jest.requireActual('../../packages/api/src/app/persistence'), + createFavoriteRuntime: jest.fn( + jest.requireActual('../../packages/api/src/app/persistence').createFavoriteRuntime, + ), matchModelName: jest.fn(), findMatchingPattern: jest.fn(), isDeploymentSkillId: jest.fn(), @@ -11,13 +24,25 @@ jest.mock('@librechat/api', () => ({ })); jest.mock('~/cache/getLogStores', () => jest.fn()); -const { createMethods } = require('@librechat/data-schemas'); -const { createMessageBudgetReader, createChannelStorageConfigReader } = require('@librechat/api'); -const { initializeMessageBudget, initializeChannelStorageConfig } = require('./index'); +const { + createToolFavoriteMethods, + createMethods, + createPostgresFavoriteBackend, +} = require('@librechat/data-schemas'); +const { + createMessageBudgetReader, + createFavoriteRuntime, + createChannelStorageConfigReader, +} = require('@librechat/api'); +const models = require('./index'); +const { initializeMessageBudget, initializeChannelStorageConfig } = models; const reader = createMessageBudgetReader.mock.results[0].value; const channelStorageConfigReader = createChannelStorageConfigReader.mock.results[0].value; +const dependencies = createMethods.mock.calls[0][1]; +const runtimeDependencies = createFavoriteRuntime.mock.calls[0]; const { getMCPAppMessageBudget, getChannelStorageConfig: channelStorageConfigDependency } = - createMethods.mock.calls[0][1]; + dependencies; +const mongoFavorites = createToolFavoriteMethods.mock.results[0].value; describe('message App budget dependency wiring', () => { it('passes the same reader to storage and initializer to the composition root', () => { @@ -27,6 +52,26 @@ describe('message App budget dependency wiring', () => { }); }); +describe('favorites composition root', () => { + it('passes stable runtime methods to storage and keeps captured deletion references usable', async () => { + expect(runtimeDependencies).toEqual([ + mongoFavorites, + process.env, + createPostgresFavoriteBackend, + ]); + const captured = models.deleteUserToolFavorites; + expect(dependencies.toolFavorites.deleteUserToolFavorites).toBe(captured); + await expect(captured('user')).rejects.toThrow('not initialized'); + await models.initializeFavorites(null); + expect(createPostgresFavoriteBackend).not.toHaveBeenCalled(); + expect(await captured('user')).toBe(2); + expect(mongoFavorites.deleteUserToolFavorites).toHaveBeenCalledWith('user'); + expect(models.deleteUserToolFavorites).toBe(captured); + await models.closeFavorites(); + await expect(captured('user')).rejects.toThrow('not initialized'); + }); +}); + describe('channel storage configuration dependency wiring', () => { it('injects the deployment reader without reading config during module initialization', () => { expect(channelStorageConfigDependency).toBe(channelStorageConfigReader.getConfig); diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js index 0c377973a5..c6392c3afb 100644 --- a/api/server/controllers/UserController.js +++ b/api/server/controllers/UserController.js @@ -17,6 +17,7 @@ const { stopUserGenerations, deleteAllSharedLinksWithCleanup, eraseUserData, + createUserDeletionAttempt, deleteUserMCPServers, revokeUserCodeEnvironmentWorkers, finalizeMCPAuthorizationMutation, @@ -370,6 +371,7 @@ const deleteUserController = async (req, res) => { let triggerDeletionFence; let scheduleSuspensionToken; let userDeleted = false; + const deletionAttempt = createUserDeletionAttempt(); try { const existingUser = await db.getUserById( @@ -391,7 +393,11 @@ const deleteUserController = async (req, res) => { // Block new trigger admissions across replicas while preserving the user // principal so a transient cleanup failure remains retryable. triggerDeletionFence = new Date(); - const fenceState = await db.beginAgentTriggerUserDeletion(user.id, triggerDeletionFence); + const fenceState = await db.beginAgentTriggerUserDeletion( + user.id, + triggerDeletionFence, + deletionAttempt.id, + ); if (fenceState === 'in_progress') { triggerDeletionFence = undefined; throw new Error('Agent trigger account deletion is already in progress'); @@ -425,6 +431,7 @@ const deleteUserController = async (req, res) => { await eraseUserData( { userId: user.id, + deletionAttemptId: deletionAttempt.id, tenantId, checkpointer: appConfig?.endpoints?.agents?.checkpointer, }, @@ -439,7 +446,12 @@ const deleteUserController = async (req, res) => { onSessionsDeleted: () => clearSessionHintCookie(res), }, ); - const deleteResult = await db.deleteUserById(user.id); + const deleteResult = await deletionAttempt.commit(() => + db.deleteUserById(user.id, { + startedAt: triggerDeletionFence, + attemptId: deletionAttempt.id, + }), + ); if (deleteResult.deletedCount !== 1) { throw new Error('User disappeared before account deletion could commit'); } @@ -506,7 +518,14 @@ const deleteUserController = async (req, res) => { ); } try { - await db.cancelAgentTriggerUserDeletion(user.id, triggerDeletionFence); + await deletionAttempt.cancel( + { + userId: user.id, + startedAt: triggerDeletionFence, + deletionAttemptId: deletionAttempt.id, + }, + db, + ); } catch (fenceError) { logger.error('[deleteUserController] Failed to release trigger deletion fence', fenceError); } diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.js index c9f242fd33..e149795a39 100644 --- a/api/server/controllers/UserController.spec.js +++ b/api/server/controllers/UserController.spec.js @@ -55,6 +55,7 @@ jest.mock('~/models', () => { deleteAclEntries: jest.fn().mockResolvedValue(undefined), deleteUserChatProjects: jest.fn().mockResolvedValue(0), deleteUserToolFavorites: jest.fn().mockResolvedValue(0), + cancelUserToolFavoriteDeletion: jest.fn().mockResolvedValue(undefined), deleteRefreshTokenBridges: jest.fn().mockResolvedValue(undefined), deleteAllAgentQueuedTurnsForUser: jest.fn().mockResolvedValue(0), findUserErasableMCPServers: jest.fn().mockResolvedValue([]), @@ -64,6 +65,7 @@ jest.mock('~/models', () => { deleteUserById: jest.fn().mockResolvedValue({ deletedCount: 1 }), deleteAllUserPluginAuths: jest.fn().mockResolvedValue(undefined), beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'), + isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true), cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true), deleteUserPrompts: jest.fn().mockResolvedValue(undefined), deleteUserSkills: jest.fn().mockResolvedValue(undefined), @@ -592,9 +594,20 @@ describe('deleteUserController', () => { expect(mockRes.status).toHaveBeenCalledWith(200); expect(mockRes.send).toHaveBeenCalledWith({ message: 'User deleted' }); + expect(deleteUserById).toHaveBeenCalledWith(userIdStr, { + startedAt: beginAgentTriggerUserDeletion.mock.calls[0][1], + attemptId: beginAgentTriggerUserDeletion.mock.calls[0][2], + }); + expect(deleteUserById.mock.calls[0][1].attemptId).not.toBe( + mockQuiesceUserSchedules.mock.calls[0][1], + ); expect(mockRes.clearCookie).toHaveBeenCalledWith('session_hint'); expect(deletePasskeysByUser).toHaveBeenCalledWith(userIdStr); - expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(userId.toString(), expect.any(Date)); + expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith( + userId.toString(), + expect.any(Date), + expect.any(String), + ); expect(mockPrepareAgentTriggerUserPurge).toHaveBeenCalledWith( userId.toString(), expect.any(Date), @@ -929,7 +942,11 @@ describe('deleteUserController', () => { expect(deleteMessages).not.toHaveBeenCalled(); const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1]; expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence); - expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence); + expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( + userIdString, + deletionFence, + expect.any(String), + ); expect(deleteUserById).not.toHaveBeenCalled(); // Account survives -> its suspended schedules are restored under the quiesce token. expect(mockRestoreUserSchedules).toHaveBeenCalledWith( @@ -944,6 +961,42 @@ describe('deleteUserController', () => { ); }); + it.each([false, 'error'])( + 'does not release the favorite marker without a proven Mongo owner: %s', + async (result) => { + const db = require('~/models'); + db.deleteUserAgents.mockRejectedValueOnce(new Error('later cascade failure')); + if (result === 'error') + db.isAgentTriggerUserDeletionOwner.mockRejectedValueOnce(new Error('Mongo unavailable')); + else db.isAgentTriggerUserDeletionOwner.mockResolvedValueOnce(false); + const userId = new mongoose.Types.ObjectId().toString(); + await deleteUserController({ user: { id: userId, tenantId: 'tenant-1' } }, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(500); + expect(db.deleteUserToolFavorites).toHaveBeenCalledWith( + userId, + beginAgentTriggerUserDeletion.mock.calls[0][2], + ); + expect(db.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled(); + }, + ); + + it('releases the cascade UUID before releasing Mongo ownership', async () => { + const db = require('~/models'); + db.deleteUserAgents.mockRejectedValueOnce(new Error('later cascade failure')); + const userId = new mongoose.Types.ObjectId().toString(); + await deleteUserController({ user: { id: userId, tenantId: 'tenant-1' } }, mockRes); + const token = beginAgentTriggerUserDeletion.mock.calls[0][2]; + expect(mockRes.status).toHaveBeenCalledWith(500); + expect(db.deleteUserToolFavorites).toHaveBeenCalledWith(userId, token); + expect(db.cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(userId, token); + expect(db.isAgentTriggerUserDeletionOwner.mock.invocationCallOrder[0]).toBeLessThan( + db.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0], + ); + expect(db.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]).toBeLessThan( + db.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0], + ); + }); + it('fails closed before data cleanup when detached subagents do not drain', async () => { const userId = new mongoose.Types.ObjectId(); mockCancelAndDrainSubagentThreads.mockRejectedValueOnce(new Error('child drain timed out')); @@ -963,6 +1016,7 @@ describe('deleteUserController', () => { expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( userId.toString(), expect.any(Date), + expect.any(String), ); expect(deleteUserById).not.toHaveBeenCalled(); }); @@ -987,7 +1041,11 @@ describe('deleteUserController', () => { expect(mockGetActiveJobIdsForUser).not.toHaveBeenCalled(); const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1]; expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence); - expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence); + expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( + userIdString, + deletionFence, + expect.any(String), + ); expect(deleteUserById).not.toHaveBeenCalled(); // Account survives -> its suspended schedules are restored under the quiesce token. expect(mockRestoreUserSchedules).toHaveBeenCalledWith( @@ -1074,6 +1132,7 @@ describe('deleteUserController', () => { expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( userId.toString(), expect.any(Date), + expect.any(String), ); expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith( userId.toString(), @@ -1090,7 +1149,22 @@ describe('deleteUserController', () => { expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled(); expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date)); - expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date)); + expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( + String(userId), + expect.any(Date), + expect.any(String), + ); + }); + + it('retains both erasure fences after an unknown final delete outcome', async () => { + const db = require('~/models'); + const userId = new mongoose.Types.ObjectId().toString(); + db.deleteUserById.mockRejectedValueOnce(new Error('delete acknowledgement lost')); + await deleteUserController({ user: { id: userId } }, mockRes); + expect(mockRes.status).toHaveBeenCalledWith(500); + expect(db.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled(); + expect(db.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled(); + expect(db.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); }); it('does not purge trigger payloads when the user deletion did not commit', async () => { @@ -1101,7 +1175,7 @@ describe('deleteUserController', () => { expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled(); expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date)); - expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date)); + expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); expect(mockRes.status).toHaveBeenCalledWith(500); }); diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js index f043b2886a..10408ede8c 100644 --- a/api/server/controllers/__tests__/deleteUser.spec.js +++ b/api/server/controllers/__tests__/deleteUser.spec.js @@ -70,6 +70,10 @@ jest.mock('@librechat/api', () => ({ getNewS3URL: jest.fn(), deleteAllSharedLinksWithCleanup: (...args) => mockDeleteAllSharedLinksWithCleanup(...args), eraseUserData: jest.requireActual('../../../../packages/api/src/user/erasure.ts').eraseUserData, + createUserDeletionAttempt: jest.requireActual('../../../../packages/api/src/user/erasure.ts') + .createUserDeletionAttempt, + cancelUserDataDeletion: jest.requireActual('../../../../packages/api/src/user/erasure.ts') + .cancelUserDataDeletion, revokeUserCodeEnvironmentWorkers: (...args) => mockRevokeUserCodeEnvironmentWorkers(...args), GenerationJobManager: { getAccountCleanupJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args), @@ -98,6 +102,7 @@ jest.mock('~/models', () => ({ deleteUserById: (...args) => mockDeleteUserById(...args), deleteAllUserPluginAuths: (...args) => mockDeleteAllUserPluginAuths(...args), beginAgentTriggerUserDeletion: (...args) => mockBeginAgentTriggerUserDeletion(...args), + isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true), cancelAgentTriggerUserDeletion: (...args) => mockCancelAgentTriggerUserDeletion(...args), deleteMessages: (...args) => mockDeleteMessages(...args), deletePresets: (...args) => mockDeletePresets(...args), @@ -429,7 +434,10 @@ describe('deleteUserController - 2FA enforcement', () => { expect(res.status).toHaveBeenCalledWith(200); expect(mockBeginAgentTriggerUserDeletion).toHaveBeenCalledTimes(1); expect(mockDeleteMessages).toHaveBeenCalled(); - expect(mockDeleteUserById).toHaveBeenCalledWith(USER_ID); + expect(mockDeleteUserById).toHaveBeenCalledWith(USER_ID, { + startedAt: mockBeginAgentTriggerUserDeletion.mock.calls[0][1], + attemptId: mockBeginAgentTriggerUserDeletion.mock.calls[0][2], + }); expect(mockDeleteUserCodeEnvironments).not.toHaveBeenCalled(); expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled(); expect(mockCancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); diff --git a/api/server/experimental.js b/api/server/experimental.js index bc4b1b1103..fcacc7d851 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -97,6 +97,8 @@ const { getRoleByName, updateAccessPermissions, seedDatabase, + initializeFavorites, + closeFavorites, recordAuditEntry, sweepOrphanedPreviews, } = require('~/models'); @@ -104,7 +106,7 @@ const { checkMigrations } = require('./services/start/migration'); const initializeMCPs = require('./services/initializeMCPs'); const configureSocialLogins = require('./socialLogins'); const createSpaFallback = require('./utils/fallback'); -const { getAppConfig } = require('./services/Config'); +const { getAppConfig, loadCustomConfig } = require('./services/Config'); const staticCache = require('./utils/staticCache'); const optionalJwtAuth = require('./middleware/optionalJwtAuth'); const noIndex = require('./middleware/noIndex'); @@ -524,6 +526,8 @@ if (cluster.isMaster) { * deletion, the expired-file sweep, tool-approval hooks and the other worker-wide settings * below are fixed for the process lifetime, so DB `__base__` overrides must not decide them. */ + await initializeFavorites(await loadCustomConfig(false, { storageSelection: true })); + registerShutdownTask('favorites persistence', closeFavorites, { priority: -100 }); const appConfig = await getAppConfig({ baseOnly: true }); initializeFileStorage(appConfig); initializeGitHubSkillSync(appConfig); @@ -791,7 +795,8 @@ if (cluster.isMaster) { setupGracefulShutdown(server); }; - startServer().catch((err) => { + startServer().catch(async (err) => { + await closeFavorites().catch(() => undefined); logger.error(`Failed to start worker ${process.pid}:`, err); process.exit(1); }); diff --git a/api/server/index.js b/api/server/index.js index 71b0fb53b8..e389450625 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -78,6 +78,8 @@ const { sweepOrphanedPreviews, getRoleByName, seedDatabase, + initializeFavorites, + closeFavorites, listAllConfigs, recordAuditEntry, } = require('~/models'); @@ -111,7 +113,7 @@ const initializeMCPs = require('./services/initializeMCPs'); const { configureSubagentTaskRouting } = require('./services/Endpoints/agents/subagentThreadStore'); const configureSocialLogins = require('./socialLogins'); const createSpaFallback = require('./utils/fallback'); -const { getAppConfig, rederiveEndpoints } = require('./services/Config'); +const { getAppConfig, rederiveEndpoints, loadCustomConfig } = require('./services/Config'); const staticCache = require('./utils/staticCache'); const noIndex = require('./middleware/noIndex'); const agentEventMethods = require('~/models'); @@ -257,6 +259,8 @@ const startServer = async () => { runAsSystem(sweepOrphanedPreviews).catch((err) => { logger.error('[sweepOrphanedPreviews] Background sweep failed:', err); }); + await initializeFavorites(await loadCustomConfig(false, { storageSelection: true })); + registerShutdownTask('favorites persistence', closeFavorites, { priority: -100 }); const appConfig = await getAppConfig({ baseOnly: true }); await configureSubagentTaskRouting(appConfig?.endpoints?.agents?.subagentActivity); registerBackgroundTaskShutdown({ @@ -618,7 +622,8 @@ const startServer = async () => { * liveness probe eventually times out. Mirrors the pattern in * `experimental.js`. */ -startServer().catch((err) => { +startServer().catch(async (err) => { + await closeFavorites().catch(() => undefined); logger.error('Failed to start server:', err); process.exit(1); }); diff --git a/api/server/persistence.spec.js b/api/server/persistence.spec.js new file mode 100644 index 0000000000..e6b5c3d104 --- /dev/null +++ b/api/server/persistence.spec.js @@ -0,0 +1,137 @@ +const { readFileSync } = require('node:fs'); +const { resolve, dirname } = require('node:path'); +const { runInNewContext } = require('node:vm'); + +describe.each(['index', 'experimental'])('%s entrypoint lifecycle wiring', (entrypoint) => { + it.each([false, true])( + 'retains the original startup error and cleans up (persistence failure: %s)', + async (failFavorites) => { + const file = resolve(__dirname, `${entrypoint}.js`); + const events = []; + const errors = []; + const tasks = new Map(); + const bootError = new Error('boot failure'); + const noop = new Proxy( + function () { + return noop; + }, + { + get(_target, key) { + return key === 'then' ? undefined : noop; + }, + }, + ); + const closeFavorites = async () => { + events.push('close'); + if (failFavorites) { + throw new Error('close failure'); + } + }; + const api = new Proxy( + { + cacheConfig: {}, + isEnabled: () => false, + waitForKeyvRedisClient: async () => {}, + createMetrics: () => ({}), + registerShutdownTask: (name, fn) => tasks.set(name, fn), + }, + { + get(target, key) { + return Reflect.has(target, key) ? Reflect.get(target, key) : noop; + }, + }, + ); + const models = new Proxy( + { + seedDatabase: async () => {}, + sweepOrphanedPreviews: async () => {}, + initializeFavorites: async (config) => { + events.push('initialize'); + expect(config.persistence.favorites).toBe('postgres'); + if (failFavorites) { + throw bootError; + } + }, + closeFavorites, + }, + { + get(target, key) { + return Reflect.has(target, key) ? Reflect.get(target, key) : noop; + }, + }, + ); + runInNewContext( + readFileSync(file, 'utf8'), + { + require: (id) => { + if (id === 'path') { + return { resolve, dirname }; + } + if (id === 'cluster') { + return { isMaster: false }; + } + if (id === 'express') { + return () => ({ locals: {}, disable() {}, set() {}, use() {} }); + } + if (id === '@librechat/api') { + return api; + } + if (id === '@librechat/data-schemas') { + return { + logger: { info() {}, warn() {}, error: (...args) => errors.push(args) }, + runAsSystem: async (fn) => fn(), + }; + } + if (id === '~/models') { + return models; + } + if (id === '~/db') { + return { connectDb: async () => {}, indexSync: async () => {} }; + } + if (id === './services/Config') { + return { + loadCustomConfig: async (print, options) => { + expect(print).toBe(false); + expect(options).toEqual({ storageSelection: true }); + return { persistence: { favorites: 'postgres' } }; + }, + getAppConfig: async () => { + events.push('config'); + throw bootError; + }, + }; + } + return noop; + }, + __dirname: dirname(file), + module: { exports: {} }, + console, + process: { + env: {}, + pid: 1, + on() {}, + exit: (code) => events.push(`exit:${code}`), + }, + setTimeout, + clearTimeout, + setInterval, + clearInterval, + Buffer, + }, + { filename: file }, + ); + await new Promise((resolve) => setImmediate(resolve)); + expect(events[0]).toBe('initialize'); + expect(events).toContain('close'); + expect(events).toContain('exit:1'); + const last = errors[errors.length - 1]; + expect(last[last.length - 1]).toBe(bootError); + if (failFavorites) { + expect(events).not.toContain('config'); + } else { + expect(tasks.get('favorites persistence')).toBe(closeFavorites); + expect(events).toContain('config'); + } + }, + ); +}); diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js index 74b1bcbf11..d0fa10503e 100644 --- a/api/server/routes/admin/users.js +++ b/api/server/routes/admin/users.js @@ -60,6 +60,7 @@ const handlers = createAdminUsersHandlers({ findUsers: db.findUsers, countUsers: db.countUsers, beginAgentTriggerUserDeletion: db.beginAgentTriggerUserDeletion, + isAgentTriggerUserDeletionOwner: db.isAgentTriggerUserDeletionOwner, cancelAgentTriggerUserDeletion: db.cancelAgentTriggerUserDeletion, drainAgentTriggerDeliveriesForUser, prepareAgentTriggerUserPurge, diff --git a/config/__tests__/delete-user.recovery.spec.js b/config/__tests__/delete-user.recovery.spec.js new file mode 100644 index 0000000000..85aafe7d5f --- /dev/null +++ b/config/__tests__/delete-user.recovery.spec.js @@ -0,0 +1,259 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const { randomUUID } = require('node:crypto'); +const mongoose = require('mongoose'); +const { MongoMemoryServer } = require('mongodb-memory-server'); +const schemas = require('@librechat/data-schemas'); +const api = require('@librechat/api'); + +const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip; +realPostgres('delete-user CLI durable erasure recovery', () => { + const schema = `cli_erasure_${randomUUID().replace(/-/g, '')}`; + const config = { + persistence: { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } }, + }; + const options = { + connectionString: process.env.PG_TEST_URL ?? '', + max: 2, + maxWaiting: 10, + acquireTimeoutMs: 1000, + queryTimeoutMs: 5000, + idleTimeoutMs: 1000, + }; + let mongo; + let admin; + let db; + const source = fs + .readFileSync(path.resolve(__dirname, '../delete-user.js'), 'utf8') + .replace(/^#!.*\n/, ''); + const execute = vm.runInThisContext( + `(function(require, __dirname, process, console, Date) { ${source}\n})`, + { filename: 'delete-user-recovery-cli.js' }, + ); + beforeAll(async () => { + mongo = await MongoMemoryServer.create(); + schemas.createModels(mongoose); + await mongoose.connect(mongo.getUri()); + admin = schemas.createPostgresDatabase(options); + await admin.query(`CREATE SCHEMA ${schema}`); + const url = new URL(options.connectionString); + url.searchParams.set('options', `-c search_path=${schema}`); + options.connectionString = url.toString(); + db = schemas.createPostgresDatabase(options); + await schemas.migratePostgres(db); + await schemas.importFavorites(db, async function* () {}, { + offline: true, + verifiedEmpty: true, + batchSize: 10, + }); + }); + afterEach(async () => { + await db.query('DROP TRIGGER IF EXISTS reject_cancel ON tool_favorite_erasure'); + await db.query('DROP FUNCTION IF EXISTS reject_cancel()'); + }); + afterAll(async () => { + await db.close(); + await admin.query(`DROP SCHEMA ${schema} CASCADE`); + await admin.close(); + await mongoose.disconnect(); + await mongo.stop(); + }); + async function runCli( + email, + now, + { failCascade = true, failAdopt = false, failCancel = false } = {}, + ) { + const runtime = api.createFavoriteRuntime( + schemas.createToolFavoriteMethods(mongoose), + { + PG_TEST_URL: options.connectionString, + }, + schemas.createPostgresFavoriteBackend, + ); + const methods = schemas.createMethods(mongoose, { toolFavorites: runtime.methods }); + const suspend = jest.spyOn(methods, 'suspendUserSchedulesForDeletion'); + if (failAdopt) + jest + .spyOn(methods, 'getAgentTriggerUserDeletionAttemptId') + .mockRejectedValueOnce(new Error('identity read unavailable')); + const disconnect = jest.spyOn(mongoose, 'disconnect').mockResolvedValue(undefined); + class Clock extends Date { + constructor(value) { + super(value === undefined ? now : value); + } + } + const quiet = Object.assign( + Object.create(console), + Object.fromEntries(['purple', 'green', 'yellow', 'error'].map((name) => [name, jest.fn()])), + ); + const code = await new Promise((resolve) => { + const modules = { + mongoose, + 'module-alias': () => undefined, + '@librechat/data-schemas': schemas, + '@librechat/api': { + ...api, + createStreamServices: () => ({ isRedis: false }), + waitForKeyvRedisClient: async () => undefined, + initializeFileStorage: () => undefined, + openCheckpointDeletion: async () => ({ + remember: async () => undefined, + cleanup: async () => undefined, + acknowledge: async () => undefined, + }), + deleteAllSharedLinksWithCleanup: async () => { + if (failCascade) throw new Error('shared links unavailable'); + }, + revokeUserCodeEnvironmentWorkers: async () => 0, + }, + '~/models': { + ...methods, + initializeFavorites: async (value) => { + await runtime.initialize(value.persistence); + if (failCancel) await installFault(); + }, + closeFavorites: async () => { + if (failCancel) await removeFault(); + await runtime.close(); + }, + }, + '~/config': {}, + '~/cache/getLogStores': () => undefined, + '~/server/services/Config': { getAppConfig: async () => ({}) }, + '~/server/services/Config/loadCustomConfig': async () => config, + '~/server/services/Files/process': { + processDeleteRequest: async () => ({ deletedFileIds: [], failedFileIds: [] }), + }, + '~/server/services/Config/getCachedTools': { invalidateCachedTools: async () => undefined }, + './helpers': { askQuestion: async () => 'y', silentExit: resolve }, + './connect': async () => undefined, + }; + execute( + (name) => (Object.hasOwn(modules, name) ? modules[name] : require(name)), + path.resolve(__dirname, '..'), + { argv: ['node', 'delete-user.js', email] }, + quiet, + Clock, + ); + }); + disconnect.mockRestore(); + return { code, suspend }; + } + async function owned(id) { + return mongoose.models.User.findById(id) + .select('+agentTriggerDeletionStartedAt +agentTriggerDeletionAttemptId') + .lean(); + } + async function installFault() { + await db.query( + `CREATE FUNCTION reject_cancel() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'cancel unavailable'; END $$`, + ); + await db.query( + 'CREATE TRIGGER reject_cancel BEFORE DELETE ON tool_favorite_erasure FOR EACH ROW EXECUTE FUNCTION reject_cancel()', + ); + } + async function removeFault() { + await db.query('DROP TRIGGER reject_cancel ON tool_favorite_erasure'); + await db.query('DROP FUNCTION reject_cancel()'); + } + it.each([ + { outcome: 'cancel', origin: 'system' }, + { outcome: 'commit', origin: 'system' }, + { outcome: 'cancel', origin: 'tenant' }, + { outcome: 'commit', origin: 'tenant' }, + ])( + 'adopts the canonical UUID from $origin through real offline CLI recovery and then $outcome', + async ({ outcome, origin }) => { + const id = new mongoose.Types.ObjectId(); + const email = `${id}@example.com`; + await mongoose.models.User.create({ + _id: id, + email, + provider: 'local', + ...(origin === 'tenant' ? { tenantId: 'tenant-a' } : {}), + }); + const time = Date.now(); + const stale = 15 * 60_000 + 1; + const priorToken = randomUUID(); + if (origin === 'tenant') { + const priorMethods = schemas.createMethods(mongoose); + await schemas.runAsSystem(() => + priorMethods.beginAgentTriggerUserDeletion( + id.toString(), + new Date(time - stale), + priorToken, + ), + ); + await schemas.tenantStorage.run({ tenantId: 'tenant-a' }, () => + schemas + .createPostgresFavoriteMethods(db) + .deleteUserToolFavorites(id.toString(), priorToken), + ); + } + const first = await runCli(email, time, { failCancel: true }); + expect(first.code).toBe(1); + const original = await owned(id); + const token = original.agentTriggerDeletionAttemptId; + expect(token).toMatch(/^[0-9a-f-]{36}$/); + if (origin === 'tenant') expect(token).toBe(priorToken); + // A takeover that loses its identity read must retain the adopted identity and owner. + if (outcome === 'cancel') { + expect((await runCli(email, time + stale, { failAdopt: true })).code).toBe(1); + expect((await owned(id)).agentTriggerDeletionAttemptId).toBe(token); + expect((await runCli(email, time + 2 * stale, { failCancel: true })).code).toBe(1); + expect((await owned(id)).agentTriggerDeletionAttemptId).toBe(token); + } + const result = await runCli(email, time + 3 * stale, { failCascade: outcome === 'cancel' }); + expect(result.code).toBe(outcome === 'cancel' ? 1 : 0); + expect(result.suspend.mock.calls[0][1]).not.toBe(first.suspend.mock.calls[0][1]); + expect(result.suspend.mock.calls[0][1]).not.toBe(token); + const methods = schemas.createMethods(mongoose); + expect( + await methods.cancelAgentTriggerUserDeletion( + id.toString(), + original.agentTriggerDeletionStartedAt, + token, + ), + ).toBe(false); + const restarted = api.createFavoriteRuntime( + methods, + { + PG_TEST_URL: options.connectionString, + }, + schemas.createPostgresFavoriteBackend, + ); + try { + await restarted.initialize(config.persistence); + const addFavorite = () => + restarted.methods.addToolFavorite({ + userId: id.toString(), + itemType: 'tool', + itemId: 'after-recovery', + }); + const add = + origin === 'tenant' + ? schemas.tenantStorage.run({ tenantId: 'tenant-a' }, addFavorite) + : addFavorite(); + if (outcome === 'cancel') { + expect((await owned(id)).agentTriggerDeletionStartedAt).toBeUndefined(); + expect((await owned(id)).agentTriggerDeletionAttemptId).toBeUndefined(); + await expect(add).resolves.toEqual({ ok: true, added: true }); + expect( + ( + await db.query( + 'SELECT scope FROM tool_favorite_erasure WHERE user_id=$1 AND attempt_id=$2', + [id.toString(), token], + ) + ).rows, + ).toEqual([]); + } else { + expect(await owned(id)).toBeNull(); + await expect(add).rejects.toThrow('erasure'); + } + } finally { + await restarted.close(); + } + }, + ); +}); diff --git a/config/__tests__/delete-user.spec.js b/config/__tests__/delete-user.spec.js index ec2b480c47..a045214be3 100644 --- a/config/__tests__/delete-user.spec.js +++ b/config/__tests__/delete-user.spec.js @@ -25,6 +25,7 @@ const mockMethods = { deleteUserById: jest.fn(), deleteAgentTriggerDeliveriesByUser: jest.fn(), cancelAgentTriggerUserPurge: jest.fn(), + isAgentTriggerUserDeletionOwner: jest.fn(), cancelAgentTriggerUserDeletion: jest.fn(), deleteUserPrompts: jest.fn(), }; @@ -34,6 +35,7 @@ const mockDestroy = jest.fn(); const mockSilentExit = jest.fn(); const mockAskQuestion = jest.fn(); const mockGetAppConfig = jest.fn(); +const mockLoadCustomConfig = jest.fn(); const mockRevokeUserCodeEnvironmentWorkers = jest.fn(); const mockEraseUserData = jest.fn(); const mockProcessDeleteRequest = jest.fn(); @@ -41,16 +43,25 @@ const mockInvalidateCachedTools = jest.fn(); const mockInitializeFileStorage = jest.fn(); const mockOpenCheckpointDeletion = jest.fn(); const mockDeleteAllSharedLinksWithCleanup = jest.fn(); +const mockMongoFavorites = { + getToolFavorites: jest.fn(), + addToolFavorite: jest.fn(), + removeToolFavorite: jest.fn(), + deleteUserToolFavorites: jest.fn(), + cancelUserToolFavoriteDeletion: jest.fn(), +}; jest.mock('../connect', () => jest.fn(async () => undefined)); jest.mock('mongoose', () => ({ disconnect: jest.fn(async () => undefined) })); jest.mock('@librechat/data-schemas', () => ({ createModels: () => new Proxy({}, { get: (_target, prop) => mockModelFor(prop) }), - createMethods: () => mockMethods, + createMethods: (_mongoose, deps) => ({ ...mockMethods, ...deps.toolFavorites }), + createToolFavoriteMethods: () => mockMongoFavorites, runAsSystem: (operation) => operation(), tenantStorage: mockTenantStorage, })); jest.mock('@librechat/api', () => ({ + ...jest.requireActual('@librechat/api'), waitForKeyvRedisClient: jest.fn(async () => undefined), createStreamServices: jest.fn(() => ({ isRedis: true })), GenerationJobManager: { @@ -67,6 +78,7 @@ jest.mock('@librechat/api', () => ({ deleteAllSharedLinksWithCleanup: mockDeleteAllSharedLinksWithCleanup, })); jest.mock('~/cache/getLogStores', () => jest.fn()); +jest.mock('~/server/services/Config/loadCustomConfig', () => mockLoadCustomConfig); jest.mock('~/server/services/Config', () => ({ getAppConfig: (...args) => mockGetAppConfig(...args), })); @@ -129,12 +141,16 @@ describe('Delete user CLI', () => { mockMethods.deleteUserById.mockReset().mockResolvedValue({ deletedCount: 1 }); mockMethods.deleteAgentTriggerDeliveriesByUser.mockReset().mockResolvedValue(undefined); mockMethods.cancelAgentTriggerUserPurge.mockReset().mockResolvedValue(true); + mockMethods.isAgentTriggerUserDeletionOwner.mockReset().mockResolvedValue(true); mockMethods.cancelAgentTriggerUserDeletion.mockReset().mockResolvedValue(true); mockMethods.deleteUserPrompts.mockReset().mockResolvedValue(undefined); mockGetCleanupBlockingJobIdsForUser.mockReset().mockResolvedValue([]); mockAbortJob.mockReset().mockResolvedValue({ success: true }); mockDestroy.mockReset().mockResolvedValue(undefined); mockGetAppConfig.mockReset().mockResolvedValue({}); + mockLoadCustomConfig.mockReset().mockResolvedValue({}); + mockMongoFavorites.deleteUserToolFavorites.mockReset(); + mockMongoFavorites.cancelUserToolFavoriteDeletion.mockReset().mockResolvedValue(undefined); mockRevokeUserCodeEnvironmentWorkers.mockReset().mockResolvedValue(0); mockEraseUserData.mockReset().mockResolvedValue(undefined); mockProcessDeleteRequest.mockReset().mockResolvedValue({ deletedFileIds: [] }); @@ -149,12 +165,38 @@ describe('Delete user CLI', () => { jest.restoreAllMocks(); }); + it('initializes the injected favorites methods and closes captured references on exit', async () => { + let deleteFavorites; + mockMongoFavorites.deleteUserToolFavorites.mockResolvedValueOnce(3); + mockEraseUserData.mockImplementationOnce(async (_params, deps) => { + deleteFavorites = deps.deleteUserToolFavorites; + expect(await deleteFavorites(USER_ID)).toBe(3); + }); + + expect(await runCli()).toBe(0); + + expect(mockLoadCustomConfig).toHaveBeenCalledWith(false, { storageSelection: true }); + expect(mockMongoFavorites.deleteUserToolFavorites).toHaveBeenCalledWith(USER_ID); + await expect(deleteFavorites(USER_ID)).rejects.toThrow('not initialized'); + }); + + it('deletes nothing when favorites persistence configuration is invalid', async () => { + mockLoadCustomConfig.mockResolvedValueOnce({ persistence: { favorites: 'invalid' } }); + + expect(await runCli()).toBe(1); + + expect(mockMethods.beginAgentTriggerUserDeletion).not.toHaveBeenCalled(); + expect(mockEraseUserData).not.toHaveBeenCalled(); + expect(mockMethods.deleteUserById).not.toHaveBeenCalled(); + }); + it('fences triggers and schedules before inspecting active work', async () => { expect(await runCli()).toBe(0); expect(mockMethods.beginAgentTriggerUserDeletion).toHaveBeenCalledWith( USER_ID, expect.any(Date), + expect.any(String), ); expect(mockMethods.prepareAgentTriggerUserPurge).toHaveBeenCalledWith( USER_ID, @@ -206,7 +248,11 @@ describe('Delete user CLI', () => { expect(mockMethods.deleteUserById).not.toHaveBeenCalled(); const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1]; expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence); - expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence); + expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( + USER_ID, + deletionFence, + expect.any(String), + ); }); /** @@ -229,6 +275,34 @@ describe('Delete user CLI', () => { ); }); + it.each([false, 'error'])( + 'keeps favorites fenced when Mongo ownership is not proven: %s', + async (result) => { + mockEraseUserData.mockRejectedValueOnce(new Error('shared link failure')); + if (result === 'error') + mockMethods.isAgentTriggerUserDeletionOwner.mockRejectedValueOnce( + new Error('Mongo unavailable'), + ); + else mockMethods.isAgentTriggerUserDeletionOwner.mockResolvedValueOnce(false); + expect(await runCli()).toBe(1); + expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled(); + }, + ); + + it('releases only the cascade UUID before releasing Mongo ownership', async () => { + mockEraseUserData.mockRejectedValueOnce(new Error('shared link failure')); + expect(await runCli()).toBe(1); + const token = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2]; + expect(mockEraseUserData.mock.calls[0][0].deletionAttemptId).toBe(token); + expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(USER_ID, token); + expect(mockMethods.isAgentTriggerUserDeletionOwner.mock.invocationCallOrder[0]).toBeLessThan( + mockMongoFavorites.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0], + ); + expect( + mockMongoFavorites.cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0], + ).toBeLessThan(mockMethods.cancelAgentTriggerUserDeletion.mock.invocationCallOrder[0]); + }); + it('never restores schedules after a committed deletion', async () => { expect(await runCli()).toBe(0); @@ -240,6 +314,13 @@ describe('Delete user CLI', () => { expect(await runCli()).toBe(0); expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true }); + expect(mockMethods.deleteUserById).toHaveBeenCalledWith(USER_ID, { + startedAt: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1], + attemptId: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2], + }); + expect(mockMethods.deleteUserById.mock.calls[0][1].attemptId).not.toBe( + mockMethods.suspendUserSchedulesForDeletion.mock.calls[0][1], + ); expect(mockRevokeUserCodeEnvironmentWorkers).toHaveBeenCalledWith( expect.objectContaining({ userId: USER_ID }), ); @@ -279,6 +360,7 @@ describe('Delete user CLI', () => { tenantId: 'tenant-1', checkpointer: { ttl: 60 }, keepTransactions: true, + deletionAttemptId: mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][2], }); expect(deps).toEqual( expect.objectContaining({ @@ -337,6 +419,8 @@ describe('Delete user CLI', () => { }); it('supplies shared-user MCP OAuth cleanup to the cascade', async () => { + mockMethods.findMCPServerByObjectId = jest.fn(async () => ({ config: { type: 'sse' } })); + mockMethods.getUserById = jest.fn(async () => ({ role: 'VIEWER' })); expect(await runCli()).toBe(0); const [, deps] = mockEraseUserData.mock.calls[0]; @@ -347,10 +431,8 @@ describe('Delete user CLI', () => { invalidateUserTools: expect.any(Function), uninstallOAuthMCP: expect.any(Function), }); - mockMethods.findMCPServerByObjectId = jest.fn(async () => ({ config: { type: 'sse' } })); expect(await deps.mcpOAuth.getServerConfig('server-1', 'server-id-1')).toEqual({ type: 'sse' }); expect(mockMethods.findMCPServerByObjectId).toHaveBeenCalledWith('server-id-1'); - mockMethods.getUserById = jest.fn(async () => ({ role: 'VIEWER' })); mockGetAppConfig.mockImplementation(async (options) => options?.userId === 'viewer-1' ? { mcpSettings: { allowedDomains: ['viewer.example'] } } : {}, ); @@ -380,6 +462,19 @@ describe('Delete user CLI', () => { ); }); + it.each(['unknown', 'absent'])( + 'keeps both erasure fences after final delete starts: %s', + async (outcome) => { + if (outcome === 'unknown') + mockMethods.deleteUserById.mockRejectedValueOnce(new Error('delete acknowledgement lost')); + else mockMethods.deleteUserById.mockResolvedValueOnce({ deletedCount: 0 }); + expect(await runCli()).toBe(1); + expect(mockMethods.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled(); + expect(mockMongoFavorites.cancelUserToolFavoriteDeletion).not.toHaveBeenCalled(); + expect(mockMethods.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); + }, + ); + it('keeps the user and releases both fences when the cascade fails', async () => { mockEraseUserData.mockRejectedValueOnce(new Error('storage unavailable')); @@ -388,7 +483,11 @@ describe('Delete user CLI', () => { expect(mockMethods.deleteUserById).not.toHaveBeenCalled(); const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1]; expect(mockMethods.restoreUserSchedulesFromDeletion).toHaveBeenCalled(); - expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence); + expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( + USER_ID, + deletionFence, + expect.any(String), + ); }); it('passes deleteUserPrompts through to the shared cascade, which keeps a synced group with a synthetic author untouched', async () => { @@ -409,6 +508,10 @@ describe('Delete user CLI', () => { expect(mockMethods.deleteUserById).not.toHaveBeenCalled(); const deletionFence = mockMethods.beginAgentTriggerUserDeletion.mock.calls[0][1]; expect(mockMethods.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(USER_ID, deletionFence); - expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(USER_ID, deletionFence); + expect(mockMethods.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( + USER_ID, + deletionFence, + expect.any(String), + ); }); }); diff --git a/config/__tests__/migrate-shared-link-permissions.spec.js b/config/__tests__/migrate-shared-link-permissions.spec.js index b944e479ea..f007d770c1 100644 --- a/config/__tests__/migrate-shared-link-permissions.spec.js +++ b/config/__tests__/migrate-shared-link-permissions.spec.js @@ -1,5 +1,6 @@ jest.mock('../connect', () => jest.fn().mockResolvedValue(true)); jest.mock('@librechat/api', () => ({ + ...jest.requireActual('@librechat/api'), ensureRequiredCollectionsExist: jest.fn().mockResolvedValue(undefined), createChannelStorageConfigReader: jest.fn(() => ({ initialize: jest.fn(), diff --git a/config/delete-user.js b/config/delete-user.js index d56e9262d3..193a96a4c4 100644 --- a/config/delete-user.js +++ b/config/delete-user.js @@ -4,16 +4,12 @@ const path = require('path'); const mongoose = require('mongoose'); const { randomUUID } = require('node:crypto'); -const { - createModels, - createMethods, - runAsSystem, - tenantStorage, -} = require('@librechat/data-schemas'); +const { createModels, runAsSystem, tenantStorage } = require('@librechat/data-schemas'); const { User } = createModels(mongoose); require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); const { eraseUserData, + createUserDeletionAttempt, GenerationJobManager, createStreamServices, initializeFileStorage, @@ -36,13 +32,16 @@ const connect = require('./connect'); const TRIGGER_DRAIN_TIMEOUT_MS = 35_000; const TRIGGER_DRAIN_POLL_MS = 100; -const methods = createMethods(mongoose, { getCache: getLogStores }); +const { initializeFavorites, closeFavorites, ...methods } = require('~/models'); const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); async function gracefulExit(code = 0) { try { - await mongoose.disconnect(); + const cleanup = await Promise.allSettled([closeFavorites(), mongoose.disconnect()]); + if (cleanup.some((result) => result.status === 'rejected')) { + throw new Error('Persistence cleanup failed'); + } } catch (err) { console.error('Error disconnecting from MongoDB:', err); } @@ -51,6 +50,9 @@ async function gracefulExit(code = 0) { (async () => { await connect(); + await initializeFavorites( + await require('~/server/services/Config/loadCustomConfig')(false, { storageSelection: true }), + ); console.purple('---------------'); console.purple('Deleting a user and all related data'); @@ -115,11 +117,12 @@ async function gracefulExit(code = 0) { let deletionFence; let scheduleSuspensionToken; let userDeleted = false; + const deletionAttempt = createUserDeletionAttempt(); try { deletionFence = new Date(); let fenceState = await runAsSystem(() => - methods.beginAgentTriggerUserDeletion(uid, deletionFence), + methods.beginAgentTriggerUserDeletion(uid, deletionFence, deletionAttempt.id), ); if (fenceState === 'in_progress') { deletionFence = undefined; @@ -135,7 +138,7 @@ async function gracefulExit(code = 0) { const recoveredAt = new Date(); fenceState = await runAsSystem(() => - methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt), + methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt, deletionAttempt.id), ); if (fenceState !== 'acquired') { throw new Error( @@ -145,6 +148,7 @@ async function gracefulExit(code = 0) { ); } deletionFence = recoveredAt; + await runAsSystem(() => deletionAttempt.adopt(uid, recoveredAt, methods)); } if (fenceState === 'missing') { deletionFence = undefined; @@ -214,6 +218,7 @@ async function gracefulExit(code = 0) { eraseUserData( { userId: uid, + deletionAttemptId: deletionAttempt.id, tenantId: user.tenantId, checkpointer: userAppConfig?.endpoints?.agents?.checkpointer, keepTransactions: !deleteTx, @@ -283,7 +288,14 @@ async function gracefulExit(code = 0) { ); // 6) Finally delete the user document itself - const deletedUser = await runAsSystem(() => methods.deleteUserById(uid)); + const deletedUser = await deletionAttempt.commit(() => + runAsSystem(() => + methods.deleteUserById(uid, { + startedAt: deletionFence, + attemptId: deletionAttempt.id, + }), + ), + ); if (deletedUser.deletedCount !== 1) { throw new Error('User disappeared before account deletion could commit'); } @@ -326,9 +338,12 @@ async function gracefulExit(code = 0) { await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch( (error) => console.error('Failed to disarm trigger purge recovery:', error), ); - await runAsSystem(() => methods.cancelAgentTriggerUserDeletion(uid, deletionFence)).catch( - (error) => console.error('Failed to release account-deletion fence:', error), - ); + await runAsSystem(() => + deletionAttempt.cancel( + { userId: uid, startedAt: deletionFence, deletionAttemptId: deletionAttempt.id }, + methods, + ), + ).catch((error) => console.error('Failed to release account-deletion fence:', error)); } if (hasSharedGenerationStore) { await GenerationJobManager.destroy().catch((error) => @@ -343,11 +358,7 @@ async function gracefulExit(code = 0) { } return gracefulExit(0); -})().catch(async (err) => { - if (!err.message.includes('fetch failed')) { - console.error('There was an uncaught error:'); - console.error(err); - await mongoose.disconnect(); - process.exit(1); - } +})().catch(async () => { + console.error('Account deletion failed; retry before considering the account erased.'); + await gracefulExit(1); }); diff --git a/config/postgres.js b/config/postgres.js new file mode 100644 index 0000000000..274b942a02 --- /dev/null +++ b/config/postgres.js @@ -0,0 +1,21 @@ +#!/usr/bin/env node +require('dotenv').config(); +require('module-alias/register'); +const path = require('node:path'); +require('module-alias').addAlias('~', path.resolve(__dirname, '../api')); +const { runPostgresCommand } = require('@librechat/api'); +const mongoose = require('mongoose'); +const { createMongoFavoriteStore, createPostgresDatabase } = require('@librechat/data-schemas'); +const loadCustomConfig = require('~/server/services/Config/loadCustomConfig'); +runPostgresCommand({ + favorites: createMongoFavoriteStore(mongoose), + createDatabase: createPostgresDatabase, + connectDb: () => mongoose.connect(process.env.MONGO_URI, { autoCreate: false, autoIndex: false }), + disconnectDb: () => mongoose.disconnect(), + loadCustomConfig, + args: process.argv.slice(2), + env: process.env, +}).catch(() => { + console.error('PostgreSQL operation failed. No automatic storage cutover was performed.'); + process.exitCode = 1; +}); diff --git a/package-lock.json b/package-lock.json index 53f98543d5..656c0a7c2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19958,6 +19958,18 @@ "@types/passport": "*" } }, + "node_modules/@types/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -35414,6 +35426,95 @@ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -35639,6 +35740,45 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/powershell-utils": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.2.1.tgz", @@ -38744,6 +38884,15 @@ "node": ">=8.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -41977,7 +42126,6 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4" @@ -45298,13 +45446,15 @@ "micromark-extension-directive": "^3.0.1", "micromark-extension-gfm": "^3.0.0", "micromark-extension-llm-math": "^3.1.0", - "micromark-util-decode-string": "^2.0.0" + "micromark-util-decode-string": "^2.0.0", + "pg": "^8.23.1" }, "devDependencies": { "@stryker-mutator/core": "10.1.1", "@stryker-mutator/jest-runner": "10.1.1", "@types/jest": "^29.5.2", "@types/node": "^24.12.4", + "@types/pg": "^8.23.1", "jest": "^30.2.0", "jest-junit": "^17.0.0", "mongodb-memory-server": "^11.3.0", diff --git a/package.json b/package.json index 268c6d5ba3..94e202f190 100644 --- a/package.json +++ b/package.json @@ -166,7 +166,8 @@ "migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50", "lighthouse": "npm run e2e:prepare && npm run lighthouse:run", "lighthouse:run": "playwright test --config=e2e/playwright.config.lighthouse.ts", - "lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run" + "lighthouse:regression": "cross-env LIGHTHOUSE_REGRESSION=serial-reads npm run lighthouse:run", + "postgres": "node config/postgres.js" }, "repository": { "type": "git", diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts index d8add91c08..c98645c217 100644 --- a/packages/api/src/admin/users.spec.ts +++ b/packages/api/src/admin/users.spec.ts @@ -9,6 +9,7 @@ import { createAdminUsersHandlers } from './users'; import { eraseUserData } from '~/user/erasure'; jest.mock('~/user/erasure', () => ({ + ...jest.requireActual('~/user/erasure'), eraseUserData: jest.fn().mockResolvedValue(undefined), })); @@ -60,6 +61,7 @@ function createDeps(overrides: Partial = {}): AdminUsersDeps { findUsers: jest.fn().mockResolvedValue([]), countUsers: jest.fn().mockResolvedValue(0), beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'), + isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(true), cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true), drainAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined), prepareAgentTriggerUserPurge: jest.fn().mockResolvedValue(undefined), @@ -392,6 +394,7 @@ describe('createAdminUsersHandlers', () => { expect(deps.beginAgentTriggerUserDeletion).toHaveBeenCalledWith( validUserId, expect.any(Date), + expect.any(String), ); expect(deps.drainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId); expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalledWith( @@ -461,7 +464,10 @@ describe('createAdminUsersHandlers', () => { await handlers.deleteUser(req, res); expect(status).toHaveBeenCalledWith(200); - expect(deps.deleteUserById).toHaveBeenCalledWith(targetId); + expect(deps.deleteUserById).toHaveBeenCalledWith(targetId, { + startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1], + attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2], + }); }); it('does not check admin count when target is a regular user', async () => { @@ -559,7 +565,12 @@ describe('createAdminUsersHandlers', () => { }), ); expect(eraseUserData).toHaveBeenCalledWith( - { userId: validUserId, tenantId: 'tenant-1', checkpointer }, + { + userId: validUserId, + tenantId: 'tenant-1', + checkpointer, + deletionAttemptId: expect.any(String), + }, erasureDeps, ); expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan( @@ -582,6 +593,34 @@ describe('createAdminUsersHandlers', () => { expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled(); }); + it.each([true, false])( + 'releases the resolved favorite UUID only after a proven Mongo owner: %s', + async (owned) => { + const cancelUserToolFavoriteDeletion = jest.fn().mockResolvedValue(undefined); + jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('later cascade failure')); + const deps = createDeps({ + isAgentTriggerUserDeletionOwner: jest.fn().mockResolvedValue(owned), + resolveErasure: jest.fn().mockResolvedValue({ deps: { cancelUserToolFavoriteDeletion } }), + }); + const { req, res } = createReqRes({ params: { id: validUserId } }); + await createAdminUsersHandlers(deps).deleteUser(req, res); + const token = jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2]; + expect(eraseUserData).toHaveBeenCalledWith( + expect.objectContaining({ deletionAttemptId: token }), + expect.anything(), + ); + if (owned) { + expect(cancelUserToolFavoriteDeletion).toHaveBeenCalledWith(validUserId, token); + expect( + jest.mocked(deps.isAgentTriggerUserDeletionOwner).mock.invocationCallOrder[0], + ).toBeLessThan(cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]); + expect(cancelUserToolFavoriteDeletion.mock.invocationCallOrder[0]).toBeLessThan( + jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0], + ); + } else expect(cancelUserToolFavoriteDeletion).not.toHaveBeenCalled(); + }, + ); + it('suspends schedules before erasing and restores them when the deletion fails', async () => { jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down')); const deps = createDeps(); @@ -677,7 +716,10 @@ describe('createAdminUsersHandlers', () => { await handlers.deleteUser(req, res); - expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId); + expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId, { + startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1], + attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2], + }); expect(deps.deleteUserCodeEnvironments).not.toHaveBeenCalled(); }); @@ -694,10 +736,7 @@ describe('createAdminUsersHandlers', () => { expect(deps.deleteUserCodeEnvironments).not.toHaveBeenCalled(); expect(deps.deleteAclEntries).not.toHaveBeenCalled(); expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled(); - expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( - validUserId, - expect.any(Date), - ); + expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date)); }); @@ -735,10 +774,8 @@ describe('createAdminUsersHandlers', () => { expect(status).toHaveBeenCalledWith(500); expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' }); - expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith( - validUserId, - expect.any(Date), - ); + expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); + expect(deps.isAgentTriggerUserDeletionOwner).not.toHaveBeenCalled(); expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date)); expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled(); }); @@ -815,7 +852,10 @@ describe('createAdminUsersHandlers', () => { expect(status).toHaveBeenCalledWith(500); expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' }); - expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId); + expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId, { + startedAt: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][1], + attemptId: jest.mocked(deps.beginAgentTriggerUserDeletion).mock.calls[0][2], + }); expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalled(); expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled(); expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled(); diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts index 2e3ce18b29..5a514edfb7 100644 --- a/packages/api/src/admin/users.ts +++ b/packages/api/src/admin/users.ts @@ -7,7 +7,7 @@ import type { ConfigRecord, AdminUserListItem, AdminUserSearchResult, - UserDeleteResult, + AllMethods, } from '@librechat/data-schemas'; import type { TCheckpointerConfig } from 'librechat-data-provider'; import type { FilterQuery } from 'mongoose'; @@ -15,10 +15,10 @@ import type { Response } from 'express'; import type { UserGenerationControl } from '~/user/generations'; import type { UserErasureDeps } from '~/user/erasure'; import type { ServerRequest } from '~/types/http'; +import { eraseUserData, createUserDeletionAttempt } from '~/user/erasure'; import { stopUserGenerations } from '~/user/generations'; import { invalidatingConfigStore } from './cascade'; import { parsePagination } from './pagination'; -import { eraseUserData } from '~/user/erasure'; const MAX_SEARCH_LENGTH = 200; @@ -75,8 +75,18 @@ export interface AdminUsersDeps { beginAgentTriggerUserDeletion: ( userId: string, startedAt: Date, + attemptId?: string, ) => Promise<'acquired' | 'in_progress' | 'missing'>; - cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise; + isAgentTriggerUserDeletionOwner: ( + userId: string, + startedAt: Date, + attemptId?: string, + ) => Promise; + cancelAgentTriggerUserDeletion: ( + userId: string, + startedAt: Date, + attemptId?: string, + ) => Promise; drainAgentTriggerDeliveriesForUser: (userId: string) => Promise; prepareAgentTriggerUserPurge: ( userId: string, @@ -102,7 +112,7 @@ export interface AdminUsersDeps { */ resolveErasure: (target: AdminErasureTarget) => Promise; /** Thin data-layer delete: removes the User document after the erasure cascade has run. */ - deleteUserById: (userId: string) => Promise; + deleteUserById: AllMethods['deleteUserById']; deleteUserCodeEnvironments: (userId: string | Types.ObjectId) => Promise; invalidateCodeEnvironmentConfigCache: (tenantId?: string) => Promise; deleteConfig: ( @@ -132,6 +142,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { findUsers, countUsers, beginAgentTriggerUserDeletion, + isAgentTriggerUserDeletionOwner, cancelAgentTriggerUserDeletion, drainAgentTriggerDeliveriesForUser, prepareAgentTriggerUserPurge, @@ -232,6 +243,8 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { let triggerDeletionFence: Date | undefined; let scheduleSuspensionToken: string | undefined; let userDeleted = false; + const deletionAttempt = createUserDeletionAttempt(); + let erasureDeps: UserErasureDeps | undefined; try { const { id } = req.params as { id: string }; @@ -255,7 +268,11 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { } triggerDeletionFence = new Date(); - const fenceState = await beginAgentTriggerUserDeletion(id, triggerDeletionFence); + const fenceState = await beginAgentTriggerUserDeletion( + id, + triggerDeletionFence, + deletionAttempt.id, + ); if (fenceState === 'in_progress') { triggerDeletionFence = undefined; return res.status(409).json({ error: 'User deletion is already in progress' }); @@ -274,22 +291,40 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { } await stopUserGenerations(id, targetUser?.tenantId, generationJobs); - const { deps: erasureDeps, checkpointer } = await resolveErasure({ + const erasure = await resolveErasure({ id, role: targetUser?.role, tenantId: targetUser?.tenantId, idOnTheSource: targetUser?.idOnTheSource ?? null, profile: targetUser, }); + erasureDeps = erasure.deps; await eraseUserData( - { userId: id, tenantId: targetUser?.tenantId, checkpointer }, + { + userId: id, + tenantId: targetUser?.tenantId, + checkpointer: erasure.checkpointer, + deletionAttemptId: deletionAttempt.id, + }, erasureDeps, ); - const result = await deleteUserById(id); + const deletionFence = { startedAt: triggerDeletionFence, attemptId: deletionAttempt.id }; + const result = await deletionAttempt.commit(() => deleteUserById(id, deletionFence)); if (result.deletedCount === 0) { await cancelAgentTriggerUserPurge(id, triggerDeletionFence); - await cancelAgentTriggerUserDeletion(id, triggerDeletionFence); + await deletionAttempt.cancel( + { + userId: id, + startedAt: triggerDeletionFence, + deletionAttemptId: deletionAttempt.id, + }, + { + isAgentTriggerUserDeletionOwner, + cancelAgentTriggerUserDeletion, + cancelUserToolFavoriteDeletion: erasureDeps?.cancelUserToolFavoriteDeletion, + }, + ); triggerDeletionFence = undefined; return res.status(404).json({ error: 'User not found' }); } @@ -353,7 +388,18 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): { logger.error('[adminUsers] failed to disarm trigger purge recovery:', purgeFenceError); } try { - await cancelAgentTriggerUserDeletion(targetUserId, triggerDeletionFence); + await deletionAttempt.cancel( + { + userId: targetUserId, + startedAt: triggerDeletionFence, + deletionAttemptId: deletionAttempt.id, + }, + { + isAgentTriggerUserDeletionOwner, + cancelAgentTriggerUserDeletion, + cancelUserToolFavoriteDeletion: erasureDeps?.cancelUserToolFavoriteDeletion, + }, + ); } catch (fenceError) { logger.error('[adminUsers] failed to release trigger deletion fence:', fenceError); } diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.ts index 5db9cab775..626f27e7df 100644 --- a/packages/api/src/app/index.ts +++ b/packages/api/src/app/index.ts @@ -22,3 +22,7 @@ export { resolveBuildInfo } from './build'; export type { BuildInfo } from './build'; export { buildPreLoginInterface, withSafePolicyLinks } from './interface'; export * from './migrations'; + +export * from './persistence'; + +export * from './postgres'; diff --git a/packages/api/src/app/loader.ts b/packages/api/src/app/loader.ts index 63b2c888ee..1d384e0386 100644 --- a/packages/api/src/app/loader.ts +++ b/packages/api/src/app/loader.ts @@ -4,6 +4,7 @@ import { createHash } from 'node:crypto'; import { logger } from '@librechat/data-schemas'; import { configSchema, + persistenceConfigSchema, EModelEndpoint, EImageOutputType, getMaxSubagents, @@ -21,6 +22,8 @@ export type CustomConfigLoadMode = 'startup' | 'reload'; export interface CustomConfigLoadOptions { mode?: CustomConfigLoadMode; + /** Storage initialization must distinguish an absent default from an unreadable source. */ + storageSelection?: boolean; } export interface LoadedCustomConfig { @@ -192,6 +195,17 @@ export function createCustomConfigLoader({ cause?: unknown, startupLogLevel: 'error' | 'info' = 'error', ): null => { + if ( + options.storageSelection && + !( + cause instanceof Error && + 'code' in cause && + cause.code === 'ENOENT' && + !process.env.CONFIG_PATH + ) + ) { + throw new ConfigReloadError('Deployment config unavailable for storage selection'); + } if (mode === 'reload') { if (cause == null) { logger.error(message); @@ -261,6 +275,16 @@ export function createCustomConfigLoader({ } /** The checks below return copies, but `modelSpecs` is assigned onto this object later. */ + if ( + typeof loadedConfig === 'object' && + loadedConfig != null && + 'persistence' in loadedConfig + ) { + const persistence = persistenceConfigSchema.safeParse(loadedConfig.persistence); + if (!persistence.success) { + throw new ConfigReloadError('Invalid persistence configuration'); + } + } const parsedConfig = loadedConfig; const themeCheck = checkConfigTheme(loadedConfig); loadedConfig = themeCheck.config; @@ -302,6 +326,13 @@ export function createCustomConfigLoader({ ); } + if ( + typeof loadedConfig === 'object' && + loadedConfig != null && + 'persistence' in loadedConfig + ) { + throw new ConfigReloadError('Invalid configuration containing persistence selection'); + } if (process.env.CONFIG_BYPASS_VALIDATION === 'true') { logger.warn( 'CONFIG_BYPASS_VALIDATION is enabled. Continuing with default configuration despite validation errors.', diff --git a/packages/api/src/app/persistence.spec.ts b/packages/api/src/app/persistence.spec.ts new file mode 100644 index 0000000000..9b146626ec --- /dev/null +++ b/packages/api/src/app/persistence.spec.ts @@ -0,0 +1,319 @@ +import pg from 'pg'; +import { randomBytes } from 'node:crypto'; +import { + createPostgresFavoriteBackend, + createPostgresDatabase, + migratePostgres, + importFavorites, +} from '@librechat/data-schemas'; +import type { ToolFavoriteMethods } from '@librechat/data-schemas'; +import type { Response } from 'express'; +import type { ServerRequest } from '~/types'; +import { createFavoriteRuntime, getPersistenceConfig } from './persistence'; +import { createToolFavoritesHandlers } from '~/favorites/handlers'; +import { createCustomConfigLoader } from './loader'; + +const mongo: ToolFavoriteMethods = { + getToolFavorites: jest.fn().mockResolvedValue([]), + addToolFavorite: jest.fn().mockResolvedValue({ ok: true, added: true }), + removeToolFavorite: jest.fn().mockResolvedValue({ ok: true, removed: true }), + deleteUserToolFavorites: jest.fn().mockResolvedValue(2), +}; + +describe('favorites process lifecycle', () => { + it('retains a captured method across initialization and erases through the selected domain', async () => { + const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend); + const captured = runtime.methods.deleteUserToolFavorites; + await expect(captured('user')).rejects.toThrow('not initialized'); + await runtime.initialize(undefined); + expect(await captured('user')).toBe(2); + await runtime.close(); + await expect(captured('user')).rejects.toThrow('not initialized'); + await expect(runtime.initialize(undefined)).rejects.toThrow('closed'); + }); + it('keeps the optional Mongo cancellation hook a no-op without changing captured references', async () => { + const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend); + const cancel = runtime.methods.cancelUserToolFavoriteDeletion!; + await expect(cancel('user', 'attempt')).rejects.toThrow('not initialized'); + await runtime.initialize(undefined); + await expect(cancel('user', 'attempt')).resolves.toBeUndefined(); + await runtime.close(); + await expect(cancel('user', 'attempt')).rejects.toThrow('not initialized'); + }); + it('fails closed on invalid explicit storage config and missing DSN', async () => { + await expect( + createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend).initialize({ + favorites: 'invalid', + }), + ).rejects.toThrow('Invalid'); + await expect( + createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend).initialize({ + favorites: 'postgres', + postgres: { connectionStringEnv: 'PG_DSN' }, + }), + ).rejects.toThrow('missing'); + }); + it('selects once for a process lifetime, never a request or config reload', async () => { + const runtime = createFavoriteRuntime(mongo, {}, createPostgresFavoriteBackend); + await runtime.initialize(undefined); + await runtime.initialize({ favorites: 'postgres' }); + expect(await runtime.methods.getToolFavorites('user')).toEqual([]); + await runtime.close(); + }); + it('constructs only the selected injected backend and retains captured method references', async () => { + const backend = { + methods: { + ...mongo, + getToolFavorites: jest.fn(async () => [{ itemType: 'tool' as const, itemId: 'injected' }]), + }, + initialize: jest.fn(async () => undefined), + close: jest.fn(async () => undefined), + }; + const factory = jest.fn(() => backend); + const runtime = createFavoriteRuntime(mongo, { INJECTED_PG: 'owned-connection' }, factory); + const captured = runtime.methods.getToolFavorites; + expect(factory).not.toHaveBeenCalled(); + await expect(captured('user')).rejects.toThrow('not initialized'); + await runtime.initialize({ + favorites: 'postgres', + postgres: { connectionStringEnv: 'INJECTED_PG' }, + }); + await runtime.initialize(undefined); + expect(factory).toHaveBeenCalledTimes(1); + expect(factory).toHaveBeenCalledWith( + expect.objectContaining({ + connectionString: 'owned-connection', + max: 10, + queryTimeoutMs: 15000, + }), + ); + expect(backend.initialize).toHaveBeenCalledTimes(1); + expect(runtime.methods.getToolFavorites).toBe(captured); + expect(await captured('user')).toEqual([{ itemType: 'tool', itemId: 'injected' }]); + await runtime.close(); + await runtime.close(); + expect(backend.close).toHaveBeenCalledTimes(1); + await expect(captured('user')).rejects.toThrow('not initialized'); + const unused = jest.fn(() => backend); + const defaultRuntime = createFavoriteRuntime(mongo, {}, unused); + await defaultRuntime.initialize(undefined); + expect(unused).not.toHaveBeenCalled(); + await defaultRuntime.close(); + }); + it.each(['construct', 'initialize'] as const)( + 'sanitizes an injected %s failure without publishing methods or falling back', + async (stage) => { + const privateError = new Error('private backend detail'); + const backend = { + methods: mongo, + initialize: jest.fn().mockRejectedValue(privateError), + close: jest.fn(async () => undefined), + }; + const factory = jest.fn(() => { + if (stage === 'construct') throw privateError; + return backend; + }); + const runtime = createFavoriteRuntime(mongo, { INJECTED_PG: 'owned-connection' }, factory); + await expect( + runtime.initialize({ + favorites: 'postgres', + postgres: { connectionStringEnv: 'INJECTED_PG' }, + }), + ).rejects.toThrow('PostgreSQL favorites initialization failed'); + await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized'); + await runtime.close(); + expect(factory).toHaveBeenCalledTimes(1); + expect(backend.close).toHaveBeenCalledTimes(stage === 'initialize' ? 1 : 0); + }, + ); + it('waits for the injected initializer during shutdown and never publishes its methods', async () => { + let ready!: () => void; + const gate = new Promise((resolve) => { + ready = resolve; + }); + const backend = { + methods: mongo, + initialize: jest.fn(() => gate), + close: jest.fn(async () => undefined), + }; + const runtime = createFavoriteRuntime( + mongo, + { INJECTED_PG: 'owned-connection' }, + () => backend, + ); + const initialization = runtime.initialize({ + favorites: 'postgres', + postgres: { connectionStringEnv: 'INJECTED_PG' }, + }); + const result = initialization.catch((error: Error) => error); + const closing = runtime.close(); + expect(backend.close).not.toHaveBeenCalled(); + ready(); + expect(await result).toEqual(new Error('PostgreSQL favorites initialization failed')); + await closing; + expect(backend.close).toHaveBeenCalledTimes(1); + await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized'); + }); + it('extracts only deployment persistence and rejects malformed deployment data', () => { + expect(getPersistenceConfig(null)).toBeUndefined(); + expect(getPersistenceConfig({ persistence: { favorites: 'mongo' } })).toEqual({ + favorites: 'mongo', + }); + expect(() => getPersistenceConfig('invalid')).toThrow(); + }); +}); + +describe('storage selection source handling', () => { + const create = (loadLocal: () => unknown) => + createCustomConfigLoader({ + defaultConfigPath: 'librechat.yaml', + loadLocal, + redactConfig: (config) => config, + }); + it('permits an absent default file, but never malformed YAML or an explicit missing source', async () => { + const original = process.env.CONFIG_PATH; + delete process.env.CONFIG_PATH; + const missing = Object.assign(new Error('missing'), { code: 'ENOENT' }); + try { + await expect(create(() => missing)(false, { storageSelection: true })).resolves.toBeNull(); + await expect( + create(() => new Error('malformed'))(false, { storageSelection: true }), + ).rejects.toThrow('unavailable'); + await expect( + create(() => Object.assign(new Error('unreadable'), { code: 'EACCES' }))(false, { + storageSelection: true, + }), + ).rejects.toThrow('unavailable'); + process.env.CONFIG_PATH = 'explicit.yaml'; + await expect(create(() => missing)(false, { storageSelection: true })).rejects.toThrow( + 'unavailable', + ); + } finally { + if (original == null) { + delete process.env.CONFIG_PATH; + } else { + process.env.CONFIG_PATH = original; + } + } + }); + it('refuses invalid persistence even with CONFIG_BYPASS_VALIDATION', async () => { + const original = process.env.CONFIG_BYPASS_VALIDATION; + process.env.CONFIG_BYPASS_VALIDATION = 'true'; + try { + await expect( + create(() => ({ version: '1.3.1', persistence: { favorites: 'postgress' } }))(false), + ).rejects.toThrow('Invalid persistence'); + } finally { + if (original == null) { + delete process.env.CONFIG_BYPASS_VALIDATION; + } else { + process.env.CONFIG_BYPASS_VALIDATION = original; + } + } + }); +}); + +const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip; +realPostgres('selected PostgreSQL lifecycle and HTTP boundary', () => { + const env = { PG_TEST_URL: process.env.PG_TEST_URL }; + const config = { + favorites: 'postgres', + postgres: { connectionStringEnv: 'PG_TEST_URL', acquireTimeoutMs: 1000, queryTimeoutMs: 5000 }, + }; + const options = { + connectionString: process.env.PG_TEST_URL ?? '', + max: 2, + maxWaiting: 10, + acquireTimeoutMs: 1000, + queryTimeoutMs: 5000, + idleTimeoutMs: 1000, + }; + const schema = `favorites_lifecycle_${randomBytes(6).toString('hex')}`; + const admin = createPostgresDatabase(options); + beforeAll(async () => { + await admin.query(`CREATE SCHEMA ${schema}`); + const url = new URL(options.connectionString); + url.searchParams.set('options', `-c search_path=${schema}`); + options.connectionString = url.toString(); + env.PG_TEST_URL = options.connectionString; + }); + afterAll(async () => { + await admin.query(`DROP SCHEMA ${schema} CASCADE`); + await admin.close(); + }); + it('refuses schema-only data, releases failed pools, and serves captured HTTP handlers after verification', async () => { + const db = createPostgresDatabase(options); + try { + await migratePostgres(db); + await db.query('TRUNCATE tool_favorites, tool_favorite_erasure'); + await db.query( + 'UPDATE tool_favorite_readiness SET verified=false, activated=false, checkpoint=NULL, snapshot=NULL', + ); + const end = jest.spyOn(pg.Pool.prototype, 'end'); + const failed = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend); + await expect(failed.initialize(config)).rejects.toThrow('initialization failed'); + expect(end).toHaveBeenCalled(); + await failed.close(); + await importFavorites(db, async function* () {}, { + offline: true, + batchSize: 10, + verifiedEmpty: true, + }); + const runtime = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend); + const handlers = createToolFavoritesHandlers(runtime.methods); + await runtime.initialize(config); + const uid = '0123456789abcdef01234567'; + const req = { + user: { id: uid }, + params: { itemType: 'tool', itemId: 'native' }, + } as ServerRequest; + const res = { + status: jest.fn().mockReturnThis(), + json: jest.fn().mockReturnThis(), + } as unknown as Response; + try { + await handlers.addToolFavorite(req, res); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ itemType: 'tool', itemId: 'native' }); + expect(await runtime.methods.deleteUserToolFavorites(uid)).toBe(1); + expect(await runtime.methods.getToolFavorites(uid)).toEqual([]); + await db.query('ALTER TABLE tool_favorites RENAME TO unavailable_favorites'); + await handlers.listToolFavorites(req, res); + expect(res.status).toHaveBeenLastCalledWith(500); + expect(res.json).toHaveBeenLastCalledWith({ message: 'Failed to retrieve favorites' }); + await db.query('ALTER TABLE unavailable_favorites RENAME TO tool_favorites'); + await expect( + runtime.methods.addToolFavorite({ userId: uid, itemType: 'tool', itemId: 'late' }), + ).rejects.toThrow('erasure'); + } finally { + await runtime.close(); + await expect(runtime.methods.getToolFavorites(uid)).rejects.toThrow('not initialized'); + } + } finally { + await db.close(); + } + }); + it('fails startup on a selected PostgreSQL outage without using Mongo', async () => { + const end = jest.spyOn(pg.Pool.prototype, 'end'); + const runtime = createFavoriteRuntime( + mongo, + { + PG_TEST_URL: 'postgresql://postgres@127.0.0.1:1/postgres', + }, + createPostgresFavoriteBackend, + ); + await expect(runtime.initialize(config)).rejects.toThrow('initialization failed'); + expect(end).toHaveBeenCalled(); + await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized'); + await runtime.close(); + }); + it('does not publish methods when shutdown races initialization', async () => { + const runtime = createFavoriteRuntime(mongo, env, createPostgresFavoriteBackend); + const initializing = runtime.initialize(config); + const stopping = runtime.close(); + await expect(initializing).rejects.toThrow('initialization failed'); + await stopping; + await expect(runtime.methods.getToolFavorites('user')).rejects.toThrow('not initialized'); + expect(runtime.close()).toBe(stopping); + }); +}); diff --git a/packages/api/src/app/persistence.ts b/packages/api/src/app/persistence.ts new file mode 100644 index 0000000000..4383933ee3 --- /dev/null +++ b/packages/api/src/app/persistence.ts @@ -0,0 +1,100 @@ +import { persistenceConfigSchema } from 'librechat-data-provider'; +import type { + ToolFavoriteMethods, + PostgresOptions, + PostgresFavoriteBackend, +} from '@librechat/data-schemas'; + +export interface FavoriteRuntime { + methods: ToolFavoriteMethods; + initialize(config: unknown): Promise; + close(): Promise; +} + +/** Captured route references never change, and no request chooses its storage engine. */ +export function createFavoriteRuntime( + mongo: ToolFavoriteMethods, + env: NodeJS.ProcessEnv, + createPostgres: (options: PostgresOptions) => PostgresFavoriteBackend, +): FavoriteRuntime { + let selected: ToolFavoriteMethods | undefined; + let backend: PostgresFavoriteBackend | undefined; + let backendShutdown: Promise | undefined; + const closeBackend = () => (backendShutdown ??= backend?.close() ?? Promise.resolve()); + let initialization: Promise | undefined; + let closed = false; + let shutdown: Promise | undefined; + function methods(): ToolFavoriteMethods { + if (!selected || closed) { + throw new Error('Favorites persistence is not initialized'); + } + return selected; + } + return { + methods: { + getToolFavorites: async (userId) => methods().getToolFavorites(userId), + addToolFavorite: async (params) => methods().addToolFavorite(params), + removeToolFavorite: async (params) => methods().removeToolFavorite(params), + deleteUserToolFavorites: async (userId, deletionAttemptId) => + deletionAttemptId === undefined + ? methods().deleteUserToolFavorites(userId) + : methods().deleteUserToolFavorites(userId, deletionAttemptId), + cancelUserToolFavoriteDeletion: async (userId, deletionAttemptId) => + methods().cancelUserToolFavoriteDeletion?.(userId, deletionAttemptId), + }, + initialize(config) { + if (closed) { + return Promise.reject(new Error('Favorites persistence is closed')); + } + initialization ??= (async () => { + if (closed) { + throw new Error('Favorites persistence is closed'); + } + const parsed = persistenceConfigSchema.safeParse(config); + if (!parsed.success) { + throw new Error('Invalid persistence configuration'); + } + if (parsed.data.favorites === 'mongo') { + selected = mongo; + return; + } + const options = parsed.data.postgres; + const connectionString = options && env[options.connectionStringEnv]; + if (!options || !connectionString) { + throw new Error('PostgreSQL connection environment variable is missing'); + } + try { + backend = createPostgres({ ...options, connectionString }); + await backend.initialize(); + if (closed) { + throw new Error('Favorites persistence is closed'); + } + selected = backend.methods; + } catch { + await closeBackend().catch(() => undefined); + throw new Error('PostgreSQL favorites initialization failed'); + } + })(); + return initialization; + }, + close() { + closed = true; + shutdown ??= (async () => { + await initialization?.catch(() => undefined); + await closeBackend(); + })(); + return shutdown; + }, + }; +} + +/** Storage selection is deployment-only, never a tenant config override. */ +export function getPersistenceConfig(config: unknown): unknown { + if (config == null) { + return undefined; + } + if (typeof config !== 'object' || Array.isArray(config)) { + throw new Error('Invalid deployment configuration'); + } + return 'persistence' in config ? config.persistence : undefined; +} diff --git a/packages/api/src/app/postgres.spec.ts b/packages/api/src/app/postgres.spec.ts new file mode 100644 index 0000000000..8b06ce0598 --- /dev/null +++ b/packages/api/src/app/postgres.spec.ts @@ -0,0 +1,648 @@ +import pg from 'pg'; +import mongoose from 'mongoose'; +import { resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { randomBytes } from 'node:crypto'; +import { execFile, spawnSync } from 'node:child_process'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { + createModels, + createMongoFavoriteStore, + createPostgresDatabase, + fingerprintFavorites, + readPostgresFavorites, +} from '@librechat/data-schemas'; +import type { MongoFavoriteStore } from '@librechat/data-schemas'; +import { runPostgresCommand } from './postgres'; + +describe('PostgreSQL command arguments', () => { + const favorites: MongoFavoriteStore = { + source: async function* () {}, + stage: jest.fn(), + }; + const disconnectDb = jest.fn(async () => undefined); + + const commands = ['migrate', 'import', 'verify', 'reconcile-mongo']; + const flags = ['--offline', '--dry-run', '--verified-empty']; + const combinations = commands.flatMap((command) => + Array.from({ length: 8 }, (_, mask) => ({ + args: [command, ...flags.filter((_flag, bit) => (mask & (1 << bit)) !== 0)], + })), + ); + it.each(combinations)('validates $args before configuration or connections', async ({ args }) => { + const [command, ...flags] = args; + const unsupported = + (command === 'migrate' && flags.length > 0) || + (command === 'verify' && flags.some((flag) => flag !== '--offline')) || + (command === 'reconcile-mongo' && flags.includes('--verified-empty')); + const missingOffline = command !== 'migrate' && !flags.includes('--offline'); + const sentinel = new Error('configuration not loaded'); + const loadCustomConfig = jest.fn().mockRejectedValue(sentinel); + const connectDb = jest.fn(); + const end = jest.spyOn(pg.Pool.prototype, 'end'); + const operation = runPostgresCommand({ + createDatabase: createPostgresDatabase, + favorites, + disconnectDb, + connectDb, + loadCustomConfig, + args, + env: {}, + }); + if (unsupported) await expect(operation).rejects.toThrow('Usage:'); + else if (missingOffline) await expect(operation).rejects.toThrow('pass --offline'); + else await expect(operation).rejects.toBe(sentinel); + expect(loadCustomConfig).toHaveBeenCalledTimes(unsupported || missingOffline ? 0 : 1); + expect(connectDb).not.toHaveBeenCalled(); + expect(end).not.toHaveBeenCalled(); + }); + it.each([ + { args: [] }, + { args: ['unknown'] }, + { args: ['__proto__'] }, + { args: ['constructor'] }, + ...commands.map((command) => ({ args: [command, '--unknown'] })), + ...commands.map((command) => ({ args: [command, '--offline', '--offline'] })), + ])('rejects unknown commands, flags, and duplicates: $args', async ({ args }) => { + const loadCustomConfig = jest.fn(); + const connectDb = jest.fn(); + await expect( + runPostgresCommand({ + createDatabase: createPostgresDatabase, + favorites, + disconnectDb, + connectDb, + loadCustomConfig, + args, + env: {}, + }), + ).rejects.toThrow('Usage:'); + expect(loadCustomConfig).not.toHaveBeenCalled(); + expect(connectDb).not.toHaveBeenCalled(); + }); +}); + +const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip; +realPostgres('PostgreSQL CLI with real Mongo and PostgreSQL', () => { + const schema = `favorites_cli_${randomBytes(6).toString('hex')}`; + const baseUrl = process.env.PG_TEST_URL ?? ''; + const url = new URL(baseUrl || 'postgresql://postgres@localhost/postgres'); + url.searchParams.set('options', `-c search_path=${schema}`); + const env = { PG_TEST_URL: url.toString() }; + const config = { + persistence: { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } }, + }; + const options = { + connectionString: env.PG_TEST_URL, + max: 2, + maxWaiting: 10, + acquireTimeoutMs: 1000, + queryTimeoutMs: 5000, + idleTimeoutMs: 1000, + }; + const admin = createPostgresDatabase({ ...options, connectionString: baseUrl }); + const db = createPostgresDatabase(options); + let mongo: MongoMemoryServer; + const connectDb = jest.fn(async () => + mongoose.connect(mongo.getUri(), { autoCreate: false, autoIndex: false }), + ); + const disconnectDb = () => mongoose.disconnect(); + const favorites = createMongoFavoriteStore(mongoose); + const loadCustomConfig = jest.fn(async () => config); + const run = (args: string[]) => + runPostgresCommand({ + createDatabase: createPostgresDatabase, + favorites, + connectDb, + disconnectDb, + loadCustomConfig, + args, + env, + }); + const collection = () => createModels(mongoose).ToolFavorite.collection; + const reconnect = () => mongoose.connect(mongo.getUri()); + const uid = new mongoose.Types.ObjectId('0123456789abcdef01234567'); + const row = (index: number) => ({ + _id: new mongoose.Types.ObjectId(index.toString(16).padStart(24, '0')), + user: uid, + tenantId: index % 2 ? 'tenant-one' : null, + itemType: 'tool', + itemId: `tool-${index}`, + createdAt: new Date('2025-01-01T00:00:00.000Z'), + updatedAt: new Date('2025-01-02T00:00:00.000Z'), + }); + beforeAll(async () => { + await admin.query(`CREATE SCHEMA ${schema}`); + mongo = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } }); + }, 90_000); + beforeEach(async () => { + jest.clearAllMocks(); + await run(['migrate']); + await db.query('TRUNCATE tool_favorites, tool_favorite_erasure'); + await db.query( + 'UPDATE tool_favorite_readiness SET verified=false, activated=false, checkpoint=NULL, snapshot=NULL, imported=0', + ); + await reconnect(); + await collection().deleteMany({}); + await collection().createIndex({ user: 1, itemType: 1, itemId: 1 }, { unique: true }); + await collection().createIndex({ tenantId: 1 }); + await mongoose.disconnect(); + connectDb.mockClear(); + }); + afterAll(async () => { + await mongoose.disconnect(); + await mongo?.stop(); + await db.close(); + await admin.query(`DROP SCHEMA ${schema} CASCADE`); + await admin.close(); + }); + it('imports and reconciles through plain snapshot, staging, and lifecycle interfaces', async () => { + const snapshot = { + id: row(1)._id.toHexString(), + userId: uid.toHexString(), + tenantId: 'tenant-one', + itemType: 'tool', + itemId: 'portable-store', + createdAt: row(1).createdAt, + updatedAt: row(1).updatedAt, + } as const; + const source = async function* () { + yield snapshot; + }; + const insert = jest.fn(async () => undefined); + const replace = jest.fn(async () => undefined); + const discard = jest.fn(async () => undefined); + const store: MongoFavoriteStore = { + source, + stage: jest.fn(async () => ({ source, insert, replace, discard })), + }; + const connect = jest.fn(async () => undefined); + const disconnect = jest.fn(async () => undefined); + const createDatabase = jest.fn(createPostgresDatabase); + for (const command of ['import', 'verify', 'reconcile-mongo']) { + await runPostgresCommand({ + createDatabase, + favorites: store, + connectDb: connect, + disconnectDb: disconnect, + loadCustomConfig, + args: [command, '--offline'], + env, + }); + } + expect(createDatabase).toHaveBeenCalledTimes(3); + expect(createDatabase).toHaveBeenCalledWith( + expect.objectContaining({ connectionString: env.PG_TEST_URL }), + ); + expect(connect.mock.calls).toEqual([[], [], []]); + expect(disconnect).toHaveBeenCalledTimes(3); + expect(store.stage).toHaveBeenCalledTimes(1); + expect(insert).toHaveBeenCalledWith([snapshot]); + expect(replace).toHaveBeenCalledTimes(1); + expect(discard).not.toHaveBeenCalled(); + expect(connectDb).not.toHaveBeenCalled(); + }); + it('runs the actual migrate entrypoint with MONGO_URI unset', () => { + const scratchRoot = resolve(__dirname, '../../../../node_modules/.cache/pg-lifecycle'); + mkdirSync(scratchRoot, { recursive: true }); + const scratch = mkdtempSync(resolve(scratchRoot, 'cli-')); + try { + const configPath = resolve(scratch, 'config.yaml'); + writeFileSync( + configPath, + 'version: 1.3.1\npersistence:\n favorites: postgres\n postgres:\n connectionStringEnv: PG_TEST_URL\n', + ); + const script = resolve(__dirname, '../../../../config/postgres.js'); + const child = spawnSync( + process.execPath, + [ + '-e', + `delete process.env.MONGO_URI; process.argv = ['node', ${JSON.stringify(script)}, 'migrate']; require(${JSON.stringify(script)});`, + ], + { + cwd: scratch, + env: { ...process.env, ...env, CONFIG_PATH: configPath }, + encoding: 'utf8', + timeout: 15000, + killSignal: 'SIGKILL', + }, + ); + expect({ status: child.status, error: child.error?.message, stderr: child.stderr }).toEqual({ + status: 0, + error: undefined, + stderr: '', + }); + expect(child.stdout).toContain('PostgreSQL schema migrated'); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } + }); + it.each([ + { args: ['import', '--offline', '--dry-run'], seeded: true }, + { args: ['import', '--offline', '--dry-run', '--verified-empty'], seeded: false }, + { args: ['reconcile-mongo', '--offline', '--dry-run'], seeded: true }, + { args: ['reconcile-mongo', '--offline', '--dry-run'], seeded: false }, + { args: ['import', '--offline'], seeded: true }, + { args: ['verify', '--offline'], seeded: true }, + { args: ['reconcile-mongo', '--offline'], seeded: true }, + ])( + 'isolates actual CLI $args from Mongo schema initialization (seeded=$seeded)', + async ({ args, seeded }) => { + const native = await mongoose + .createConnection(mongo.getUri(`cli_${randomBytes(6).toString('hex')}`), { + autoCreate: false, + autoIndex: false, + }) + .asPromise(); + const scratchRoot = resolve(__dirname, '../../../../node_modules/.cache/pg-lifecycle'); + mkdirSync(scratchRoot, { recursive: true }); + const scratch = mkdtempSync(resolve(scratchRoot, 'cli-')); + const favorites = native.collection(collection().name); + const mongoSnapshot = async () => { + const collections = await native.db!.listCollections().toArray(); + return Promise.all( + collections + .sort((a, b) => a.name.localeCompare(b.name)) + .map(async (metadata) => ({ + metadata, + indexes: await native.collection(metadata.name).indexes(), + rows: await native.collection(metadata.name).find().sort({ _id: 1 }).toArray(), + })), + ); + }; + const postgresSnapshot = async () => { + const tables = await db.query<{ tablename: string }>( + 'SELECT tablename FROM pg_tables WHERE schemaname=current_schema() ORDER BY tablename', + ); + return { + columns: ( + await db.query( + 'SELECT * FROM information_schema.columns WHERE table_schema=current_schema() ORDER BY table_name,ordinal_position', + ) + ).rows, + indexes: ( + await db.query( + 'SELECT * FROM pg_indexes WHERE schemaname=current_schema() ORDER BY tablename,indexname', + ) + ).rows, + tables: await Promise.all( + tables.rows.map(async ({ tablename }) => ({ + tablename, + rows: (await db.query(`SELECT * FROM "${tablename}" ORDER BY 1`)).rows, + })), + ), + }; + }; + try { + if (seeded) await favorites.insertOne(row(1)); + if (args[0] !== 'import') { + await db.query( + 'INSERT INTO tool_favorites(id,user_id,tenant_id,item_type,item_id,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7)', + [ + row(1)._id.toHexString(), + uid.toHexString(), + row(1).tenantId, + 'tool', + row(1).itemId, + row(1).createdAt, + row(1).updatedAt, + ], + ); + const snapshot = await fingerprintFavorites(readPostgresFavorites(db, 250)); + await db.query( + 'UPDATE tool_favorite_readiness SET verified=true,activated=true,snapshot=$1', + [snapshot.digest], + ); + } + const beforeMongo = await mongoSnapshot(); + const beforePostgres = await postgresSnapshot(); + const configPath = resolve(scratch, 'config.yaml'); + writeFileSync( + configPath, + 'version: 1.3.1\npersistence:\n favorites: postgres\n postgres:\n connectionStringEnv: PG_TEST_URL\n', + ); + const guardPath = resolve(scratch, 'guard.cjs'); + writeFileSync( + guardPath, + ` +const assert = require('node:assert/strict'); +const mongoose = require(${JSON.stringify(require.resolve('mongoose'))}); +const connect = mongoose.connect.bind(mongoose); +mongoose.connect = (uri, options) => { + assert.equal(options.autoCreate, false); + assert.equal(options.autoIndex, false); + return connect(uri, options); +}; +mongoose.model = () => { throw new Error('CLI registered a connected model'); }; +process.on('beforeExit', () => { + assert.deepEqual(mongoose.modelNames(), []); + assert.equal(Object.keys(require.cache).some(path => path.endsWith('/api/db/index.js')), false); +}); +`, + ); + const child = await promisify(execFile)( + process.execPath, + ['--require', guardPath, resolve(__dirname, '../../../../config/postgres.js'), ...args], + { + cwd: scratch, + env: { + ...process.env, + ...env, + CONFIG_PATH: configPath, + MONGO_URI: mongo.getUri(native.name), + MONGO_AUTO_CREATE: 'true', + MONGO_AUTO_INDEX: 'true', + }, + encoding: 'utf8', + timeout: 15000, + killSignal: 'SIGKILL', + }, + ); + expect(child.stderr).toBe(''); + const afterMongo = await mongoSnapshot(); + if (args[0] === 'reconcile-mongo' && !args.includes('--dry-run')) { + expect(afterMongo).toHaveLength(1); + const before = beforeMongo[0].metadata; + const after = afterMongo[0].metadata; + if (!('info' in before) || !before.info || !('info' in after) || !after.info) { + throw new Error('Full Mongo collection metadata is required'); + } + expect(after.info.uuid).not.toEqual(before.info.uuid); + before.info.uuid = after.info.uuid; + } + expect(afterMongo).toEqual(beforeMongo); + if (args[0] !== 'import' || args.includes('--dry-run')) { + expect(await postgresSnapshot()).toEqual(beforePostgres); + } else { + expect((await db.query('SELECT count(*) FROM tool_favorites')).rows[0].count).toBe('1'); + } + } finally { + await native.dropDatabase(); + await native.close(); + rmSync(scratch, { recursive: true, force: true }); + } + }, + ); + it('rejects a migration dry run before creating any schema objects or opening connections', async () => { + const emptySchema = `favorites_dry_run_${randomBytes(6).toString('hex')}`; + await admin.query(`CREATE SCHEMA ${emptySchema}`); + const target = new URL(baseUrl); + target.searchParams.set('options', `-c search_path=${emptySchema}`); + const end = jest.spyOn(pg.Pool.prototype, 'end'); + loadCustomConfig.mockClear(); + try { + const result = await runPostgresCommand({ + createDatabase: createPostgresDatabase, + favorites, + disconnectDb, + connectDb, + loadCustomConfig, + args: ['migrate', '--dry-run'], + env: { PG_TEST_URL: target.toString() }, + }).then( + () => 'success', + (error: Error) => error.message, + ); + const objects = await admin.query('SELECT tablename FROM pg_tables WHERE schemaname=$1', [ + emptySchema, + ]); + expect(objects.rows).toEqual([]); + expect(result).toContain('Usage:'); + expect(loadCustomConfig).not.toHaveBeenCalled(); + expect(connectDb).not.toHaveBeenCalled(); + expect(end).not.toHaveBeenCalled(); + } finally { + await admin.query(`DROP SCHEMA ${emptySchema} CASCADE`); + } + }); + it('migrates without connecting Mongo and releases its pool', async () => { + const end = jest.spyOn(pg.Pool.prototype, 'end'); + await run(['migrate']); + expect(connectDb).not.toHaveBeenCalled(); + expect(end).toHaveBeenCalledTimes(1); + expect(loadCustomConfig).toHaveBeenCalledWith(false, { + mode: 'reload', + storageSelection: true, + }); + }); + it.each(['import', 'verify', 'reconcile-mongo'])( + 'requires offline for %s before opening connections', + async (command) => { + const end = jest.spyOn(pg.Pool.prototype, 'end'); + const disconnect = jest.spyOn(mongoose, 'disconnect'); + await expect(run([command])).rejects.toThrow('--offline'); + expect(connectDb).not.toHaveBeenCalled(); + expect(end).not.toHaveBeenCalled(); + expect(disconnect).toHaveBeenCalledTimes(1); + }, + ); + it('validates an import dry run without changing data or readiness', async () => { + await reconnect(); + await collection().insertMany([row(1), row(2)]); + const original = await collection().find().sort({ _id: 1 }).toArray(); + await mongoose.disconnect(); + const state = await db.query('SELECT * FROM tool_favorite_readiness'); + await run(['import', '--offline', '--dry-run']); + expect((await db.query('SELECT * FROM tool_favorites')).rows).toEqual([]); + expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(state.rows); + await reconnect(); + expect(await collection().find().sort({ _id: 1 }).toArray()).toEqual(original); + }); + it('imports and verifies identity, tenant and dates, then reconciles deletes and preserves indexes without changing authority', async () => { + await reconnect(); + await collection().insertMany([row(1), row(2)]); + const indexes = await collection().indexes(); + await mongoose.disconnect(); + await run(['import', '--offline']); + await run(['verify', '--offline']); + const expected = await fingerprintFavorites(readPostgresFavorites(db, 250)); + await db.query('DELETE FROM tool_favorites WHERE id=$1', [row(1)._id.toHexString()]); + const state = await db.query('SELECT * FROM tool_favorite_readiness'); + await expect(run(['verify', '--offline'])).rejects.toThrow('differ'); + await run(['reconcile-mongo', '--offline', '--dry-run']); + await reconnect(); + expect(await collection().countDocuments()).toBe(2); + await mongoose.disconnect(); + await run(['reconcile-mongo', '--offline']); + await run(['verify', '--offline']); + await reconnect(); + expect(await collection().findOne({ _id: row(1)._id })).toBeNull(); + expect(await collection().findOne({ _id: row(2)._id })).toEqual(row(2)); + expect(await collection().indexes()).toEqual(indexes); + expect( + await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(), + ).toEqual([]); + expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual(state.rows); + expect(expected.count).toBe(2); + }); + it.each(['schema-only', 'partial', 'missing-marker'] as const)( + 'refuses destructive reconciliation from %s PostgreSQL without modifying Mongo', + async (state) => { + await reconnect(); + await collection().insertMany([row(1), row(2)]); + const original = await collection().find().sort({ _id: 1 }).toArray(); + const indexes = await collection().indexes(); + await mongoose.disconnect(); + if (state !== 'schema-only') { + await run(['import', '--offline']); + await db.query('DELETE FROM tool_favorites WHERE id=$1', [row(2)._id.toHexString()]); + await db.query('UPDATE tool_favorite_readiness SET verified=false,checkpoint=$1', [ + row(1)._id.toHexString(), + ]); + } + if (state === 'missing-marker') await db.query('DELETE FROM tool_favorite_readiness'); + const authority = await db.query('SELECT * FROM tool_favorite_readiness'); + for (const flags of [[], ['--dry-run']]) { + await expect(run(['reconcile-mongo', '--offline', ...flags])).rejects.toThrow( + 'not verified', + ); + await reconnect(); + expect(await collection().find().sort({ _id: 1 }).toArray()).toEqual(original); + expect(await collection().indexes()).toEqual(indexes); + expect( + await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(), + ).toEqual([]); + await mongoose.disconnect(); + } + expect((await db.query('SELECT * FROM tool_favorite_readiness')).rows).toEqual( + authority.rows, + ); + if (state === 'missing-marker') + await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)'); + }, + ); + it('allows verified activated PostgreSQL to reconcile subsequent deletions, including an empty snapshot', async () => { + await reconnect(); + await collection().insertOne(row(1)); + await mongoose.disconnect(); + await run(['import', '--offline']); + await db.query('UPDATE tool_favorite_readiness SET activated=true'); + await db.query('DELETE FROM tool_favorites'); + await run(['reconcile-mongo', '--offline']); + await reconnect(); + expect(await collection().countDocuments()).toBe(0); + expect((await db.query('SELECT verified,activated FROM tool_favorite_readiness')).rows).toEqual( + [{ verified: true, activated: true }], + ); + }); + it('refuses a same-count corrupted staging snapshot, retains Mongo, and removes staging', async () => { + await reconnect(); + await collection().insertOne(row(1)); + await mongoose.disconnect(); + await run(['import', '--offline']); + const original = mongoose.connection.collection.bind(mongoose.connection); + const staging = jest + .spyOn(mongoose.connection, 'collection') + .mockImplementation((name, options) => { + const target = original(name, options); + if (name.includes('_reconcile_')) { + const insert = target.insertMany.bind(target); + jest.spyOn(target, 'insertMany').mockImplementation(async (docs, options) => { + const result = await insert(docs, options); + await target.updateOne({}, { $set: { itemId: 'corrupt' } }); + return result; + }); + } + return target; + }); + await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow( + 'changed during reconciliation', + ); + staging.mockRestore(); + await reconnect(); + expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1)); + expect( + await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(), + ).toEqual([]); + }); + it('rechecks PostgreSQL readiness before Mongo replacement and discards untrusted staging', async () => { + await reconnect(); + await collection().insertOne(row(1)); + await mongoose.disconnect(); + await run(['import', '--offline']); + const original = mongoose.connection.collection.bind(mongoose.connection); + jest.spyOn(mongoose.connection, 'collection').mockImplementation((name, options) => { + const target = original(name, options); + if (name.includes('_reconcile_')) { + const insert = target.insertMany.bind(target); + jest.spyOn(target, 'insertMany').mockImplementation(async (docs, options) => { + const result = await insert(docs, options); + await db.query('UPDATE tool_favorite_readiness SET verified=false'); + return result; + }); + } + return target; + }); + await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow('not verified'); + jest.restoreAllMocks(); + await reconnect(); + expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1)); + expect( + await mongoose.connection.db!.listCollections({ name: /_reconcile_/ }).toArray(), + ).toEqual([]); + }); + it('does not race a migration or import holding the same schema authority', async () => { + await reconnect(); + await collection().insertOne(row(1)); + await mongoose.disconnect(); + await run(['import', '--offline']); + await db.session(async (session) => { + await session.query('SELECT pg_advisory_lock(17490261,hashtext(current_schema()))'); + try { + await expect(run(['reconcile-mongo', '--offline'])).rejects.toThrow( + 'Another favorite migration', + ); + } finally { + await session.query('SELECT pg_advisory_unlock(17490261,hashtext(current_schema()))'); + } + }); + await reconnect(); + expect(await collection().findOne({ _id: row(1)._id })).toEqual(row(1)); + }); + it('reports cleanup failure after a successful command instead of silently succeeding', async () => { + const original = pg.Pool.prototype.end; + jest.spyOn(pg.Pool.prototype, 'end').mockImplementationOnce(async function (this: pg.Pool) { + await original.bind(this)(); + throw new Error('cleanup failed'); + }); + const disconnect = jest.spyOn(mongoose, 'disconnect'); + await expect(run(['migrate'])).rejects.toThrow('command cleanup failed'); + expect(disconnect).toHaveBeenCalledTimes(1); + }); + it.each([{ user: null }, { tenantId: 42 }, { createdAt: null }, { updatedAt: 'invalid' }])( + 'closes pools and Mongo after malformed source validation without exposing the record: %j', + async (invalid) => { + await reconnect(); + await collection().insertOne({ ...row(1), ...invalid }); + await mongoose.disconnect(); + const end = jest.spyOn(pg.Pool.prototype, 'end'); + const disconnect = jest.spyOn(mongoose, 'disconnect'); + await expect(run(['import', '--offline'])).rejects.toThrow( + 'Invalid favorite snapshot record', + ); + expect(end).toHaveBeenCalledTimes(1); + expect(disconnect).toHaveBeenCalledTimes(1); + expect((await db.query('SELECT count(*) FROM tool_favorites')).rows[0].count).toBe('0'); + }, + ); + it('keeps the operational error when pool shutdown also fails, and still disconnects Mongo', async () => { + await reconnect(); + await collection().insertOne({ ...row(1), user: null }); + await mongoose.disconnect(); + const original = pg.Pool.prototype.end; + jest.spyOn(pg.Pool.prototype, 'end').mockImplementationOnce(async function (this: pg.Pool) { + await original.bind(this)(); + throw new Error('cleanup failed'); + }); + const disconnect = jest.spyOn(mongoose, 'disconnect'); + await expect(run(['import', '--offline'])).rejects.toThrow('Invalid favorite snapshot record'); + expect(disconnect).toHaveBeenCalledTimes(1); + }); + it('disconnects Mongo even when configuration loading fails before a pool exists', async () => { + loadCustomConfig.mockRejectedValueOnce(new Error('Deployment config unavailable')); + const disconnect = jest.spyOn(mongoose, 'disconnect'); + await expect(run(['migrate'])).rejects.toThrow('Deployment config unavailable'); + expect(disconnect).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/app/postgres.ts b/packages/api/src/app/postgres.ts new file mode 100644 index 0000000000..005f4c4d68 --- /dev/null +++ b/packages/api/src/app/postgres.ts @@ -0,0 +1,164 @@ +import { persistenceConfigSchema } from 'librechat-data-provider'; +import { + migratePostgres, + importFavorites, + fingerprintFavorites, + readPostgresFavorites, + assertPostgresDataReady, +} from '@librechat/data-schemas'; +import type { + FavoriteSnapshotRow, + MongoFavoriteStore, + PostgresDatabase, + PostgresOptions, +} from '@librechat/data-schemas'; +import { getPersistenceConfig } from './persistence'; + +export async function runPostgresCommand(deps: { + favorites: MongoFavoriteStore; + createDatabase(options: PostgresOptions): PostgresDatabase; + connectDb(): Promise; + disconnectDb(): Promise; + loadCustomConfig( + print: boolean, + options: { mode: 'reload'; storageSelection: true }, + ): Promise; + args: string[]; + env: NodeJS.ProcessEnv; +}): Promise { + let db: PostgresDatabase | undefined; + const close = () => Promise.allSettled([db?.close(), deps.disconnectDb()]); + const execute = async () => { + const [command, ...flags] = deps.args; + const commands: Record = { + migrate: [], + import: ['--offline', '--dry-run', '--verified-empty'], + verify: ['--offline'], + 'reconcile-mongo': ['--offline', '--dry-run'], + }; + const allowed = Object.prototype.hasOwnProperty.call(commands, command) + ? commands[command] + : undefined; + if ( + !allowed || + flags.some((flag) => !allowed.includes(flag)) || + new Set(flags).size !== flags.length + ) { + throw new Error( + 'Usage: postgres migrate | import --offline [--dry-run] [--verified-empty] | verify --offline | reconcile-mongo --offline [--dry-run]', + ); + } + if (command !== 'migrate' && !flags.includes('--offline')) { + throw new Error('Stop all app, worker, and migration writers and pass --offline'); + } + const config = persistenceConfigSchema.parse( + getPersistenceConfig( + await deps.loadCustomConfig(false, { mode: 'reload', storageSelection: true }), + ), + ); + const options = config.postgres; + const connectionString = options && deps.env[options.connectionStringEnv]; + if (!options || !connectionString) { + throw new Error('Missing PostgreSQL connection environment variable'); + } + db = deps.createDatabase({ ...options, connectionString }); + if (command === 'migrate') { + await migratePostgres(db); + console.info('PostgreSQL schema migrated. Favorites data is not yet verified.'); + return; + } + await deps.connectDb(); + const mongo = deps.favorites; + const source = mongo.source; + if (command === 'import') { + const result = await importFavorites(db, source, { + offline: true, + batchSize: 250, + dryRun: flags.includes('--dry-run'), + verifiedEmpty: flags.includes('--verified-empty'), + }); + console.info( + `Favorites snapshot ${flags.includes('--dry-run') ? 'validated' : 'imported and verified'}: ${result.count} rows`, + ); + return; + } + if (command === 'verify') { + const postgres = await fingerprintFavorites(readPostgresFavorites(db, 250)); + const mongo = await fingerprintFavorites(source); + if (postgres.digest !== mongo.digest || postgres.count !== mongo.count) { + throw new Error('Favorite snapshots differ'); + } + console.info(`Favorites snapshots match: ${postgres.count} rows`); + return; + } + await db.session(async (session) => { + const lock = await session.query<{ acquired: boolean }>( + 'SELECT pg_try_advisory_lock(17490261, hashtext(current_schema())) AS acquired', + ); + if (!lock.rows[0].acquired) + throw new Error('Another favorite migration or import is running'); + try { + await assertPostgresDataReady(session); + const pgSource = readPostgresFavorites(session, 250); + const postgres = await fingerprintFavorites(pgSource); + // Build and validate a replacement before an atomic rename. Deletes are reconciled too. + if (flags.includes('--dry-run')) { + console.info( + `Offline reconciliation would replace Mongo favorites with ${postgres.count} PostgreSQL rows`, + ); + return; + } + const staging = await mongo.stage(); + let renamed = false; + try { + let batch: FavoriteSnapshotRow[] = []; + const write = async () => { + await staging.insert(batch); + batch = []; + }; + for await (const row of pgSource()) { + batch.push(row); + if (batch.length === 250) { + await write(); + } + } + if (batch.length) { + await write(); + } + const staged = await fingerprintFavorites(staging.source); + const current = await fingerprintFavorites(pgSource); + if ( + current.digest !== postgres.digest || + current.count !== postgres.count || + staged.digest !== postgres.digest || + staged.count !== postgres.count + ) { + throw new Error('PostgreSQL changed during reconciliation'); + } + await assertPostgresDataReady(session); + await staging.replace(); + renamed = true; + console.info( + `Mongo favorites reconciled: ${postgres.count} rows. Storage selection was not changed.`, + ); + } finally { + if (!renamed) { + await staging.discard(); + } + } + } finally { + await session.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))'); + } + }); + }; + try { + await execute(); + } catch (error) { + await close(); + throw error; + } + const cleanup = await close(); + if (cleanup.some((result) => result.status === 'rejected')) { + throw new Error('PostgreSQL command cleanup failed'); + } +} diff --git a/packages/api/src/user/erasure.spec.ts b/packages/api/src/user/erasure.spec.ts index 999d0d80d7..b68221e1da 100644 --- a/packages/api/src/user/erasure.spec.ts +++ b/packages/api/src/user/erasure.spec.ts @@ -1,23 +1,35 @@ import { Keyv } from 'keyv'; import mongoose from 'mongoose'; +import { randomUUID } from 'node:crypto'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { ResourceType, PrincipalType } from 'librechat-data-provider'; import { + createPostgresFavoriteBackend, + createPostgresDatabase, + migratePostgres, + importFavorites, createModels, createMethods, tenantStorage, createSkillDeletionPersistenceAdapter, logger, } from '@librechat/data-schemas'; +import type { PostgresDatabase } from '@librechat/data-schemas'; import type { AppConfig } from '@librechat/data-schemas'; import type { CheckpointDeletion } from '~/agents/checkpoints/deletion'; import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure'; import type { MCPOAuthTokens } from '~/mcp/oauth/types'; +import { + deleteUserMCPServers, + eraseUserData, + cancelUserDataDeletion, + createUserDeletionAttempt, +} from './erasure'; import { MCPOAuthHandler, MCPTokenStorage, getMCPOAuthLeaseId } from '~/mcp/oauth'; import { deleteAllSharedLinksWithCleanup } from '~/shared-links/service'; import { createSkillDeletionService } from '../skills/deletionService'; -import { deleteUserMCPServers, eraseUserData } from './erasure'; import { cleanupMCPServerOAuth } from '~/mcp/oauth/cleanup'; +import { createFavoriteRuntime } from '~/app/persistence'; import { FlowStateManager } from '~/flow/manager'; /** Stored OAuth credentials are encrypted with a key read when the module loads. */ @@ -903,3 +915,332 @@ describe('removing a shared MCP server with its owner', () => { expect(await count('MCPServer', { _id: mcpServerId })).toBe(1); }); }); + +const realPostgres = process.env.PG_TEST_URL ? describe : describe.skip; +realPostgres('attempt-scoped PostgreSQL account erasure', () => { + const schema = `erasure_${randomUUID().replace(/-/g, '')}`; + const config = { favorites: 'postgres', postgres: { connectionStringEnv: 'PG_TEST_URL' } }; + const options = { + connectionString: process.env.PG_TEST_URL ?? '', + max: 2, + maxWaiting: 10, + acquireTimeoutMs: 1000, + queryTimeoutMs: 5000, + idleTimeoutMs: 1000, + }; + let admin: PostgresDatabase; + let db: PostgresDatabase; + beforeAll(async () => { + admin = createPostgresDatabase(options); + await admin.query(`CREATE SCHEMA ${schema}`); + const url = new URL(options.connectionString); + url.searchParams.set('options', `-c search_path=${schema}`); + options.connectionString = url.toString(); + db = createPostgresDatabase(options); + await migratePostgres(db); + await importFavorites(db, async function* () {}, { + offline: true, + verifiedEmpty: true, + batchSize: 10, + }); + }); + afterAll(async () => { + await db.close(); + await admin.query(`DROP SCHEMA ${schema} CASCADE`); + await admin.close(); + }); + async function setup() { + const userId = new mongoose.Types.ObjectId().toString(); + const startedAt = new Date(); + const deletionAttemptId = randomUUID(); + await model('User').insertOne({ + _id: new mongoose.Types.ObjectId(userId), + agentTriggerDeletionStartedAt: startedAt, + agentTriggerDeletionAttemptId: deletionAttemptId, + }); + const runtime = createFavoriteRuntime( + methods, + { PG_TEST_URL: options.connectionString }, + createPostgresFavoriteBackend, + ); + await runtime.initialize(config); + const { deps } = createDeps(); + const erasure = { ...deps, ...runtime.methods }; + const cancel = { ...methods, ...runtime.methods }; + const params = { userId, startedAt, deletionAttemptId }; + const favorite = { userId, itemType: 'tool', itemId: 'saved' } as const; + await runtime.methods.addToolFavorite(favorite); + return { runtime, erasure, cancel, params, favorite }; + } + it('unfences a retained account after a later cascade failure, including after restart', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + erasure.deleteSharedLinks = jest.fn().mockRejectedValue(new Error('shared links unavailable')); + try { + await expect(eraseUserData(params, erasure)).rejects.toThrow('shared links unavailable'); + expect(await count('User', { _id: new mongoose.Types.ObjectId(params.userId) })).toBe(1); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + await cancelUserDataDeletion(params, cancel); + await runtime.close(); + const restarted = createFavoriteRuntime( + methods, + { PG_TEST_URL: options.connectionString }, + createPostgresFavoriteBackend, + ); + try { + await restarted.initialize(config); + await expect(restarted.methods.addToolFavorite(favorite)).resolves.toEqual({ + ok: true, + added: true, + }); + expect(await restarted.methods.getToolFavorites(params.userId)).toEqual([ + { itemType: 'tool', itemId: 'saved' }, + ]); + } finally { + await restarted.close(); + } + } finally { + await runtime.close(); + } + }); + it.each(['deleted', 'other-attempt', 'mongo-error'] as const)( + 'keeps the marker when cancellation cannot prove ownership: %s', + async (reason) => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + try { + await eraseUserData(params, erasure); + if (reason === 'deleted') await methods.deleteUserById(params.userId); + if (reason === 'other-attempt') + await model('User').updateOne( + { _id: new mongoose.Types.ObjectId(params.userId) }, + { $set: { agentTriggerDeletionStartedAt: new Date(params.startedAt.getTime() + 1) } }, + ); + if (reason === 'mongo-error') + jest + .spyOn(mongoose.models.User.collection, 'findOne') + .mockRejectedValueOnce(new Error('Mongo unavailable')); + if (reason === 'mongo-error') + await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow('Mongo unavailable'); + else await cancelUserDataDeletion(params, cancel); + if (reason === 'mongo-error') + expect( + await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt), + ).toBe(true); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + await runtime.close(); + const restarted = createFavoriteRuntime( + methods, + { PG_TEST_URL: options.connectionString }, + createPostgresFavoriteBackend, + ); + try { + await restarted.initialize(config); + await expect(restarted.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + } finally { + await restarted.close(); + } + } finally { + await runtime.close(); + } + }, + ); + it('retains Mongo ownership through a PostgreSQL cancellation fault, then retries after restart', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + try { + await eraseUserData(params, erasure); + await db.query( + `CREATE FUNCTION reject_cancel() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'cancel unavailable'; END $$`, + ); + await db.query( + 'CREATE TRIGGER reject_cancel BEFORE DELETE ON tool_favorite_erasure FOR EACH ROW EXECUTE FUNCTION reject_cancel()', + ); + try { + await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow('cancel unavailable'); + } finally { + await db.query('DROP TRIGGER reject_cancel ON tool_favorite_erasure'); + await db.query('DROP FUNCTION reject_cancel()'); + } + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe( + true, + ); + await runtime.close(); + const restarted = createFavoriteRuntime( + methods, + { PG_TEST_URL: options.connectionString }, + createPostgresFavoriteBackend, + ); + try { + await restarted.initialize(config); + await cancelUserDataDeletion(params, { ...methods, ...restarted.methods }); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe( + false, + ); + await expect(restarted.methods.addToolFavorite(favorite)).resolves.toEqual({ + ok: true, + added: true, + }); + } finally { + await restarted.close(); + } + } finally { + await runtime.close(); + } + }); + it('retries Mongo release after PostgreSQL cleanup without losing the durable owner', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + try { + await eraseUserData(params, erasure); + // Fail the real release write after the ownership read and PostgreSQL cancellation. + jest + .spyOn(mongoose.models.User.collection, 'updateOne') + .mockRejectedValueOnce(new Error('Mongo release unavailable')); + await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow( + 'Mongo release unavailable', + ); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe( + true, + ); + await expect(runtime.methods.addToolFavorite(favorite)).resolves.toEqual({ + ok: true, + added: true, + }); + await cancelUserDataDeletion(params, cancel); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe( + false, + ); + } finally { + await runtime.close(); + } + }); + it('tolerates a lost Mongo release acknowledgement after PostgreSQL cleanup', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + try { + await eraseUserData(params, erasure); + cancel.cancelAgentTriggerUserDeletion = async (userId, startedAt) => { + await methods.cancelAgentTriggerUserDeletion(userId, startedAt, params.deletionAttemptId); + throw new Error('release acknowledgement lost'); + }; + await expect(cancelUserDataDeletion(params, cancel)).rejects.toThrow( + 'release acknowledgement lost', + ); + await cancelUserDataDeletion(params, { ...methods, ...runtime.methods }); + await expect(runtime.methods.addToolFavorite(favorite)).resolves.toEqual({ + ok: true, + added: true, + }); + } finally { + await runtime.close(); + } + }); + it('makes concurrent cancellation idempotent without releasing a successor UUID', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + try { + await eraseUserData(params, erasure); + const successor = randomUUID(); + await runtime.methods.deleteUserToolFavorites(params.userId, successor); + await Promise.all([ + cancelUserDataDeletion(params, cancel), + cancelUserDataDeletion(params, cancel), + ]); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + expect( + ( + await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [ + params.userId, + ]) + ).rows, + ).toEqual([{ attempt_id: successor }]); + } finally { + await runtime.close(); + } + }); + it('preserves a successor owner and UUID acquired after the ownership proof', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + const successor = randomUUID(); + const nextStartedAt = new Date(params.startedAt.getTime() + 1); + try { + await eraseUserData(params, erasure); + cancel.isAgentTriggerUserDeletionOwner = async (userId, startedAt) => { + const owned = await methods.isAgentTriggerUserDeletionOwner(userId, startedAt); + await methods.cancelAgentTriggerUserDeletion(userId, startedAt, params.deletionAttemptId); + await methods.beginAgentTriggerUserDeletion(userId, nextStartedAt, successor); + await runtime.methods.deleteUserToolFavorites(userId, successor); + return owned; + }; + await cancelUserDataDeletion(params, cancel); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, nextStartedAt)).toBe( + true, + ); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + expect( + ( + await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [ + params.userId, + ]) + ).rows, + ).toEqual([{ attempt_id: successor }]); + } finally { + await runtime.close(); + } + }); + it('keeps both fences when a final delete is pending or loses its acknowledgement', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + const attempt = createUserDeletionAttempt(); + let releaseCommit!: () => void; + const gate = new Promise((resolve) => { + releaseCommit = resolve; + }); + try { + await eraseUserData(params, erasure); + const committing = attempt.commit(async () => { + await gate; + await methods.deleteUserById(params.userId, { + startedAt: params.startedAt, + attemptId: params.deletionAttemptId, + }); + throw new Error('delete acknowledgement lost'); + }); + await attempt.cancel(params, cancel); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe( + true, + ); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + releaseCommit(); + await expect(committing).rejects.toThrow('delete acknowledgement lost'); + await attempt.cancel(params, cancel); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + } finally { + releaseCommit(); + await runtime.close(); + } + }); + it('keeps both fences after a final delete throws without a known outcome', async () => { + const { runtime, erasure, cancel, params, favorite } = await setup(); + const attempt = createUserDeletionAttempt(); + try { + await eraseUserData(params, erasure); + await expect( + attempt.commit(async () => { + throw new Error('delete outcome unknown'); + }), + ).rejects.toThrow('delete outcome unknown'); + await attempt.cancel(params, cancel); + expect(await methods.isAgentTriggerUserDeletionOwner(params.userId, params.startedAt)).toBe( + true, + ); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + } finally { + await runtime.close(); + } + }); + it('never cancels a permanent marker when no UUID was supplied', async () => { + const { runtime, cancel, params, favorite } = await setup(); + try { + await runtime.methods.deleteUserToolFavorites(params.userId); + await cancelUserDataDeletion({ userId: params.userId, startedAt: params.startedAt }, cancel); + await expect(runtime.methods.addToolFavorite(favorite)).rejects.toThrow('erasure'); + } finally { + await runtime.close(); + } + }); +}); diff --git a/packages/api/src/user/erasure.ts b/packages/api/src/user/erasure.ts index 2bc83165a8..66e61ad512 100644 --- a/packages/api/src/user/erasure.ts +++ b/packages/api/src/user/erasure.ts @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import { randomUUID } from 'node:crypto'; import { logger } from '@librechat/data-schemas'; import type { TCheckpointerConfig, MCPOptions } from 'librechat-data-provider'; import type { AllMethods, IMongoFile } from '@librechat/data-schemas'; @@ -47,6 +48,7 @@ type ErasureMethods = Pick< | 'deleteSchedulesByUser' | 'deleteUserChatProjects' | 'deleteUserToolFavorites' + | 'cancelUserToolFavoriteDeletion' | 'deleteRefreshTokenBridges' | 'deleteAllAgentQueuedTurnsForUser' >; @@ -99,10 +101,76 @@ export interface UserErasureParams { userId: string; tenantId?: string; checkpointer?: TCheckpointerConfig; + /** UUID owned by this deletion attempt, distinct from the Mongo ownership timestamp. */ + deletionAttemptId?: string; /** Operator option to keep the transaction ledger; account self-deletion always erases it. */ keepTransactions?: boolean; } +type UserDeletionCancellationParams = { + userId: string; + startedAt: Date; + deletionAttemptId?: string; +}; +type UserDeletionCancellationDeps = Pick< + AllMethods, + | 'isAgentTriggerUserDeletionOwner' + | 'cancelAgentTriggerUserDeletion' + | 'cancelUserToolFavoriteDeletion' +>; + +export interface UserDeletionAttempt { + readonly id: string; + adopt( + userId: string, + startedAt: Date, + deps: Pick, + ): Promise; + commit(operation: () => Promise): Promise; + cancel(params: UserDeletionCancellationParams, deps: UserDeletionCancellationDeps): Promise; +} + +/** A started user delete may still commit after an error or a lost acknowledgement. */ +export function createUserDeletionAttempt(): UserDeletionAttempt { + let commitStarted = false; + let identityKnown = true; + let attemptId: string = randomUUID(); + return { + get id() { + return attemptId; + }, + async adopt(userId, startedAt, deps) { + identityKnown = false; + const stored = await deps.getAgentTriggerUserDeletionAttemptId(userId, startedAt); + if (stored === null) throw new Error('User deletion attempt identity unavailable'); + attemptId = stored; + identityKnown = true; + }, + async commit(operation: () => Promise): Promise { + commitStarted = true; + return operation(); + }, + async cancel(params, deps) { + if (!commitStarted && identityKnown) await cancelUserDataDeletion(params, deps); + }, + }; +} + +/** Keep Mongo ownership retryable until this attempt's favorites fence is released. */ +export async function cancelUserDataDeletion( + params: UserDeletionCancellationParams, + deps: UserDeletionCancellationDeps, +): Promise { + const { userId, startedAt, deletionAttemptId } = params; + if (deletionAttemptId !== undefined && deps.cancelUserToolFavoriteDeletion) { + if ((await deps.isAgentTriggerUserDeletionOwner(userId, startedAt, deletionAttemptId)) !== true) + return; + await deps.cancelUserToolFavoriteDeletion(userId, deletionAttemptId); + } + if (deletionAttemptId === undefined) await deps.cancelAgentTriggerUserDeletion(userId, startedAt); + else await deps.cancelAgentTriggerUserDeletion(userId, startedAt, deletionAttemptId); +} + export type MCPServerCleanupDeps = Pick< UserErasureDeps, | 'findUserErasableMCPServers' @@ -307,7 +375,9 @@ export async function eraseUserData( deps.deleteAllUserPluginAuths(userId), deps.deleteToolCalls(userId), deps.deleteConversationTags({ user: userId }), - deps.deleteUserToolFavorites(userId), + params.deletionAttemptId === undefined + ? deps.deleteUserToolFavorites(userId) + : deps.deleteUserToolFavorites(userId, params.deletionAttemptId), deps.deleteAllUserMemories(userId), ]); await deps.deleteSharedLinks(userId); diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts index be16c9923c..732bfef7d2 100644 --- a/packages/api/src/user/resolver.spec.ts +++ b/packages/api/src/user/resolver.spec.ts @@ -102,6 +102,8 @@ describe('createErasureResolver', () => { const { deps } = createDeps({ db: { deleteUserAgents: jest.fn(record('agents')), + deleteUserToolFavorites: jest.fn(record('favorites')), + cancelUserToolFavoriteDeletion: jest.fn(record('cancelFavorites')), findMCPServerOAuthTokenUserIds: jest.fn(record('tokens')), findMCPServerViewerEntries: jest.fn(record('viewers')), findMCPServerViewerUserIds: jest.fn(record('viewerUsers')), @@ -118,6 +120,8 @@ describe('createErasureResolver', () => { await tenantStorage.run({ tenantId: 'admin-tenant' }, async () => { await erasure.deleteUserAgents('user-1'); + await erasure.deleteUserToolFavorites('user-1', 'attempt'); + await erasure.cancelUserToolFavoriteDeletion?.('user-1', 'attempt'); await erasure.findMCPServerOAuthTokenUserIds?.('srv'); await erasure.findMCPServerViewerEntries?.('srv-id'); await erasure.findMCPServerViewerUserIds?.([]); @@ -130,7 +134,7 @@ describe('createErasureResolver', () => { await erasure.mcpOAuth?.disconnectUser?.('other', 'srv'); }); - expect(Object.keys(seen)).toHaveLength(10); + expect(Object.keys(seen)).toHaveLength(12); expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true); }); diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts index c4d918060a..98bac2ed8b 100644 --- a/packages/api/src/user/resolver.ts +++ b/packages/api/src/user/resolver.ts @@ -106,6 +106,16 @@ export function createErasureResolver(deps: ErasureResolverDeps) { ...deps.db, openCheckpointDeletion: deps.openCheckpointDeletion, deleteSharedLinks: deps.deleteSharedLinks, + deleteUserToolFavorites: (userId, deletionAttemptId) => + inTargetTenant(async () => + deletionAttemptId === undefined + ? deps.db.deleteUserToolFavorites(userId) + : deps.db.deleteUserToolFavorites(userId, deletionAttemptId), + ), + cancelUserToolFavoriteDeletion: (userId, deletionAttemptId) => + inTargetTenant(async () => + deps.db.cancelUserToolFavoriteDeletion?.(userId, deletionAttemptId), + ), deleteUserAgents: (userId) => inTargetTenant(async () => deps.db.deleteUserAgents(userId)), findMCPServerOAuthTokenUserIds: (serverName) => inTargetTenant(async () => deps.db.findMCPServerOAuthTokenUserIds(serverName)), diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 0267c5f93e..a4594ca454 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -179,6 +179,7 @@ export type DeploymentOnlyConfigReason = 'startup' | 'operator' | 'unused'; * a stored override had no effect rather than a dangerous one. */ export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly> = { + persistence: 'operator', 'telemetry.stackTraceLimit': 'operator', 'telemetry.rum': 'operator', clientLimits: 'operator', @@ -4666,7 +4667,35 @@ export const toolCallPreviewsConfigSchema = z.object({ export type TToolCallPreviewsConfig = z.infer; +export const persistenceConfigSchema = z + .object({ + favorites: z.enum(['mongo', 'postgres']).default('mongo'), + postgres: z + .object({ + connectionStringEnv: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/), + max: z.number().int().min(1).max(100).default(10), + maxWaiting: z.number().int().min(1).max(1000).default(100), + acquireTimeoutMs: z.number().int().min(1).max(60000).default(5000), + queryTimeoutMs: z.number().int().min(1).max(300000).default(15000), + idleTimeoutMs: z.number().int().min(1).max(300000).default(30000), + }) + .strict() + .optional(), + }) + .strict() + .default({ favorites: 'mongo' }) + .superRefine((config, context) => { + if (config.favorites === 'postgres' && !config.postgres) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['postgres'], + message: 'PostgreSQL configuration is required', + }); + } + }); + export const configSchema = z.object({ + persistence: persistenceConfigSchema, version: z.string(), telemetry: telemetrySchema.optional(), clientLimits: clientLimitsSchema.optional(), diff --git a/packages/data-provider/src/persistence.spec.ts b/packages/data-provider/src/persistence.spec.ts new file mode 100644 index 0000000000..160dd49b8e --- /dev/null +++ b/packages/data-provider/src/persistence.spec.ts @@ -0,0 +1,33 @@ +import { configSchema, persistenceConfigSchema, DEPLOYMENT_ONLY_CONFIG_PATHS } from './config'; + +describe('deployment persistence configuration', () => { + it('defaults existing deployments to Mongo and keeps selection deployment-only', () => { + expect(configSchema.parse({ version: '1.3.1' }).persistence).toEqual({ favorites: 'mongo' }); + expect(DEPLOYMENT_ONLY_CONFIG_PATHS.persistence).toBe('operator'); + }); + it('accepts bounded pool settings and an environment-variable name, never a DSN', () => { + const config = persistenceConfigSchema.parse({ + favorites: 'postgres', + postgres: { connectionStringEnv: 'LIBRECHAT_POSTGRES_DSN' }, + }); + expect(config.postgres).toMatchObject({ + max: 10, + maxWaiting: 100, + acquireTimeoutMs: 5000, + queryTimeoutMs: 15000, + }); + expect( + persistenceConfigSchema.safeParse({ + favorites: 'postgres', + postgres: { connectionStringEnv: 'postgres://secret' }, + }).success, + ).toBe(false); + }); + it.each([ + { favorites: 'postgres' }, + { favorites: 'invalid' }, + { favorites: 'postgres', postgres: { connectionStringEnv: 'PG', max: 0 } }, + ])('refuses an explicit invalid selection: %j', (config) => { + expect(persistenceConfigSchema.safeParse(config).success).toBe(false); + }); +}); diff --git a/packages/data-schemas/jest.postgres.config.mjs b/packages/data-schemas/jest.postgres.config.mjs new file mode 100644 index 0000000000..fd34ec2806 --- /dev/null +++ b/packages/data-schemas/jest.postgres.config.mjs @@ -0,0 +1,9 @@ +import config from './jest.config.mjs'; + +export default { + ...config, + testMatch: ['/src/postgres/*.spec.ts'], + globalSetup: undefined, + testResultsProcessor: undefined, + maxWorkers: 1, +}; diff --git a/packages/data-schemas/package.json b/packages/data-schemas/package.json index 5d6110bd50..593a59da9e 100644 --- a/packages/data-schemas/package.json +++ b/packages/data-schemas/package.json @@ -47,7 +47,8 @@ "test:quality:favorites": "jest --config jest.favorite.config.mjs --coverage && node ../../scripts/crap.mts coverage/favorites/coverage-final.json src/methods/favorite.ts", "test:contracts:leases": "jest --config jest.lease.config.mjs --coverage=false", "test:mutations:leases": "cd ../.. && stryker run packages/data-schemas/stryker.lease.config.mjs", - "test:quality:leases": "jest --config jest.lease.config.mjs --coverage && node ../../scripts/crap.mts coverage/leases/coverage-final.json src/methods/resourceLock.ts" + "test:quality:leases": "jest --config jest.lease.config.mjs --coverage && node ../../scripts/crap.mts coverage/leases/coverage-final.json src/methods/resourceLock.ts", + "test:postgres": "jest --config jest.postgres.config.mjs --coverage=false --runInBand" }, "repository": { "type": "git", @@ -67,13 +68,15 @@ "micromark-extension-directive": "^3.0.1", "micromark-extension-gfm": "^3.0.0", "micromark-extension-llm-math": "^3.1.0", - "micromark-util-decode-string": "^2.0.0" + "micromark-util-decode-string": "^2.0.0", + "pg": "^8.23.1" }, "devDependencies": { "@stryker-mutator/core": "10.1.1", "@stryker-mutator/jest-runner": "10.1.1", "@types/jest": "^29.5.2", "@types/node": "^24.12.4", + "@types/pg": "^8.23.1", "jest": "^30.2.0", "jest-junit": "^17.0.0", "mongodb-memory-server": "^11.3.0", diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index ffd53bbbeb..9a3381932a 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -47,6 +47,7 @@ export type { } from './methods/response'; export { createMethods, + createToolFavoriteMethods, createChannelIdentityStorage, createChannelFeedbackStorage, ChannelFeedbackAuthorityError, @@ -182,3 +183,10 @@ export type { SearchTokenBackfillResult } from './migrations'; export { createBrowserSetupStore, createBrowserSetupBackend } from './methods/browserSetup'; export type * from './types/browserSetup'; export type * from './types/codeEnvironmentSetup'; + +export * from './postgres/backend'; +export * from './postgres/pool'; +export * from './postgres/migrations'; +export * from './postgres/favorite'; +export * from './postgres/import'; +export * from './postgres/mongo'; diff --git a/packages/data-schemas/src/methods/favorite.ts b/packages/data-schemas/src/methods/favorite.ts index da78591713..41b237ec61 100644 --- a/packages/data-schemas/src/methods/favorite.ts +++ b/packages/data-schemas/src/methods/favorite.ts @@ -19,12 +19,9 @@ function capError(): Error { return error; } -export function createToolFavoriteMethods(mongoose: typeof import('mongoose')): { - getToolFavorites: (userId: string) => Promise; - addToolFavorite: (params: t.ToolFavoriteParams) => Promise; - removeToolFavorite: (params: t.ToolFavoriteParams) => Promise; - deleteUserToolFavorites: (userId: string) => Promise; -} { +export function createToolFavoriteMethods( + mongoose: typeof import('mongoose'), +): t.ToolFavoriteMethods { async function getToolFavorites(userId: string): Promise { const ToolFavorite = mongoose.models.ToolFavorite; return ToolFavorite.find({ user: userId }) @@ -96,4 +93,4 @@ export function createToolFavoriteMethods(mongoose: typeof import('mongoose')): }; } -export type ToolFavoriteMethods = ReturnType; +export type { ToolFavoriteMethods } from '~/types/favorite'; diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index 6bb47a9c52..bc642cd501 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -368,6 +368,7 @@ export type AllMethods = PermissionPrincipalStorage & /** Dependencies injected from the api layer into createMethods */ export interface CreateMethodsDeps { + toolFavorites?: ToolFavoriteMethods; getChannelStorageConfig?: () => Promise>; /** Matches a model name to a canonical key. From @librechat/api. */ matchModelName?: (model: string, endpoint?: string) => string | undefined; @@ -575,7 +576,7 @@ export function createMethods( ...createFileMethods(mongoose), ...createFileStorageMethods(mongoose), ...createMemoryMethods(mongoose), - ...createToolFavoriteMethods(mongoose), + ...(deps.toolFavorites ?? createToolFavoriteMethods(mongoose)), ...createAgentCategoryMethods(mongoose), ...createAgentApiKeyMethods(mongoose), ...createAgentChannelMethods(mongoose, deps.getChannelStorageConfig), @@ -730,3 +731,5 @@ export type { }; export { recordAgentEventActorReceiptMetric, setAgentEventActorReceiptMetricObserver }; + +export { createToolFavoriteMethods } from './favorite'; diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts index cac9a016e8..aa6cf5ee13 100644 --- a/packages/data-schemas/src/methods/user.methods.spec.ts +++ b/packages/data-schemas/src/methods/user.methods.spec.ts @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import { randomUUID } from 'node:crypto'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { CacheKeys, @@ -1556,6 +1557,111 @@ describe('User Methods - Database Tests', () => { expect(found).toBeNull(); }); + test.each(['cancel', 'commit'] as const)( + 'rejects an old queued delete after an offline handoff and %s', + async (outcome) => { + const user = await User.create({ + email: `delayed-${outcome}@example.com`, + provider: 'local', + }); + const id = user._id.toString(); + const attemptId = randomUUID(); + const startedAt = new Date('2026-01-01'); + const recoveredAt = new Date(startedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1); + await methods.beginAgentTriggerUserDeletion(id, startedAt, attemptId); + let release!: () => void; + let entered!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const pendingWrite = new Promise((resolve) => { + entered = resolve; + }); + const original = User.collection.deleteOne.bind(User.collection); + const write = jest + .spyOn(User.collection, 'deleteOne') + .mockImplementationOnce(async (filter, options) => { + entered(); + await gate; + return original(filter, options); + }); + const pendingDelete = methods.deleteUserById(id, { startedAt, attemptId }); + try { + await pendingWrite; + expect(write.mock.calls[0][0]).toMatchObject({ + agentTriggerDeletionStartedAt: startedAt, + agentTriggerDeletionAttemptId: attemptId, + }); + expect( + await methods.recoverStaleAgentTriggerUserDeletion(id, recoveredAt, randomUUID()), + ).toBe('acquired'); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, recoveredAt)).toBe( + attemptId, + ); + if (outcome === 'cancel') { + expect(await methods.cancelAgentTriggerUserDeletion(id, recoveredAt, attemptId)).toBe( + true, + ); + } else { + expect( + await methods.deleteUserById(id, { startedAt: recoveredAt, attemptId }), + ).toMatchObject({ deletedCount: 1 }); + } + release(); + expect(await pendingDelete).toMatchObject({ deletedCount: 0 }); + expect(await User.exists({ _id: id })).toEqual( + outcome === 'cancel' ? { _id: user._id } : null, + ); + } finally { + release(); + await pendingDelete; + write.mockRestore(); + } + }, + ); + + test('rejects the wrong UUID or Date and commits only the current owner', async () => { + const user = await User.create({ email: 'fenced-delete@example.com', provider: 'local' }); + const id = user._id.toString(); + const attemptId = randomUUID(); + const startedAt = new Date(); + await methods.beginAgentTriggerUserDeletion(id, startedAt, attemptId); + expect( + await methods.deleteUserById(id, { startedAt, attemptId: randomUUID() }), + ).toMatchObject({ deletedCount: 0 }); + expect( + await methods.deleteUserById(id, { + startedAt: new Date(startedAt.getTime() + 1), + attemptId, + }), + ).toMatchObject({ deletedCount: 0 }); + expect(await methods.isAgentTriggerUserDeletionOwner(id, startedAt, attemptId)).toBe(true); + expect(await methods.deleteUserById(id, { startedAt, attemptId })).toMatchObject({ + deletedCount: 1, + }); + expect( + await methods.recoverStaleAgentTriggerUserDeletion( + id, + new Date(startedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1), + randomUUID(), + ), + ).toBe('missing'); + }); + + test('rejects invalid commit fences before any cleanup or delete', async () => { + const id = new mongoose.Types.ObjectId().toString(); + const write = jest.spyOn(User, 'deleteOne'); + const grants = jest.spyOn(mongoose.models.ToolApprovalGrant, 'deleteMany'); + await expect( + methods.deleteUserById(id, { startedAt: new Date(Number.NaN), attemptId: randomUUID() }), + ).rejects.toThrow('startedAt must be a valid Date'); + await expect( + methods.deleteUserById(id, { startedAt: new Date(), attemptId: '' }), + ).rejects.toThrow('Invalid user deletion attempt ID'); + expect(write).not.toHaveBeenCalled(); + expect(grants).not.toHaveBeenCalled(); + }); + test('should return zero count for non-existent user', async () => { const fakeId = new mongoose.Types.ObjectId(); const result = await methods.deleteUserById(fakeId.toString()); @@ -1590,6 +1696,88 @@ describe('User Methods - Database Tests', () => { await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true); }); + test('proves the exact live owner from primary without releasing its fence', async () => { + const user = await User.create({ email: 'deletion-owner@example.com', provider: 'local' }); + const userId = user._id.toString(); + const startedAt = new Date('2026-11-10T12:00:00.000Z'); + await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(false); + await methods.beginAgentTriggerUserDeletion(userId, startedAt); + const read = jest.spyOn(mongoose.Query.prototype, 'read'); + await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(true); + expect(read).toHaveBeenCalledWith('primary'); + await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false); + await expect( + methods.isAgentTriggerUserDeletionOwner(userId, new Date(startedAt.getTime() + 1)), + ).resolves.toBe(false); + await methods.deleteUserById(userId); + await expect(methods.isAgentTriggerUserDeletionOwner(userId, startedAt)).resolves.toBe(false); + }); + + test('propagates ownership-read errors instead of treating them as absent ownership', async () => { + const failure = new Error('ownership read unavailable'); + jest.spyOn(User.collection, 'findOne').mockRejectedValueOnce(failure); + await expect( + methods.isAgentTriggerUserDeletionOwner( + new mongoose.Types.ObjectId().toString(), + new Date(), + ), + ).rejects.toThrow(failure); + }); + + test('persists identity across repeated offline recoveries and fences stale Date and UUID owners', async () => { + const user = await User.create({ email: 'durable-attempt@example.com', provider: 'local' }); + const id = user._id.toString(); + const attempt = randomUUID(); + const other = randomUUID(); + const first = new Date('2026-01-01'); + const second = new Date(first.getTime() + USER_DELETION_FENCE_STALE_MS + 1); + const third = new Date(second.getTime() + USER_DELETION_FENCE_STALE_MS + 1); + expect(await methods.beginAgentTriggerUserDeletion(id, first, attempt)).toBe('acquired'); + expect((await User.findById(id).lean())?.agentTriggerDeletionAttemptId).toBeUndefined(); + expect(await methods.recoverStaleAgentTriggerUserDeletion(id, second, other)).toBe( + 'acquired', + ); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, first)).toBeNull(); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBe(attempt); + expect(await methods.cancelAgentTriggerUserDeletion(id, first, attempt)).toBe(false); + expect(await methods.cancelAgentTriggerUserDeletion(id, second, other)).toBe(false); + expect(await methods.isAgentTriggerUserDeletionOwner(id, second, other)).toBe(false); + // Recovery can itself crash before adopting the identity; the next process still finds it. + expect(await methods.recoverStaleAgentTriggerUserDeletion(id, third, randomUUID())).toBe( + 'acquired', + ); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, third)).toBe(attempt); + expect(await methods.cancelAgentTriggerUserDeletion(id, third, attempt)).toBe(true); + expect( + await User.findById(id) + .select('+agentTriggerDeletionAttemptId +agentTriggerDeletionStartedAt') + .lean(), + ).toEqual(expect.not.objectContaining({ agentTriggerDeletionAttemptId: expect.anything() })); + expect(await methods.beginAgentTriggerUserDeletion(id, third, other)).toBe('acquired'); + expect(await methods.cancelAgentTriggerUserDeletion(id, third, attempt)).toBe(false); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, third)).toBe(other); + }); + test('initializes only a legacy missing UUID and clears stray identity on a no-token acquisition', async () => { + const user = await User.create({ email: 'legacy-attempt@example.com', provider: 'local' }); + const id = user._id.toString(); + const first = new Date('2026-01-01'); + const second = new Date(first.getTime() + USER_DELETION_FENCE_STALE_MS + 1); + const fallback = randomUUID(); + await methods.beginAgentTriggerUserDeletion(id, first); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, first)).toBeNull(); + expect(await methods.recoverStaleAgentTriggerUserDeletion(id, second, fallback)).toBe( + 'acquired', + ); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBe(fallback); + await methods.cancelAgentTriggerUserDeletion(id, second, fallback); + await User.updateOne({ _id: id }, { $set: { agentTriggerDeletionAttemptId: randomUUID() } }); + await methods.beginAgentTriggerUserDeletion(id, second); + expect(await methods.getAgentTriggerUserDeletionAttemptId(id, second)).toBeNull(); + await expect(methods.beginAgentTriggerUserDeletion(id, second, '')).rejects.toThrow( + 'attempt', + ); + }); + test('reports a missing principal without creating a deletion fence', async () => { const userId = new mongoose.Types.ObjectId().toString(); diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts index bc2caa057f..52378bc0d5 100644 --- a/packages/data-schemas/src/methods/user.ts +++ b/packages/data-schemas/src/methods/user.ts @@ -183,18 +183,33 @@ export function createUserMethods( beginAgentTriggerUserDeletion: ( userId: string, startedAt: Date, + attemptId?: string, ) => Promise<'acquired' | 'in_progress' | 'missing'>; recoverStaleAgentTriggerUserDeletion: ( userId: string, recoveredAt: Date, + legacyAttemptId?: string, ) => Promise<'acquired' | 'in_progress' | 'missing'>; - cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise; + getAgentTriggerUserDeletionAttemptId: (userId: string, startedAt: Date) => Promise; + isAgentTriggerUserDeletionOwner: ( + userId: string, + startedAt: Date, + attemptId?: string, + ) => Promise; + cancelAgentTriggerUserDeletion: ( + userId: string, + startedAt: Date, + attemptId?: string, + ) => Promise; isAgentTriggerPrincipalActive: (userId: string) => Promise; fenceSubagentAdmission: (userId: string, token: string, fencedUntil: Date) => Promise; renewSubagentAdmission: (userId: string, token: string, fencedUntil: Date) => Promise; releaseSubagentAdmission: (userId: string, token: string) => Promise; isSubagentOwnerAdmissible: (userId: string) => Promise; - deleteUserById: (userId: string) => Promise; + deleteUserById: ( + userId: string, + fence?: { startedAt: Date; attemptId: string }, + ) => Promise; updateUserPlugins: ( userId: string, plugins: string[] | undefined, @@ -645,7 +660,16 @@ export function createUserMethods( /** * Delete a user by their unique ID. */ - async function deleteUserById(userId: string): Promise { + async function deleteUserById( + userId: string, + fence?: { startedAt: Date; attemptId: string }, + ): Promise { + if (fence !== undefined) { + if (!(fence?.startedAt instanceof Date) || !Number.isFinite(fence.startedAt.getTime())) { + throw new TypeError('startedAt must be a valid Date'); + } + validateDeletionAttemptId(fence.attemptId); + } try { const User = mongoose.models.User; const owner = await User.findById(userId).select('tenantId').lean>(); @@ -664,7 +688,15 @@ export function createUserMethods( ); } await mongoose.models.ToolApprovalGrant?.deleteMany({ user: userId }); - const result = await User.deleteOne({ _id: userId }); + const result = await User.deleteOne({ + _id: userId, + ...(fence === undefined + ? {} + : { + agentTriggerDeletionStartedAt: fence.startedAt, + agentTriggerDeletionAttemptId: fence.attemptId, + }), + }); if (result.deletedCount === 0) { return { deletedCount: 0, message: 'No user found with that ID.' }; } @@ -676,18 +708,32 @@ export function createUserMethods( } } + function validateDeletionAttemptId(attemptId: string): void { + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(attemptId)) { + throw new TypeError('Invalid user deletion attempt ID'); + } + } + /** Establishes the durable admission fence used while account deletion drains triggers. */ async function beginAgentTriggerUserDeletion( userId: string, startedAt: Date, + attemptId?: string, ): Promise<'acquired' | 'in_progress' | 'missing'> { if (!(startedAt instanceof Date) || !Number.isFinite(startedAt.getTime())) { throw new TypeError('startedAt must be a valid Date'); } + if (attemptId !== undefined) validateDeletionAttemptId(attemptId); const User = mongoose.models.User; const result = await User.updateOne( { _id: userId, agentTriggerDeletionStartedAt: { $exists: false } }, - { $set: { agentTriggerDeletionStartedAt: startedAt } }, + { + $set: { + agentTriggerDeletionStartedAt: startedAt, + ...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }), + }, + ...(attemptId === undefined ? { $unset: { agentTriggerDeletionAttemptId: 1 } } : {}), + }, ); if (result.modifiedCount === 1) { await invalidateAuthUserDocCache(userId); @@ -701,12 +747,34 @@ export function createUserMethods( async function recoverStaleAgentTriggerUserDeletion( userId: string, recoveredAt: Date, + legacyAttemptId?: string, ): Promise<'acquired' | 'in_progress' | 'missing'> { if (!(recoveredAt instanceof Date) || !Number.isFinite(recoveredAt.getTime())) { throw new TypeError('recoveredAt must be a valid Date'); } const User = mongoose.models.User; + if (legacyAttemptId !== undefined) validateDeletionAttemptId(legacyAttemptId); const staleBefore = new Date(recoveredAt.getTime() - USER_DELETION_FENCE_STALE_MS); + // Two ordinary CAS updates keep DocumentDB compatibility and never overwrite a stored UUID. + if (legacyAttemptId !== undefined) { + const legacy = await User.updateOne( + { + _id: userId, + agentTriggerDeletionStartedAt: { $lte: staleBefore }, + agentTriggerDeletionAttemptId: { $exists: false }, + }, + { + $set: { + agentTriggerDeletionStartedAt: recoveredAt, + agentTriggerDeletionAttemptId: legacyAttemptId, + }, + }, + ); + if (legacy.modifiedCount === 1) { + await invalidateAuthUserDocCache(userId); + return 'acquired'; + } + } const result = await User.updateOne( { _id: userId, agentTriggerDeletionStartedAt: { $lte: staleBefore } }, { $set: { agentTriggerDeletionStartedAt: recoveredAt } }, @@ -718,12 +786,55 @@ export function createUserMethods( return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress'; } + async function getAgentTriggerUserDeletionAttemptId( + userId: string, + startedAt: Date, + ): Promise { + const user = await mongoose.models.User.findOne({ + _id: userId, + agentTriggerDeletionStartedAt: startedAt, + }) + .select({ agentTriggerDeletionAttemptId: 1, _id: 0 }) + .read('primary') + .lean>(); + const attemptId = user?.agentTriggerDeletionAttemptId; + if (attemptId == null) return null; + validateDeletionAttemptId(attemptId); + return attemptId; + } + + /** Reads current ownership without releasing the durable retry anchor. */ + async function isAgentTriggerUserDeletionOwner( + userId: string, + startedAt: Date, + attemptId?: string, + ): Promise { + if (attemptId !== undefined) validateDeletionAttemptId(attemptId); + const User = mongoose.models.User; + return ( + (await User.exists({ + _id: userId, + agentTriggerDeletionStartedAt: startedAt, + ...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }), + }).read('primary')) != null + ); + } + /** Releases only the account-deletion attempt that owns this exact fence. */ - async function cancelAgentTriggerUserDeletion(userId: string, startedAt: Date): Promise { + async function cancelAgentTriggerUserDeletion( + userId: string, + startedAt: Date, + attemptId?: string, + ): Promise { + if (attemptId !== undefined) validateDeletionAttemptId(attemptId); const User = mongoose.models.User; const result = await User.updateOne( - { _id: userId, agentTriggerDeletionStartedAt: startedAt }, - { $unset: { agentTriggerDeletionStartedAt: 1 } }, + { + _id: userId, + agentTriggerDeletionStartedAt: startedAt, + ...(attemptId === undefined ? {} : { agentTriggerDeletionAttemptId: attemptId }), + }, + { $unset: { agentTriggerDeletionStartedAt: 1, agentTriggerDeletionAttemptId: 1 } }, ); if (result.modifiedCount === 1) { await invalidateAuthUserDocCache(userId); @@ -1088,6 +1199,8 @@ export function createUserMethods( generateToken, beginAgentTriggerUserDeletion, recoverStaleAgentTriggerUserDeletion, + getAgentTriggerUserDeletionAttemptId, + isAgentTriggerUserDeletionOwner, cancelAgentTriggerUserDeletion, isAgentTriggerPrincipalActive, fenceSubagentAdmission, diff --git a/packages/data-schemas/src/postgres/backend.spec.ts b/packages/data-schemas/src/postgres/backend.spec.ts new file mode 100644 index 0000000000..5df9f54e75 --- /dev/null +++ b/packages/data-schemas/src/postgres/backend.spec.ts @@ -0,0 +1,117 @@ +import mongoose from 'mongoose'; +import type { PostgresDatabase, PostgresOptions } from './pool'; +import type { PostgresFavoriteBackend } from './backend'; +import { createPostgresTestDatabase } from './postgres.helper'; +import { createPostgresFavoriteBackend } from './backend'; +import { migratePostgres } from './migrations'; +import { importFavorites } from './import'; + +const postgres = process.env.PG_TEST_URL ? describe : describe.skip; +const userId = '0123456789abcdef01234567'; + +postgres('PostgreSQL favorite backend composition', () => { + let db: PostgresDatabase | undefined; + let backend: PostgresFavoriteBackend | undefined; + let options: PostgresOptions; + + beforeEach(async () => { + ({ db, options } = await createPostgresTestDatabase()); + await migratePostgres(db); + }); + + afterEach(async () => { + try { + await backend?.close(); + } finally { + backend = undefined; + await db?.close(); + db = undefined; + } + }); + + async function verifyEmpty(): Promise { + await importFavorites( + db!, + async function* () { + /* verified empty source */ + }, + { offline: true, batchSize: 1, verifiedEmpty: true }, + ); + } + + async function readiness() { + return (await db!.query('SELECT * FROM tool_favorite_readiness')).rows; + } + + it('does not activate verified data when constructed or read before initialization', async () => { + await verifyEmpty(); + const before = await readiness(); + expect(before).toEqual([expect.objectContaining({ verified: true, activated: false })]); + backend = createPostgresFavoriteBackend(options); + expect(await backend.methods.getToolFavorites(userId)).toEqual([]); + expect(await readiness()).toEqual(before); + }); + + it('initializes verified-empty data by activating readiness without touching Mongo', async () => { + await verifyEmpty(); + const before = await readiness(); + const connect = jest.spyOn(mongoose, 'connect'); + const collection = jest.spyOn(mongoose.connection, 'collection'); + const model = jest.spyOn(mongoose, 'model'); + backend = createPostgresFavoriteBackend(options); + await backend.initialize(); + expect(await readiness()).toEqual([{ ...before[0], activated: true }]); + expect(await backend.methods.getToolFavorites(userId)).toEqual([]); + expect(connect).not.toHaveBeenCalled(); + expect(collection).not.toHaveBeenCalled(); + expect(model).not.toHaveBeenCalled(); + }); + + it('rejects unverified data without activating it', async () => { + backend = createPostgresFavoriteBackend(options); + await expect(backend.initialize()).rejects.toThrow('PostgreSQL favorites data not verified'); + expect(await readiness()).toEqual([ + expect.objectContaining({ verified: false, activated: false }), + ]); + }); + + it('exposes the native favorite add, read, and removal contract', async () => { + await verifyEmpty(); + backend = createPostgresFavoriteBackend(options); + await backend.initialize(); + const favorite = { userId, itemType: 'tool', itemId: 'backend-tool' } as const; + await expect(backend.methods.addToolFavorite(favorite)).resolves.toEqual({ + ok: true, + added: true, + }); + await expect(backend.methods.getToolFavorites(userId)).resolves.toEqual([ + { itemType: 'tool', itemId: 'backend-tool' }, + ]); + expect((await db!.query('SELECT count(*) FROM tool_favorites')).rows).toEqual([{ count: '1' }]); + await expect(backend.methods.removeToolFavorite(favorite)).resolves.toEqual({ + ok: true, + removed: true, + }); + await expect(backend.methods.getToolFavorites(userId)).resolves.toEqual([]); + }); + + it('closes idempotently and rejects methods and initialization after close', async () => { + await verifyEmpty(); + backend = createPostgresFavoriteBackend(options); + await backend.initialize(); + const closing = backend.close(); + expect(backend.close()).toBe(closing); + await closing; + await expect(backend.close()).resolves.toBeUndefined(); + await expect(backend.methods.getToolFavorites(userId)).rejects.toThrow('admission'); + await expect(backend.initialize()).rejects.toThrow('admission'); + }); + + it('can be closed idempotently after initialization fails', async () => { + backend = createPostgresFavoriteBackend(options); + await expect(backend.initialize()).rejects.toThrow('not verified'); + await expect(backend.close()).resolves.toBeUndefined(); + await expect(backend.close()).resolves.toBeUndefined(); + await expect(backend.methods.getToolFavorites(userId)).rejects.toThrow('admission'); + }); +}); diff --git a/packages/data-schemas/src/postgres/backend.ts b/packages/data-schemas/src/postgres/backend.ts new file mode 100644 index 0000000000..89c983bd53 --- /dev/null +++ b/packages/data-schemas/src/postgres/backend.ts @@ -0,0 +1,20 @@ +import type { ToolFavoriteMethods } from '~/types/favorite'; +import type { PostgresOptions } from './pool'; +import { createPostgresFavoriteMethods } from './favorite'; +import { assertPostgresReady } from './migrations'; +import { createPostgresDatabase } from './pool'; + +export interface PostgresFavoriteBackend { + methods: ToolFavoriteMethods; + initialize(): Promise; + close(): Promise; +} + +export function createPostgresFavoriteBackend(options: PostgresOptions): PostgresFavoriteBackend { + const db = createPostgresDatabase(options); + return { + methods: createPostgresFavoriteMethods(db), + initialize: () => assertPostgresReady(db), + close: () => db.close(), + }; +} diff --git a/packages/data-schemas/src/postgres/favorite.spec.ts b/packages/data-schemas/src/postgres/favorite.spec.ts new file mode 100644 index 0000000000..4c36a38fc7 --- /dev/null +++ b/packages/data-schemas/src/postgres/favorite.spec.ts @@ -0,0 +1,303 @@ +import mongoose from 'mongoose'; +import { randomUUID } from 'node:crypto'; +import type { FavoriteHarness } from '~/contracts/favorite'; +import type { FavoriteSnapshotRow } from './import'; +import type { PostgresDatabase } from './pool'; +import type { PostgresOptions } from './pool'; +import { importFavorites, readPostgresFavorites, fingerprintFavorites } from './import'; +import { tenantStorage, SYSTEM_TENANT_ID } from '~/config/tenantContext'; +import { migratePostgres, assertPostgresReady } from './migrations'; +import { describeFavoriteConformance } from '~/contracts/favorite'; +import { createPostgresTestDatabase } from './postgres.helper'; +import { createPostgresFavoriteMethods } from './favorite'; +import { resetTenantStrictCache } from '~/tenant/policy'; +import { createPostgresDatabase } from './pool'; + +const postgres = process.env.PG_TEST_URL ? describe : describe.skip; +let db: PostgresDatabase; +const users = ['0123456789abcdef01234567', '1123456789abcdef01234567'] as const; +let options: PostgresOptions; +const harness: FavoriteHarness = { + name: 'PostgreSQL 16', + users, + async setup() { + ({ db, options } = await createPostgresTestDatabase()); + await migratePostgres(db); + return createPostgresFavoriteMethods(db); + }, + async teardown() { + await db?.close(); + }, + async reset() { + jest.restoreAllMocks(); + await db.query('TRUNCATE tool_favorites,tool_favorite_erasure'); + await db.query( + 'UPDATE tool_favorite_readiness SET checkpoint=NULL, snapshot=NULL, verified=false, activated=false, imported=0', + ); + }, + async seed(rows) { + for (const row of rows) { + await db.query( + `INSERT INTO tool_favorites (id,user_id,item_type,item_id,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$5)`, + [ + new mongoose.Types.ObjectId().toString(), + row.userId, + row.itemType, + row.itemId, + row.createdAt ?? new Date(), + ], + ); + } + }, + inTenant: (tenantId, operation) => tenantStorage.run({ tenantId }, operation), + failNext(operation, error) { + if (operation === 'read' || operation === 'remove') { + jest.spyOn(db, 'query').mockRejectedValueOnce(error); + } else { + jest.spyOn(db, 'transaction').mockRejectedValueOnce(error); + } + }, +}; + +postgres('real PostgreSQL favorites', () => { + describeFavoriteConformance(harness); + describe('adapter, migrations, import, and readiness', () => { + beforeAll(() => harness.setup()); + afterAll(() => harness.teardown()); + beforeEach(() => harness.reset()); + const source = async function* () { + yield { + id: '2123456789abcdef01234567', + userId: users[0], + tenantId: 'tenant-a', + itemType: 'mcp', + itemId: 'server', + createdAt: new Date('2026-01-01'), + updatedAt: new Date('2026-01-02'), + } satisfies FavoriteSnapshotRow; + }; + it('keeps schema migration idempotent, checksum-checked, and separate from data readiness', async () => { + await migratePostgres(db); + await expect(assertPostgresReady(db)).rejects.toThrow('not verified'); + await db.query("UPDATE librechat_migrations SET checksum='changed'"); + await expect(migratePostgres(db)).rejects.toThrow('checksum'); + // Restore through the exact original checksum, not by rerunning altered SQL. + await db.query('DELETE FROM librechat_migrations'); + await db.query('DROP TABLE tool_favorites,tool_favorite_erasure,tool_favorite_readiness'); + await migratePostgres(db); + }); + it('requires verified-empty approval, then activates and permanently refuses import replay', async () => { + const empty = async function* () { + /* verified empty source */ + }; + await expect(importFavorites(db, empty, { offline: true, batchSize: 2 })).rejects.toThrow( + 'verified-empty', + ); + await importFavorites(db, empty, { offline: true, batchSize: 2, verifiedEmpty: true }); + await assertPostgresReady(db); + await expect( + importFavorites(db, empty, { offline: true, batchSize: 2, verifiedEmpty: true }), + ).rejects.toThrow('activated'); + }); + it('preserves source identity, tenant, timestamps, and supports idempotent checkpoint replay', async () => { + await importFavorites(db, source, { offline: true, batchSize: 1 }); + await importFavorites(db, source, { offline: true, batchSize: 1 }); + expect(await fingerprintFavorites(readPostgresFavorites(db, 1))).toEqual( + await fingerprintFavorites(source), + ); + await assertPostgresReady(db); + }); + it('does not write in dry-run and refuses a false empty-source assertion', async () => { + await importFavorites(db, source, { offline: true, batchSize: 1, dryRun: true }); + expect((await db.query('SELECT * FROM tool_favorites')).rowCount).toBe(0); + await expect( + importFavorites(db, source, { offline: true, batchSize: 1, verifiedEmpty: true }), + ).rejects.toThrow('not empty'); + }); + it('rolls back a failed transaction', async () => { + await expect( + db.transaction(async (sql) => { + await sql.query("INSERT INTO tool_favorite_erasure VALUES ($1,'*')", [users[0]]); + throw new Error('fault'); + }), + ).rejects.toThrow('fault'); + expect((await db.query('SELECT * FROM tool_favorite_erasure')).rowCount).toBe(0); + }); + it('denies missing strict context and distinguishes system all-tenants from tenantless', async () => { + const methods = createPostgresFavoriteMethods(db); + await harness.inTenant('tenant-a', () => + methods.addToolFavorite({ userId: users[0], itemType: 'skill', itemId: 'a' }), + ); + expect( + await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () => + methods.getToolFavorites(users[0]), + ), + ).toHaveLength(1); + const old = process.env.TENANT_ISOLATION_STRICT; + process.env.TENANT_ISOLATION_STRICT = 'true'; + resetTenantStrictCache(); + try { + await expect(methods.getToolFavorites(users[0])).rejects.toThrow('without tenant context'); + } finally { + resetTenantStrictCache(); + if (old == null) { + delete process.env.TENANT_ISOLATION_STRICT; + } else { + process.env.TENANT_ISOLATION_STRICT = old; + } + } + }); + it('accepts ObjectId input and prevents late recreation after erasure', async () => { + const methods = createPostgresFavoriteMethods(db); + const params = { + userId: new mongoose.Types.ObjectId(users[0]), + itemType: 'tool', + itemId: 'x', + } as const; + await methods.addToolFavorite(params); + await methods.deleteUserToolFavorites(users[0]); + await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure'); + }); + it('cancels only its UUID and scope, never another attempt or a permanent marker', async () => { + const methods = createPostgresFavoriteMethods(db); + const first = randomUUID(); + const second = randomUUID(); + const params = { userId: users[0], itemType: 'tool', itemId: 'retry' } as const; + await harness.inTenant('tenant-a', async () => { + await methods.deleteUserToolFavorites(users[0], first); + await methods.deleteUserToolFavorites(users[0], second); + }); + await harness.inTenant('tenant-b', () => + methods.cancelUserToolFavoriteDeletion!(users[0], first), + ); + await harness.inTenant('tenant-a', async () => { + await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure'); + await methods.cancelUserToolFavoriteDeletion!(users[0], first); + await methods.cancelUserToolFavoriteDeletion!(users[0], first); + await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure'); + await methods.cancelUserToolFavoriteDeletion!(users[0], second); + await expect(methods.addToolFavorite(params)).resolves.toEqual({ ok: true, added: true }); + await methods.deleteUserToolFavorites(users[0]); + await methods.cancelUserToolFavoriteDeletion!(users[0], first); + await expect(methods.addToolFavorite(params)).rejects.toThrow('erasure'); + }); + await expect(methods.cancelUserToolFavoriteDeletion!(users[0], '')).rejects.toThrow( + 'attempt', + ); + await expect(methods.deleteUserToolFavorites(users[0], '')).rejects.toThrow('attempt'); + }); + it('lets system recovery cancel its UUID across scopes without clearing successors or permanent markers', async () => { + const methods = createPostgresFavoriteMethods(db); + const owned = randomUUID(); + const successor = randomUUID(); + for (const tenantId of ['tenant-a', 'tenant-b', SYSTEM_TENANT_ID]) { + await tenantStorage.run({ tenantId }, () => + methods.deleteUserToolFavorites(users[0], owned), + ); + } + await harness.inTenant('tenant-b', () => + methods.deleteUserToolFavorites(users[0], successor), + ); + await methods.deleteUserToolFavorites(users[1]); + await harness.inTenant('tenant-a', () => + methods.cancelUserToolFavoriteDeletion!(users[0], owned), + ); + expect( + ( + await db.query( + 'SELECT scope FROM tool_favorite_erasure WHERE user_id=$1 AND attempt_id=$2 ORDER BY scope', + [users[0], owned], + ) + ).rows, + ).toEqual([{ scope: '*' }, { scope: 'tenant:tenant-b' }]); + await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () => + methods.cancelUserToolFavoriteDeletion!(users[0], owned), + ); + expect( + ( + await db.query( + 'SELECT scope,attempt_id FROM tool_favorite_erasure WHERE user_id=$1 ORDER BY scope', + [users[0]], + ) + ).rows, + ).toEqual([{ scope: 'tenant:tenant-b', attempt_id: successor }]); + expect( + ( + await db.query('SELECT attempt_id FROM tool_favorite_erasure WHERE user_id=$1', [ + users[1], + ]) + ).rows, + ).toEqual([{ attempt_id: '' }]); + await harness.inTenant('tenant-b', () => + expect( + methods.addToolFavorite({ userId: users[0], itemType: 'tool', itemId: 'still-fenced' }), + ).rejects.toThrow('erasure'), + ); + }); + it('keeps global markers fenced against tenant cancellation and across fresh adapters', async () => { + const methods = createPostgresFavoriteMethods(db); + const attempt = randomUUID(); + const params = { userId: users[0], itemType: 'tool', itemId: 'global' } as const; + await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () => + methods.deleteUserToolFavorites(users[0], attempt), + ); + await harness.inTenant('tenant-a', () => + methods.cancelUserToolFavoriteDeletion!(users[0], attempt), + ); + await harness.inTenant('tenant-b', () => + expect(createPostgresFavoriteMethods(db).addToolFavorite(params)).rejects.toThrow( + 'erasure', + ), + ); + await tenantStorage.run({ tenantId: SYSTEM_TENANT_ID }, () => + methods.cancelUserToolFavoriteDeletion!(users[0], attempt), + ); + await expect(methods.addToolFavorite(params)).resolves.toEqual({ ok: true, added: true }); + }); + it('bounds concurrent cap admission and enforces uniqueness across tenants', async () => { + await harness.seed( + Array.from({ length: 99 }, (_, i) => ({ + userId: users[0], + itemType: 'tool', + itemId: `${i}`, + })), + ); + const methods = createPostgresFavoriteMethods(db); + await Promise.allSettled( + Array.from({ length: 8 }, (_, i) => + methods.addToolFavorite({ userId: users[0], itemType: 'tool', itemId: `race-${i}` }), + ), + ); + expect(await methods.getToolFavorites(users[0])).toHaveLength(100); + }); + it('bounds query duration, releases failed transactions, and rejects work after close', async () => { + const bounded = createPostgresDatabase({ ...options, queryTimeoutMs: 50 }); + try { + await expect( + bounded.transaction((sql) => sql.query('SELECT pg_sleep(1)')), + ).rejects.toThrow(); + expect((await bounded.query('SELECT 1 AS value')).rows).toEqual([{ value: 1 }]); + } finally { + await bounded.close(); + } + await expect(bounded.query('SELECT 1')).rejects.toThrow('admission'); + }); + it('refuses concurrent import writers using the pinned session lock', async () => { + const other = createPostgresDatabase(options); + try { + await other.session(async (sql) => { + await sql.query('SELECT pg_advisory_lock(17490261, hashtext(current_schema()))'); + try { + await expect( + importFavorites(db, source, { offline: true, batchSize: 1 }), + ).rejects.toThrow('Another'); + } finally { + await sql.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))'); + } + }); + } finally { + await other.close(); + } + }); + }); +}); diff --git a/packages/data-schemas/src/postgres/favorite.ts b/packages/data-schemas/src/postgres/favorite.ts new file mode 100644 index 0000000000..d79f5c4c96 --- /dev/null +++ b/packages/data-schemas/src/postgres/favorite.ts @@ -0,0 +1,121 @@ +import { randomBytes } from 'node:crypto'; +import type { ToolFavoriteMethods, ToolFavoriteParams } from '~/types/favorite'; +import type { PostgresDatabase, SqlExecutor } from './pool'; +import { resolveTenantScope, tenantFilter, stampTenantOnDocument } from '~/tenant/policy'; +import { MAX_TOOL_FAVORITES } from '~/methods/favorite'; + +function userKey(value: ToolFavoriteParams['userId']): string { + const user = String(value).toLowerCase(); + if (!/^[0-9a-f]{24}$/.test(user)) { + throw new Error('Invalid favorite user ID'); + } + return user; +} + +function validateAttemptId(attemptId: string): void { + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(attemptId)) { + throw new Error('Invalid favorite deletion attempt ID'); + } +} + +export function createPostgresFavoriteMethods(db: PostgresDatabase): ToolFavoriteMethods { + function scope(): string | null { + return tenantFilter(resolveTenantScope('Query'))?.tenantId ?? null; + } + async function lock(sql: SqlExecutor, user: string): Promise { + await sql.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [user]); + } + return { + async getToolFavorites(userId) { + const result = await db.query<{ itemType: ToolFavoriteParams['itemType']; itemId: string }>( + `SELECT item_type AS "itemType", item_id AS "itemId" FROM tool_favorites + WHERE user_id = $1 AND ($2::text IS NULL OR tenant_id = $2) ORDER BY created_at, id`, + [userKey(userId), scope()], + ); + return result.rows; + }, + async addToolFavorite({ userId, itemType, itemId }) { + const user = userKey(userId); + const tenant = scope(); + const document: { tenantId?: string } = {}; + stampTenantOnDocument(resolveTenantScope('Query'), document); + if ( + !['builtin', 'tool', 'mcp', 'skill'].includes(itemType) || + typeof itemId !== 'string' || + itemId.length < 1 || + itemId.length > 256 + ) { + throw new Error('Invalid favorite item'); + } + return db.transaction(async (sql) => { + await lock(sql, user); + const erased = await sql.query( + 'SELECT 1 FROM tool_favorite_erasure WHERE user_id = $1 AND (scope = $2 OR scope = $3) LIMIT 1', + [user, '*', tenant == null ? '*' : `tenant:${tenant}`], + ); + if (erased.rowCount) { + throw new Error('Favorite account erasure in progress'); + } + const exists = await sql.query( + 'SELECT 1 FROM tool_favorites WHERE user_id=$1 AND item_type=$2 AND item_id=$3 AND ($4::text IS NULL OR tenant_id=$4)', + [user, itemType, itemId, tenant], + ); + if (exists.rowCount) { + return { ok: true, added: false }; + } + const count = await sql.query<{ count: string }>( + 'SELECT count(*) FROM tool_favorites WHERE user_id=$1 AND ($2::text IS NULL OR tenant_id=$2)', + [user, tenant], + ); + if (Number(count.rows[0].count) >= MAX_TOOL_FAVORITES) { + throw Object.assign(new Error(`Maximum of ${MAX_TOOL_FAVORITES} favorites reached`), { + code: 'MAX_FAVORITES_EXCEEDED', + limit: MAX_TOOL_FAVORITES, + }); + } + await sql.query( + `INSERT INTO tool_favorites (id,user_id,tenant_id,item_type,item_id,created_at,updated_at) + VALUES ($1,$2,$3,$4,$5,now(),now()) ON CONFLICT (user_id,item_type,item_id) DO NOTHING`, + [randomBytes(12).toString('hex'), user, document.tenantId ?? null, itemType, itemId], + ); + return { ok: true, added: true }; + }); + }, + async removeToolFavorite({ userId, itemType, itemId }) { + const result = await db.query( + 'DELETE FROM tool_favorites WHERE user_id=$1 AND item_type=$2 AND item_id=$3 AND ($4::text IS NULL OR tenant_id=$4)', + [userKey(userId), itemType, itemId, scope()], + ); + return { ok: true, removed: (result.rowCount ?? 0) > 0 }; + }, + async cancelUserToolFavoriteDeletion(userId, deletionAttemptId) { + validateAttemptId(deletionAttemptId); + const user = userKey(userId); + const tenant = scope(); + await db.transaction(async (sql) => { + await lock(sql, user); + await sql.query( + 'DELETE FROM tool_favorite_erasure WHERE user_id=$1 AND ($2::text IS NULL OR scope=$2) AND attempt_id=$3', + [user, tenant == null ? null : `tenant:${tenant}`, deletionAttemptId], + ); + }); + }, + async deleteUserToolFavorites(userId, deletionAttemptId) { + if (deletionAttemptId !== undefined) validateAttemptId(deletionAttemptId); + const user = userKey(userId); + const tenant = scope(); + return db.transaction(async (sql) => { + await lock(sql, user); + await sql.query( + 'INSERT INTO tool_favorite_erasure (user_id,scope,attempt_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING', + [user, tenant == null ? '*' : `tenant:${tenant}`, deletionAttemptId ?? ''], + ); + const result = await sql.query( + 'DELETE FROM tool_favorites WHERE user_id=$1 AND ($2::text IS NULL OR tenant_id=$2)', + [user, tenant], + ); + return result.rowCount ?? 0; + }); + }, + }; +} diff --git a/packages/data-schemas/src/postgres/import.spec.ts b/packages/data-schemas/src/postgres/import.spec.ts new file mode 100644 index 0000000000..a570d980e7 --- /dev/null +++ b/packages/data-schemas/src/postgres/import.spec.ts @@ -0,0 +1,369 @@ +import type { QueryResultRow } from 'pg'; +import type { FavoriteSnapshotRow, FavoriteSnapshotSource } from './import'; +import type { PostgresDatabase, SqlExecutor } from './pool'; +import { fingerprintFavorites, importFavorites, readPostgresFavorites } from './import'; +import { createPostgresTestDatabase } from './postgres.helper'; +import { migratePostgres } from './migrations'; + +const postgres = process.env.PG_TEST_URL ? describe : describe.skip; +const userId = '0123456789abcdef01234567'; +const options = { offline: true, batchSize: 2 }; +const rows: FavoriteSnapshotRow[] = Array.from({ length: 5 }, (_, i) => ({ + id: (i + 1).toString(16).padStart(24, '0'), + userId, + tenantId: i % 2 ? null : 'tenant-a', + itemType: 'mcp', + itemId: `server-${i}`, + createdAt: new Date('2026-01-01T01:02:03.123Z'), + updatedAt: new Date('2026-01-02T04:05:06.789Z'), +})); +const source = (data = rows): FavoriteSnapshotSource => + async function* () { + yield* data; + }; + +postgres('real PostgreSQL snapshot recovery', () => { + let db: PostgresDatabase; + beforeAll(async () => { + ({ db } = await createPostgresTestDatabase()); + await migratePostgres(db); + }); + afterAll(async () => db?.close()); + beforeEach(async () => { + await db.query('TRUNCATE tool_favorites,tool_favorite_erasure'); + await db.query( + 'UPDATE tool_favorite_readiness SET checkpoint=NULL,snapshot=NULL,verified=false,activated=false,imported=0', + ); + }); + + function interruptAfter(prefix: string, occurrence: number): PostgresDatabase { + let remaining = occurrence; + return { + ...db, + session: (operation) => + db.session((session) => + operation({ + ...session, + transaction: (work) => + session.transaction((sql) => + work({ + async query(text: string, values?: unknown[]) { + const result = await sql.query(text, values); + if (text.startsWith(prefix) && --remaining === 0) { + throw new Error('Injected interruption'); + } + return result; + }, + } satisfies SqlExecutor), + ), + }), + ), + }; + } + + async function state() { + return ( + await db.query( + 'SELECT checkpoint,snapshot,verified,activated,imported FROM tool_favorite_readiness', + ) + ).rows[0]; + } + + it.each([ + ['INSERT INTO tool_favorites', 1, 0], + ['INSERT INTO tool_favorites', 2, 2], + ['UPDATE tool_favorite_readiness SET checkpoint', 2, 2], + ])( + 'rolls back rows and checkpoint after %s interruption %i, then resumes', + async (prefix, occurrence, committed) => { + await expect( + importFavorites(interruptAfter(prefix, occurrence), source(), options), + ).rejects.toThrow('Injected'); + expect((await db.query('SELECT id FROM tool_favorites ORDER BY id')).rows).toEqual( + rows.slice(0, committed).map(({ id }) => ({ id })), + ); + expect(await state()).toMatchObject({ + checkpoint: committed ? rows[committed - 1].id : null, + verified: false, + imported: '0', + }); + await importFavorites(db, source(), options); + await importFavorites(db, source(), { ...options, batchSize: 3 }); + expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual( + await fingerprintFavorites(source()), + ); + expect(await state()).toMatchObject({ + checkpoint: rows[4].id, + verified: true, + imported: '5', + }); + }, + ); + + it('resumes when the source iterator fails after a committed batch', async () => { + let pass = 0; + const interrupted: FavoriteSnapshotSource = async function* () { + pass++; + for (const row of rows) { + yield row; + if (pass === 2 && row.id === rows[1].id) throw new Error('Source interrupted'); + } + }; + await expect(importFavorites(db, interrupted, options)).rejects.toThrow('Source interrupted'); + expect(await state()).toMatchObject({ checkpoint: rows[1].id, verified: false }); + expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2); + await importFavorites(db, source(), options); + expect(await state()).toMatchObject({ verified: true, imported: '5' }); + }); + + it('keeps verification false when the source changes during the final verification pass', async () => { + let pass = 0; + const changing: FavoriteSnapshotSource = () => + source(++pass === 3 ? rows.map((row) => ({ ...row, tenantId: 'changed' })) : rows)(); + await expect(importFavorites(db, changing, options)).rejects.toThrow('verification failed'); + expect(await state()).toMatchObject({ verified: false }); + expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual( + await fingerprintFavorites(source()), + ); + }); + + it.each([false, true])( + 'survives a final source scan beyond the idle transaction timeout (changed: %s)', + async (changed) => { + let pass = 0; + const slow: FavoriteSnapshotSource = async function* () { + if (++pass === 3) { + await new Promise((resolve) => setTimeout(resolve, 300)); + yield* changed ? rows.map((row) => ({ ...row, tenantId: 'changed' })) : rows; + return; + } + yield* rows; + }; + const shortTimeout: PostgresDatabase = { + ...db, + session: (operation) => + db.session(async (session) => { + await session.query("SET idle_in_transaction_session_timeout = '100ms'"); + try { + return await operation(session); + } finally { + await session.query('RESET idle_in_transaction_session_timeout'); + } + }), + }; + if (changed) { + await expect(importFavorites(shortTimeout, slow, options)).rejects.toThrow( + 'verification failed', + ); + } else { + await expect(importFavorites(shortTimeout, slow, options)).resolves.toEqual( + await fingerprintFavorites(source()), + ); + } + expect(pass).toBe(3); + expect(await state()).toMatchObject({ verified: !changed, imported: changed ? '0' : '5' }); + expect(await fingerprintFavorites(readPostgresFavorites(db, 2))).toEqual( + await fingerprintFavorites(source()), + ); + }, + ); + + it('holds the pinned migration lock without a transaction during every source pass', async () => { + let pid = 0; + let passes = 0; + const pinned: PostgresDatabase = { + ...db, + session: (operation) => + db.session(async (session) => { + pid = (await session.query<{ pid: number }>('SELECT pg_backend_pid() AS pid')).rows[0] + .pid; + return operation(session); + }), + }; + const checking: FavoriteSnapshotSource = async function* () { + passes++; + expect( + (await db.query('SELECT state FROM pg_stat_activity WHERE pid=$1', [pid])).rows, + ).toEqual([{ state: 'idle' }]); + await expect(importFavorites(db, source(), options)).rejects.toThrow( + 'Another favorite migration or import is running', + ); + yield* rows; + }; + await importFavorites(pinned, checking, options); + expect(passes).toBe(3); + expect(await state()).toMatchObject({ verified: true, imported: '5' }); + await importFavorites(db, source(), options); + }); + + it.each(['activated', 'snapshot'])( + 'refuses %s authority change between batches', + async (change) => { + let pass = 0; + const changing: FavoriteSnapshotSource = async function* () { + pass++; + for (const row of rows) { + yield row; + if (pass !== 2 || row.id !== rows[1].id) continue; + if (change === 'activated') + await db.query('UPDATE tool_favorite_readiness SET activated=true'); + else await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'"); + } + }; + await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed'); + expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2); + expect(await state()).toMatchObject({ checkpoint: rows[1].id, verified: false }); + }, + ); + + it('preserves canonical identity, tenant and millisecond timestamps exactly', async () => { + await importFavorites(db, source(), options); + const actual: FavoriteSnapshotRow[] = []; + for await (const row of readPostgresFavorites(db, 2)()) actual.push(row); + expect(actual).toEqual(rows); + }); + + it('refuses a changed fingerprint on resume without modifying committed rows', async () => { + await expect( + importFavorites(interruptAfter('INSERT INTO tool_favorites', 2), source(), options), + ).rejects.toThrow('Injected'); + const before = await state(); + const changed = rows.map((row, i) => (i ? row : { ...row, tenantId: 'changed' })); + await expect(importFavorites(db, source(changed), options)).rejects.toThrow( + 'different snapshot', + ); + expect(await state()).toEqual(before); + expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(2); + }); + + it('refuses duplicate logical favorites instead of silently checkpointing them', async () => { + const duplicate = { ...rows[1], itemId: rows[0].itemId }; + await expect(importFavorites(db, source([rows[0], duplicate]), options)).rejects.toThrow( + 'conflicts', + ); + expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0); + expect(await state()).toMatchObject({ checkpoint: null, verified: false }); + }); + + it('refuses a destination mapping collision and preserves unrelated rows', async () => { + await db.query('INSERT INTO tool_favorites VALUES ($1,$2,$3,$4,$5,$6,$7)', [ + rows[0].id, + userId, + 'other-tenant', + 'tool', + 'unrelated', + rows[0].createdAt, + rows[0].updatedAt, + ]); + await expect(importFavorites(db, source(), options)).rejects.toThrow('conflicts'); + expect((await db.query('SELECT tenant_id,item_id FROM tool_favorites')).rows).toEqual([ + { tenant_id: 'other-tenant', item_id: 'unrelated' }, + ]); + expect(await state()).toMatchObject({ checkpoint: null, verified: false }); + }); + + it('detects a corrupted checkpoint prefix without overwriting it', async () => { + await expect( + importFavorites(interruptAfter('INSERT INTO tool_favorites', 2), source(), options), + ).rejects.toThrow('Injected'); + await db.query('UPDATE tool_favorites SET tenant_id=$1 WHERE id=$2', ['wrong', rows[0].id]); + await expect(importFavorites(db, source(), options)).rejects.toThrow('verification failed'); + expect(await state()).toMatchObject({ verified: false }); + expect( + (await db.query('SELECT tenant_id FROM tool_favorites WHERE id=$1', [rows[0].id])).rows[0] + .tenant_id, + ).toBe('wrong'); + }); + + it.each(['activated', 'snapshot', 'missing'])( + 'refuses %s authority change after the last batch', + async (change) => { + let pass = 0; + const changing: FavoriteSnapshotSource = async function* () { + pass++; + yield* rows.slice(0, 2); + if (pass !== 2) return; + if (change === 'activated') + await db.query('UPDATE tool_favorite_readiness SET activated=true'); + if (change === 'snapshot') + await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'"); + if (change === 'missing') await db.query('DELETE FROM tool_favorite_readiness'); + }; + try { + await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed'); + if (change !== 'missing') expect(await state()).toMatchObject({ verified: false }); + } finally { + if (change === 'missing') + await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)'); + } + }, + ); + + it.each(['activated', 'snapshot', 'missing'])( + 'refuses %s authority change during the final source pass', + async (change) => { + let pass = 0; + const changing: FavoriteSnapshotSource = async function* () { + if (++pass === 3) { + if (change === 'activated') + await db.query('UPDATE tool_favorite_readiness SET activated=true'); + if (change === 'snapshot') + await db.query("UPDATE tool_favorite_readiness SET snapshot='changed'"); + if (change === 'missing') await db.query('DELETE FROM tool_favorite_readiness'); + } + yield* rows; + }; + try { + await expect(importFavorites(db, changing, options)).rejects.toThrow('authority changed'); + if (change !== 'missing') expect(await state()).toMatchObject({ verified: false }); + } finally { + if (change === 'missing') + await db.query('INSERT INTO tool_favorite_readiness(singleton) VALUES(true)'); + } + }, + ); + + it('refuses a changed middle source pass even when the final source would match', async () => { + let pass = 0; + const changing: FavoriteSnapshotSource = () => source(++pass === 2 ? rows.slice(0, 2) : rows)(); + await expect(importFavorites(db, changing, options)).rejects.toThrow('source changed'); + expect(await state()).toMatchObject({ verified: false }); + }); + + it('refuses unsorted rows in the write pass', async () => { + let pass = 0; + const changing: FavoriteSnapshotSource = () => + source(++pass === 2 ? [rows[1], rows[0]] : rows)(); + await expect(importFavorites(db, changing, options)).rejects.toThrow('uniquely ordered'); + expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0); + }); + + it.each([ + { userId: 'not-canonical' }, + { id: 'bad-id' }, + { itemType: 'unknown' }, + { itemId: 42 }, + { itemId: '' }, + { tenantId: 12 }, + { createdAt: new Date('invalid') }, + { updatedAt: 'not-a-date' }, + ])('refuses malformed input before claiming a snapshot: %p', async (invalid) => { + const malformed = { ...rows[0], ...invalid } as FavoriteSnapshotRow; + await expect(importFavorites(db, source([malformed]), options)).rejects.toThrow( + 'Invalid favorite snapshot record', + ); + expect(await state()).toMatchObject({ snapshot: null, checkpoint: null, verified: false }); + expect((await db.query('SELECT id FROM tool_favorites')).rowCount).toBe(0); + }); + + it('refuses duplicate IDs before claiming a snapshot', async () => { + await expect(importFavorites(db, source([rows[0], rows[0]]), options)).rejects.toThrow( + 'uniquely ordered', + ); + expect(await state()).toMatchObject({ snapshot: null }); + }); + + it('rejects a zero-sized PostgreSQL reader rather than fingerprinting a false empty source', () => { + expect(() => readPostgresFavorites(db, 0)).toThrow('batch size'); + }); +}); diff --git a/packages/data-schemas/src/postgres/import.ts b/packages/data-schemas/src/postgres/import.ts new file mode 100644 index 0000000000..8e64501cac --- /dev/null +++ b/packages/data-schemas/src/postgres/import.ts @@ -0,0 +1,243 @@ +import { createHash } from 'node:crypto'; +import type { PostgresDatabase, PostgresSession, SqlExecutor } from './pool'; +import type { FavoriteItemType } from '~/types/favorite'; +import { lockFavoriteMigration } from './migrations'; + +export interface FavoriteSnapshotRow { + id: string; + userId: string; + tenantId: string | null; + itemType: FavoriteItemType; + itemId: string; + createdAt: Date; + updatedAt: Date; +} +export type FavoriteSnapshotSource = () => AsyncIterable; + +function encoded(row: FavoriteSnapshotRow): string { + if ( + typeof row.id !== 'string' || + !/^[0-9a-f]{24}$/.test(row.id) || + typeof row.userId !== 'string' || + !/^[0-9a-f]{24}$/.test(row.userId) || + !['builtin', 'tool', 'mcp', 'skill'].includes(row.itemType) || + typeof row.itemId !== 'string' || + !row.itemId || + row.itemId.length > 256 || + (row.tenantId !== null && typeof row.tenantId !== 'string') || + !(row.createdAt instanceof Date) || + !Number.isFinite(row.createdAt.getTime()) || + !(row.updatedAt instanceof Date) || + !Number.isFinite(row.updatedAt.getTime()) + ) { + throw new Error('Invalid favorite snapshot record'); + } + return JSON.stringify([ + row.id, + row.userId, + row.tenantId, + row.itemType, + row.itemId, + row.createdAt.toISOString(), + row.updatedAt.toISOString(), + ]); +} + +export async function fingerprintFavorites( + source: FavoriteSnapshotSource, +): Promise<{ count: number; digest: string }> { + const hash = createHash('sha256'); + let count = 0; + let previous = ''; + for await (const row of source()) { + if (row.id <= previous) { + throw new Error('Favorite snapshot must be uniquely ordered by ID'); + } + hash.update(encoded(row)).update('\n'); + previous = row.id; + count++; + } + return { count, digest: hash.digest('hex') }; +} + +export function readPostgresFavorites(db: SqlExecutor, batchSize: number): FavoriteSnapshotSource { + if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1000) { + throw new Error('Favorite snapshot batch size must be between 1 and 1000'); + } + return async function* () { + let after = ''; + for (;;) { + const result = await db.query( + `SELECT id, user_id AS "userId", tenant_id AS "tenantId", + item_type AS "itemType", item_id AS "itemId", created_at AS "createdAt", updated_at AS "updatedAt" + FROM tool_favorites WHERE id > $1 ORDER BY id LIMIT $2`, + [after, batchSize], + ); + if (!result.rows.length) { + return; + } + for (const row of result.rows) { + yield row; + after = row.id; + } + } + }; +} + +/** Source and destination writers must be stopped. Activation permanently blocks replay. */ +interface ImportOptions { + offline: boolean; + batchSize: number; + dryRun?: boolean; + verifiedEmpty?: boolean; +} + +export async function importFavorites( + db: PostgresDatabase, + source: FavoriteSnapshotSource, + options: ImportOptions, +): Promise<{ count: number; digest: string }> { + return db.session(async (session) => { + const lock = await session.query<{ acquired: boolean }>( + 'SELECT pg_try_advisory_lock(17490261, hashtext(current_schema())) AS acquired', + ); + if (!lock.rows[0].acquired) { + throw new Error('Another favorite migration or import is running'); + } + try { + return await importSnapshot(session, source, options); + } finally { + await session.query('SELECT pg_advisory_unlock(17490261, hashtext(current_schema()))'); + } + }); +} + +async function importSnapshot( + db: PostgresSession, + source: FavoriteSnapshotSource, + options: ImportOptions, +): Promise<{ count: number; digest: string }> { + if ( + !options.offline || + !Number.isInteger(options.batchSize) || + options.batchSize < 1 || + options.batchSize > 1000 + ) { + throw new Error('Offline assertion and batch size between 1 and 1000 are required'); + } + const snapshot = await fingerprintFavorites(source); + if (snapshot.count === 0 && !options.verifiedEmpty) { + throw new Error('Empty source requires explicit verified-empty approval'); + } + if (options.verifiedEmpty && snapshot.count !== 0) { + throw new Error('Verified-empty refused: Mongo favorites are not empty'); + } + if (options.dryRun) { + return snapshot; + } + await db.transaction(async (sql) => { + await lockFavoriteMigration(sql); + const result = await sql.query<{ activated: boolean; snapshot: string | null }>( + 'SELECT activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE', + ); + if ( + !result.rows[0] || + result.rows[0].activated || + (result.rows[0].snapshot && result.rows[0].snapshot !== snapshot.digest) + ) { + throw new Error('Favorite import cannot replace an activated or different snapshot'); + } + await sql.query( + 'UPDATE tool_favorite_readiness SET snapshot=$1, verified=false WHERE singleton', + [snapshot.digest], + ); + }); + let batch: FavoriteSnapshotRow[] = []; + const write = async () => { + await db.transaction(async (sql) => { + await lockFavoriteMigration(sql); + const state = await sql.query<{ + checkpoint: string | null; + activated: boolean; + snapshot: string; + }>( + 'SELECT checkpoint, activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE', + ); + if (!state.rows[0] || state.rows[0].activated || state.rows[0].snapshot !== snapshot.digest) { + throw new Error('Favorite import authority changed'); + } + const pending = batch.filter((row) => row.id > (state.rows[0].checkpoint ?? '')); + if (pending.length) { + const inserted = await sql.query( + `INSERT INTO tool_favorites (id,user_id,tenant_id,item_type,item_id,created_at,updated_at) + SELECT * FROM unnest($1::text[],$2::text[],$3::text[],$4::text[],$5::text[],$6::timestamptz[],$7::timestamptz[]) + ON CONFLICT DO NOTHING`, + [ + pending.map((row) => row.id), + pending.map((row) => row.userId), + pending.map((row) => row.tenantId), + pending.map((row) => row.itemType), + pending.map((row) => row.itemId), + pending.map((row) => row.createdAt), + pending.map((row) => row.updatedAt), + ], + ); + if (inserted.rowCount !== pending.length) { + throw new Error('Favorite snapshot conflicts with destination mapping'); + } + } + await sql.query( + "UPDATE tool_favorite_readiness SET checkpoint=GREATEST(COALESCE(checkpoint,''),$1) WHERE singleton", + [batch[batch.length - 1].id], + ); + }); + batch = []; + }; + const written = createHash('sha256'); + let count = 0; + let previous = ''; + for await (const row of source()) { + if (row.id <= previous) { + throw new Error('Favorite snapshot must be uniquely ordered by ID'); + } + written.update(encoded(row)).update('\n'); + previous = row.id; + count++; + batch.push(row); + if (batch.length === options.batchSize) { + await write(); + } + } + if (batch.length) { + await write(); + } + if (count !== snapshot.count || written.digest('hex') !== snapshot.digest) { + throw new Error('Favorite source changed during import'); + } + const current = await fingerprintFavorites(source); + await db.transaction(async (sql) => { + await lockFavoriteMigration(sql); + const state = await sql.query<{ activated: boolean; snapshot: string | null }>( + 'SELECT activated, snapshot FROM tool_favorite_readiness WHERE singleton FOR UPDATE', + ); + if (!state.rows[0] || state.rows[0].activated || state.rows[0].snapshot !== snapshot.digest) { + throw new Error('Favorite import authority changed'); + } + const destination = await fingerprintFavorites(readPostgresFavorites(sql, options.batchSize)); + if ( + current.digest !== snapshot.digest || + destination.digest !== snapshot.digest || + destination.count !== snapshot.count + ) { + throw new Error('Favorite snapshot verification failed'); + } + const verified = await sql.query( + 'UPDATE tool_favorite_readiness SET verified=true, imported=$1 WHERE singleton AND NOT activated AND snapshot=$2', + [snapshot.count, snapshot.digest], + ); + if (verified.rowCount !== 1) { + throw new Error('Favorite import authority changed'); + } + }); + return snapshot; +} diff --git a/packages/data-schemas/src/postgres/migrations.spec.ts b/packages/data-schemas/src/postgres/migrations.spec.ts new file mode 100644 index 0000000000..4665e0751f --- /dev/null +++ b/packages/data-schemas/src/postgres/migrations.spec.ts @@ -0,0 +1,98 @@ +import { randomBytes } from 'node:crypto'; +import type { PostgresDatabase } from './pool'; +import { migratePostgres, assertPostgresReady } from './migrations'; +import { createPostgresTestDatabase } from './postgres.helper'; +import { importFavorites } from './import'; + +const postgres = process.env.PG_TEST_URL ? describe : describe.skip; +postgres('PostgreSQL catalog readiness', () => { + let db: PostgresDatabase; + let schema: string; + beforeAll(async () => { + ({ db, schema } = await createPostgresTestDatabase()); + }); + afterAll(() => db?.close()); + beforeEach(async () => { + await db.query( + 'DROP TABLE IF EXISTS tool_favorites,tool_favorite_erasure,tool_favorite_readiness,librechat_migrations', + ); + await migratePostgres(db); + await importFavorites(db, async function* () {}, { + offline: true, + batchSize: 10, + verifiedEmpty: true, + }); + }); + it.each([ + 'DROP INDEX tool_favorites_user_created', + 'ALTER TABLE tool_favorites DROP CONSTRAINT tool_favorites_user_id_item_type_item_id_key', + 'ALTER TABLE tool_favorites DROP CONSTRAINT tool_favorites_item_id_check', + 'ALTER TABLE tool_favorites ALTER COLUMN item_id DROP NOT NULL', + 'ALTER TABLE tool_favorites ENABLE ROW LEVEL SECURITY', + 'DROP TABLE tool_favorite_erasure', + ])('refuses schema drift before activation: %s', async (statement) => { + await db.query(statement); + await expect(assertPostgresReady(db)).rejects.toThrow('schema'); + expect((await db.query('SELECT activated FROM tool_favorite_readiness')).rows).toEqual([ + { activated: false }, + ]); + }); + it('supports concurrent idempotent migration runners', async () => { + await Promise.all([migratePostgres(db), migratePostgres(db), migratePostgres(db)]); + expect((await db.query('SELECT version FROM librechat_migrations')).rows).toEqual([ + { version: 1 }, + ]); + await assertPostgresReady(db); + }); + it('does not turn nested work into a premature commit', async () => { + await expect( + db.session(async (session) => + session.transaction(async (sql) => { + await sql.query("INSERT INTO tool_favorite_erasure VALUES ('nested','*')"); + await session.transaction(async () => {}); + }), + ), + ).rejects.toThrow('Nested'); + expect((await db.query('SELECT * FROM tool_favorite_erasure')).rowCount).toBe(0); + }); + it('validates runtime DML privileges rather than requiring migration-owner permissions', async () => { + const role = `favorites_role_${randomBytes(8).toString('hex')}`; + await db.query(`DROP ROLE IF EXISTS ${role}`); + await db.query(`CREATE ROLE ${role}`); + try { + await db.query(`GRANT USAGE ON SCHEMA ${schema} TO ${role}`); + await db.query(`GRANT SELECT ON librechat_migrations TO ${role}`); + await db.query( + `GRANT SELECT, INSERT, DELETE ON tool_favorites,tool_favorite_erasure,tool_favorite_readiness TO ${role}`, + ); + await db.query(`GRANT UPDATE ON tool_favorite_readiness TO ${role}`); + const runAsRole = (operation: (database: PostgresDatabase) => Promise) => + db.session(async (session) => { + await session.query(`SET ROLE ${role}`); + try { + const selected: PostgresDatabase = { + ...db, + query: session.query, + transaction: session.transaction, + }; + await operation(selected); + } finally { + await session.query('RESET ROLE'); + } + }); + await runAsRole((database) => assertPostgresReady(database)); + await db.query(`REVOKE DELETE ON tool_favorite_erasure FROM ${role}`); + await runAsRole(async (database) => { + await expect(assertPostgresReady(database)).rejects.toThrow('permissions'); + }); + await db.query(`GRANT DELETE ON tool_favorite_erasure TO ${role}`); + await db.query(`REVOKE INSERT ON tool_favorites FROM ${role}`); + await runAsRole(async (database) => { + await expect(assertPostgresReady(database)).rejects.toThrow('permissions'); + }); + } finally { + await db.query(`DROP OWNED BY ${role}`); + await db.query(`DROP ROLE ${role}`); + } + }); +}); diff --git a/packages/data-schemas/src/postgres/migrations.ts b/packages/data-schemas/src/postgres/migrations.ts new file mode 100644 index 0000000000..989a0bef8d --- /dev/null +++ b/packages/data-schemas/src/postgres/migrations.ts @@ -0,0 +1,158 @@ +import { createHash } from 'node:crypto'; +import type { PostgresDatabase, SqlExecutor } from './pool'; + +const migration = { + version: 1, + sql: `CREATE TABLE tool_favorites ( + id text PRIMARY KEY, + user_id text NOT NULL CHECK (user_id ~ '^[0-9a-f]{24}$'), + tenant_id text, + item_type text NOT NULL CHECK (item_type IN ('builtin', 'tool', 'mcp', 'skill')), + item_id text NOT NULL CHECK (char_length(item_id) BETWEEN 1 AND 256), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + UNIQUE (user_id, item_type, item_id) + ); + CREATE INDEX tool_favorites_user_created ON tool_favorites (user_id, created_at, id); + CREATE INDEX tool_favorites_tenant_user ON tool_favorites (tenant_id, user_id); + CREATE TABLE tool_favorite_erasure ( + user_id text NOT NULL, + scope text NOT NULL, + attempt_id text NOT NULL DEFAULT '', + PRIMARY KEY (user_id, scope, attempt_id) + ); + CREATE TABLE tool_favorite_readiness ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + checkpoint text, + verified boolean NOT NULL DEFAULT false, + activated boolean NOT NULL DEFAULT false, + snapshot text, + imported bigint NOT NULL DEFAULT 0 + ); + INSERT INTO tool_favorite_readiness (singleton) VALUES (true);`, +}; +const checksum = createHash('sha256').update(migration.sql).digest('hex'); + +export async function lockFavoriteMigration(sql: SqlExecutor): Promise { + await sql.query('SELECT pg_advisory_xact_lock(17490261, hashtext(current_schema()))'); +} + +async function assertVersion(sql: SqlExecutor): Promise { + const result = await sql.query<{ version: string }>( + "SELECT current_setting('server_version_num') AS version", + ); + if (Number(result.rows[0].version) < 160000 || Number(result.rows[0].version) >= 170000) { + throw new Error('PostgreSQL 16 is required'); + } +} + +const tables = ['tool_favorites', 'tool_favorite_erasure', 'tool_favorite_readiness']; + +/** Compare deployed columns, constraints and indexes, not only the applied-version marker. */ +async function schemaChecksum(sql: SqlExecutor): Promise { + const catalog = await sql.query<{ definition: unknown }>( + `SELECT jsonb_build_object( + 'name', expected.name, 'kind', relation.relkind, + 'rowSecurity', relation.relrowsecurity, 'forceRowSecurity', relation.relforcerowsecurity, + 'columns', (SELECT jsonb_agg(jsonb_build_array(attribute.attname, + format_type(attribute.atttypid, attribute.atttypmod), attribute.attnotnull, + pg_get_expr(defaults.adbin, defaults.adrelid), attribute.attidentity, attribute.attgenerated) + ORDER BY attribute.attnum) + FROM pg_attribute attribute LEFT JOIN pg_attrdef defaults + ON defaults.adrelid=attribute.attrelid AND defaults.adnum=attribute.attnum + WHERE attribute.attrelid=relation.oid AND attribute.attnum>0 AND NOT attribute.attisdropped), + 'constraints', (SELECT jsonb_agg(jsonb_build_array(constraint_name.conname, + pg_get_constraintdef(constraint_name.oid), constraint_name.convalidated) + ORDER BY constraint_name.conname) + FROM pg_constraint constraint_name WHERE constraint_name.conrelid=relation.oid), + 'indexes', (SELECT jsonb_agg(jsonb_build_array(pg_get_indexdef(index_definition.indexrelid), + index_definition.indisvalid, index_definition.indisready) ORDER BY pg_get_indexdef(index_definition.indexrelid)) + FROM pg_index index_definition WHERE index_definition.indrelid=relation.oid), + 'triggers', (SELECT jsonb_agg(pg_get_triggerdef(trigger_definition.oid) + ORDER BY trigger_definition.tgname) FROM pg_trigger trigger_definition + WHERE trigger_definition.tgrelid=relation.oid AND NOT trigger_definition.tgisinternal), + 'policies', (SELECT jsonb_agg(jsonb_build_array(policy.polname, policy.polcmd, + policy.polpermissive, policy.polroles, pg_get_expr(policy.polqual, policy.polrelid), + pg_get_expr(policy.polwithcheck, policy.polrelid)) ORDER BY policy.polname) + FROM pg_policy policy WHERE policy.polrelid=relation.oid) + ) AS definition FROM unnest($1::text[]) AS expected(name) + LEFT JOIN pg_class relation ON relation.oid=to_regclass(expected.name) + ORDER BY expected.name`, + [tables], + ); + return createHash('sha256').update(JSON.stringify(catalog.rows)).digest('hex'); +} + +async function assertSchema(sql: SqlExecutor): Promise { + const applied = await sql.query<{ version: number; checksum: string; schema_checksum: string }>( + 'SELECT version, checksum, schema_checksum FROM librechat_migrations ORDER BY version', + ); + if ( + applied.rows.length !== 1 || + applied.rows[0].version !== migration.version || + applied.rows[0].checksum !== checksum || + applied.rows[0].schema_checksum !== (await schemaChecksum(sql)) + ) { + throw new Error('PostgreSQL schema checksum or version mismatch'); + } + const privileges = await sql.query<{ allowed: boolean }>( + `SELECT has_table_privilege(current_user, 'tool_favorites', 'SELECT') + AND has_table_privilege(current_user, 'tool_favorites', 'INSERT') + AND has_table_privilege(current_user, 'tool_favorites', 'DELETE') + AND has_table_privilege(current_user, 'tool_favorite_erasure', 'SELECT') + AND has_table_privilege(current_user, 'tool_favorite_erasure', 'INSERT') + AND has_table_privilege(current_user, 'tool_favorite_erasure', 'DELETE') + AND has_table_privilege(current_user, 'tool_favorite_readiness', 'SELECT') + AND has_table_privilege(current_user, 'tool_favorite_readiness', 'UPDATE') AS allowed`, + ); + if (!privileges.rows[0]?.allowed) throw new Error('PostgreSQL runtime permissions not ready'); +} + +export async function migratePostgres(db: PostgresDatabase): Promise { + await db.transaction(async (sql) => { + await assertVersion(sql); + await lockFavoriteMigration(sql); + await sql.query(`CREATE TABLE IF NOT EXISTS librechat_migrations ( + version integer PRIMARY KEY, checksum text NOT NULL, schema_checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now() + )`); + const result = await sql.query<{ version: number; checksum: string }>( + 'SELECT version, checksum FROM librechat_migrations ORDER BY version', + ); + if (result.rows.length) { + if ( + result.rows.length !== 1 || + result.rows[0].version !== migration.version || + result.rows[0].checksum !== checksum + ) { + throw new Error('PostgreSQL migration checksum or version mismatch'); + } + await assertSchema(sql); + return; + } + await sql.query(migration.sql); + await sql.query( + 'INSERT INTO librechat_migrations (version, checksum, schema_checksum) VALUES ($1, $2, $3)', + [migration.version, checksum, await schemaChecksum(sql)], + ); + }); +} + +/** Readiness check without selecting an authority or changing its activation marker. */ +export async function assertPostgresDataReady(sql: SqlExecutor): Promise { + await assertVersion(sql); + await assertSchema(sql); + const ready = await sql.query<{ verified: boolean; snapshot: string | null }>( + 'SELECT verified, snapshot FROM tool_favorite_readiness WHERE singleton', + ); + if (ready.rows[0]?.verified !== true || !/^[a-f0-9]{64}$/.test(ready.rows[0].snapshot ?? '')) { + throw new Error('PostgreSQL favorites data not verified'); + } +} + +export async function assertPostgresReady(db: PostgresDatabase): Promise { + await db.transaction(async (sql) => { + await lockFavoriteMigration(sql); + await assertPostgresDataReady(sql); + await sql.query('UPDATE tool_favorite_readiness SET activated = true WHERE singleton'); + }); +} diff --git a/packages/data-schemas/src/postgres/mongo.spec.ts b/packages/data-schemas/src/postgres/mongo.spec.ts new file mode 100644 index 0000000000..0f3dba32eb --- /dev/null +++ b/packages/data-schemas/src/postgres/mongo.spec.ts @@ -0,0 +1,102 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { createToolFavoriteModel } from '~/models/favorite'; +import toolFavoriteSchema from '~/schema/favorite'; +import { createMongoFavoriteStore } from './mongo'; + +async function rows(source: AsyncIterable): Promise { + const result: unknown[] = []; + for await (const row of source) result.push(row); + return result; +} + +describe('CLI Mongo favorite snapshots', () => { + let server: MongoMemoryServer; + const database = new mongoose.Mongoose(); + beforeAll(async () => { + server = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } }); + }, 90000); + beforeEach(async () => { + await database.connect(server.getUri(), { autoCreate: false, autoIndex: false }); + await database.connection.dropDatabase(); + }); + afterEach(async () => { + await database.disconnect(); + }); + afterAll(async () => { + await server.stop(); + }); + it('reads missing collections without DDL, model registration, or inherited schema options', async () => { + const previous = { + autoCreate: toolFavoriteSchema.get('autoCreate'), + autoIndex: toolFavoriteSchema.get('autoIndex'), + autoSearchIndex: toolFavoriteSchema.get('autoSearchIndex'), + }; + toolFavoriteSchema.set('autoCreate', true); + toolFavoriteSchema.set('autoIndex', true); + toolFavoriteSchema.set('autoSearchIndex', true); + try { + for (let attempt = 0; attempt < 2; attempt++) { + const store = createMongoFavoriteStore(database); + expect(await rows(store.source())).toEqual([]); + expect(database.modelNames()).toEqual([]); + expect(await database.connection.db!.listCollections().toArray()).toEqual([]); + await database.disconnect(); + await database.connect(server.getUri(), { autoCreate: false, autoIndex: false }); + } + } finally { + toolFavoriteSchema.set('autoCreate', previous.autoCreate); + toolFavoriteSchema.set('autoIndex', previous.autoIndex); + toolFavoriteSchema.set('autoSearchIndex', previous.autoSearchIndex); + } + }); + it('uses the data-schema collection name and preserves raw documents and indexes', async () => { + const metadata = new mongoose.Mongoose(); + metadata.set('autoCreate', false); + metadata.set('autoIndex', false); + const name = createToolFavoriteModel(metadata).collection.name; + const collection = database.connection.collection(name); + const row = { + _id: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + tenantId: 'tenant-one', + itemType: 'tool', + itemId: 'favorite', + createdAt: new Date('2025-01-01'), + updatedAt: new Date('2025-01-02'), + }; + await collection.insertOne(row); + const before = await database.connection.db!.listCollections().toArray(); + const indexes = await collection.indexes(); + const store = createMongoFavoriteStore(database); + expect(await rows(store.source())).toEqual([ + { + id: row._id.toHexString(), + userId: row.user.toHexString(), + tenantId: row.tenantId, + itemType: row.itemType, + itemId: row.itemId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }, + ]); + expect(await collection.find().toArray()).toEqual([row]); + expect(await collection.indexes()).toEqual(indexes); + expect(await database.connection.db!.listCollections().toArray()).toEqual(before); + expect(database.modelNames()).toEqual([]); + }); + it('can replace a missing collection with an empty snapshot using only explicit staging DDL', async () => { + const store = createMongoFavoriteStore(database); + const staging = await store.stage(); + expect(await rows(staging.source())).toEqual([]); + const staged = await database.connection.db!.listCollections().toArray(); + expect(staged).toHaveLength(1); + expect(staged[0].name).toContain('_reconcile_'); + await staging.replace(); + const replaced = await database.connection.db!.listCollections().toArray(); + expect(replaced).toHaveLength(1); + expect(replaced[0].name).toBe(staged[0].name.split('_reconcile_')[0]); + expect(await rows(store.source())).toEqual([]); + expect(database.modelNames()).toEqual([]); + }); +}); diff --git a/packages/data-schemas/src/postgres/mongo.ts b/packages/data-schemas/src/postgres/mongo.ts new file mode 100644 index 0000000000..82c6e19153 --- /dev/null +++ b/packages/data-schemas/src/postgres/mongo.ts @@ -0,0 +1,95 @@ +import { randomBytes } from 'node:crypto'; +import type mongoose from 'mongoose'; +import type { FavoriteSnapshotRow, FavoriteSnapshotSource } from './import'; +import { buildIndexWithRetry } from '~/utils/retry'; +import toolFavoriteSchema from '~/schema/favorite'; + +export interface MongoFavoriteStaging { + source: FavoriteSnapshotSource; + insert(rows: FavoriteSnapshotRow[]): Promise; + replace(): Promise; + discard(): Promise; +} + +export interface MongoFavoriteStore { + source: FavoriteSnapshotSource; + stage(): Promise; +} + +/** Never register application models on the connected CLI instance. */ +export function createMongoFavoriteStore(database: typeof mongoose): MongoFavoriteStore { + const metadata = new database.Mongoose(); + const schema = toolFavoriteSchema.clone(); + schema.set('autoCreate', false); + schema.set('autoIndex', false); + schema.set('autoSearchIndex', false); + // eslint-disable-next-line no-restricted-syntax -- Collection name metadata only, no raw driver operations. + const name = metadata.model('ToolFavorite', schema).collection.name; + const collection = database.connection.collection(name); + const snapshot = (target: typeof collection): FavoriteSnapshotSource => + async function* () { + const cursor = target.find({}).sort({ _id: 1 }).batchSize(250); + try { + for await (const row of cursor) { + yield { + id: String(row._id), + userId: String(row.user), + tenantId: row.tenantId ?? null, + itemType: row.itemType, + itemId: row.itemId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; + } + } finally { + await cursor.close(); + } + }; + return { + source: snapshot(collection), + async stage() { + const stagingName = `${name}_reconcile_${randomBytes(6).toString('hex')}`; + const staging = database.connection.collection(stagingName); + const discard = async () => { + await staging.drop().catch(() => undefined); + }; + try { + await database.connection.createCollection(stagingName); + const exists = await database.connection.db!.listCollections({ name }).hasNext(); + for (const index of exists ? await collection.indexes() : []) { + if (index.name === '_id_') continue; + const options = Object.fromEntries( + Object.entries(index).filter(([key]) => !['key', 'v', 'ns'].includes(key)), + ); + await buildIndexWithRetry( + () => staging.createIndex(index.key, options), + 'favorites_reconciliation_index', + ); + } + } catch (error) { + await discard(); + throw error; + } + return { + source: snapshot(staging), + async insert(rows) { + await staging.insertMany( + rows.map((row) => ({ + _id: new database.Types.ObjectId(row.id), + user: new database.Types.ObjectId(row.userId), + tenantId: row.tenantId, + itemType: row.itemType, + itemId: row.itemId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + })), + ); + }, + async replace() { + await staging.rename(name, { dropTarget: true }); + }, + discard, + }; + }, + }; +} diff --git a/packages/data-schemas/src/postgres/pool.spec.ts b/packages/data-schemas/src/postgres/pool.spec.ts new file mode 100644 index 0000000000..6f5874c0fb --- /dev/null +++ b/packages/data-schemas/src/postgres/pool.spec.ts @@ -0,0 +1,237 @@ +import pg from 'pg'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { EventEmitter } from 'node:events'; +import { execFile } from 'node:child_process'; +import type { PostgresOptions } from './pool'; +import { createPostgresDatabase } from './pool'; + +const postgres = process.env.PG_TEST_URL ? describe : describe.skip; +const options = (): PostgresOptions => ({ + connectionString: process.env.PG_TEST_URL ?? '', + max: 1, + maxWaiting: 10, + acquireTimeoutMs: 2000, + queryTimeoutMs: 10000, + idleTimeoutMs: 1000, +}); +const safeMessage = 'PostgreSQL connection unavailable'; +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +postgres('PostgreSQL borrowed connections', () => { + it.each(['session', 'transaction'] as const)( + 'survives an idle %s disconnect in a disposable child process', + async (kind) => { + const source = ` + import pg from 'pg'; + import { createPostgresDatabase } from ${JSON.stringify(path.resolve(__dirname, 'pool.ts'))}; + const db = createPostgresDatabase({ ...${JSON.stringify({ ...options(), connectionString: undefined })}, connectionString: process.env.PG_TEST_URL }); + const killer = new pg.Client({ connectionString: process.env.PG_TEST_URL }); + await killer.connect(); + let openGate; + const gate = new Promise(resolve => { openGate = resolve; }); + let session; + let entered; + const ready = new Promise(resolve => { entered = resolve; }); + const operation = db[${JSON.stringify(kind)}](async sql => { + session = sql; + const { rows } = await sql.query('SELECT pg_backend_pid() AS pid'); + entered(rows[0].pid); + await gate; + return 'must not succeed'; + }); + const outcome = operation.then(() => 'success', error => error.message); + await killer.query('SELECT pg_terminate_backend($1)', [await ready]); + const observed = await Promise.race([outcome, new Promise(resolve => setTimeout(() => resolve('not reported'), 1000))]); + if (observed !== ${JSON.stringify(safeMessage)}) throw new Error(observed); + for (const attempt of [ + () => session.query('SELECT 1'), + () => session.transaction(async () => 'must not succeed'), + ]) { + const message = await attempt().then(() => 'success', error => error.message); + if (message !== ${JSON.stringify(safeMessage)}) throw new Error(message); + } + let closed = false; + const closing = db.close().then(() => { closed = true; }); + await new Promise(resolve => setTimeout(resolve, 30)); + if (closed) throw new Error('close released an unfinished callback'); + openGate(); + await outcome; + await closing; + await killer.end(); + if (observed !== ${JSON.stringify(safeMessage)}) throw new Error(observed); + console.log('borrowed-disconnect-safe'); + `; + const { stdout } = await promisify(execFile)( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { timeout: 10000 }, + ); + expect(stdout).toContain('borrowed-disconnect-safe'); + }, + ); + + it.each(['query', 'transaction'] as const)( + 'sanitizes an active %s disconnect and replaces the failed backend', + async (kind) => { + const db = createPostgresDatabase(options()); + const killer = new pg.Client({ connectionString: process.env.PG_TEST_URL }); + await killer.connect(); + try { + const pid = (await db.query('SELECT pg_backend_pid() AS pid')).rows[0].pid; + const operation = + kind === 'query' + ? db.query('SELECT pg_sleep(5)') + : db.transaction(async (sql) => { + await sql.query('SELECT pg_sleep(5)'); + }); + const outcome = operation.catch((error: Error) => error); + for (let i = 0; i < 100; i++) { + const { rows } = await killer.query('SELECT state FROM pg_stat_activity WHERE pid=$1', [ + pid, + ]); + if (rows[0]?.state === 'active') break; + await delay(10); + } + await killer.query('SELECT pg_terminate_backend($1)', [pid]); + const error = await outcome; + expect(error).toEqual(new Error(safeMessage)); + expect(error).not.toHaveProperty('cause'); + const next = await db.query('SELECT pg_backend_pid() AS pid'); + expect(next.rows[0].pid).not.toBe(pid); + } finally { + await db.close(); + await killer.end(); + } + }, + ); +}); + +describe('PostgreSQL lease lifecycle', () => { + function setup() { + const client = new EventEmitter(); + const idle = () => undefined; + client.on('error', idle); + const query = jest.fn(async (_text: string) => ({ rows: [], rowCount: 0 })); + const release = jest.fn((_discard: boolean) => { + client.on('error', idle); + }); + Object.assign(client, { query, release }); + const pool = new EventEmitter(); + let onHandoff: () => void = () => undefined; + const connect = jest.fn((callback: (error: Error | undefined, client: unknown) => void) => { + client.removeListener('error', idle); + callback(undefined, client); + onHandoff(); + }); + const end = jest.fn(async () => undefined); + Object.assign(pool, { connect, end, waitingCount: 0 }); + jest.spyOn(pg, 'Pool').mockImplementation(() => pool as pg.Pool); + const db = createPostgresDatabase(options()); + return { + db, + connect, + client, + query, + release, + onHandoff: (callback: () => void) => { + onHandoff = callback; + }, + }; + } + + it.each(['callback', 'throw'] as const)('sanitizes a driver connect %s failure', async (kind) => { + const { db, connect, release } = setup(); + connect.mockImplementation((callback) => { + const original = new Error('postgresql://private:secret@host/private'); + if (kind === 'throw') throw original; + callback(original, undefined); + }); + const error = await db.query('SELECT 1').catch((error: Error) => error); + expect(error).toEqual(new Error(safeMessage)); + expect(error).not.toHaveProperty('cause'); + expect(release).not.toHaveBeenCalled(); + await db.close(); + }); + + it('protects the synchronous connect handoff and releases exactly once', async () => { + const fixture = setup(); + fixture.onHandoff(() => fixture.client.emit('error', new Error('private DSN'))); + await expect(fixture.db.query('private query')).rejects.toThrow(safeMessage); + expect(fixture.query).not.toHaveBeenCalled(); + expect(fixture.release).toHaveBeenCalledTimes(1); + expect(fixture.release).toHaveBeenCalledWith(true); + expect(fixture.client.listenerCount('error')).toBe(1); + await fixture.db.close(); + }); + + it('leaves only the pool idle listener across successful leases', async () => { + const { db, client, release } = setup(); + for (let i = 0; i < 20; i++) { + await db.session(async (sql) => { + expect(client.listenerCount('error')).toBe(1); + await sql.query('SELECT 1'); + }); + expect(client.listenerCount('error')).toBe(1); + } + expect(release).toHaveBeenCalledTimes(20); + expect(release).toHaveBeenLastCalledWith(false); + await db.close(); + }); + + it('handles an error racing release without returning success or leaking listeners', async () => { + const { db, client, release } = setup(); + release.mockImplementation(() => { + client.on('error', () => undefined); + client.emit('error', new Error('private release fault')); + }); + await expect(db.query('SELECT 1')).rejects.toThrow(safeMessage); + expect(release).toHaveBeenCalledTimes(1); + expect(client.listenerCount('error')).toBe(1); + await db.close(); + }); + + it('retains local SQL and callback exception identity', async () => { + const { db, query, release } = setup(); + const sqlError = Object.assign(new Error('local constraint'), { code: '23505' }); + query.mockRejectedValueOnce(sqlError); + await expect(db.query('INSERT')).rejects.toBe(sqlError); + const original = new Error('local callback'); + await expect( + db.transaction(async () => { + throw original; + }), + ).rejects.toBe(original); + expect(query.mock.calls.map(([text]) => text)).toEqual(['INSERT', 'BEGIN', 'ROLLBACK']); + expect(release).toHaveBeenLastCalledWith(false); + await db.close(); + }); + + it('discards on rollback failure while preserving the original callback exception', async () => { + const { db, query, release } = setup(); + const original = new Error('local callback'); + query.mockResolvedValueOnce({ rows: [], rowCount: 0 }); + query.mockRejectedValueOnce(Object.assign(new Error('rollback fault'), { code: 'XX000' })); + await expect( + db.transaction(async () => { + throw original; + }), + ).rejects.toBe(original); + expect(release).toHaveBeenCalledWith(true); + await db.close(); + }); + + it('reports a lost COMMIT acknowledgement without issuing a misleading ROLLBACK', async () => { + const { db, query, release } = setup(); + query.mockImplementation(async (text) => { + if (text === 'COMMIT') + throw Object.assign(new Error('private commit detail'), { code: 'ECONNRESET' }); + return { rows: [], rowCount: 0 }; + }); + await expect(db.transaction(async () => 'value')).rejects.toThrow(safeMessage); + expect(query.mock.calls.map(([text]) => text)).toEqual(['BEGIN', 'COMMIT']); + expect(release).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledWith(true); + await db.close(); + }); +}); diff --git a/packages/data-schemas/src/postgres/pool.ts b/packages/data-schemas/src/postgres/pool.ts new file mode 100644 index 0000000000..ecdfe038ec --- /dev/null +++ b/packages/data-schemas/src/postgres/pool.ts @@ -0,0 +1,190 @@ +import pg from 'pg'; +import type { QueryResult, QueryResultRow } from 'pg'; + +export interface PostgresOptions { + connectionString: string; + max: number; + maxWaiting: number; + acquireTimeoutMs: number; + queryTimeoutMs: number; + idleTimeoutMs: number; +} + +export interface SqlExecutor { + query( + text: string, + values?: unknown[], + ): Promise>; +} + +export interface PostgresSession extends SqlExecutor { + transaction(operation: (sql: SqlExecutor) => Promise): Promise; +} + +export interface PostgresDatabase extends PostgresSession { + session(operation: (sql: PostgresSession) => Promise): Promise; + transaction(operation: (sql: SqlExecutor) => Promise): Promise; + close(): Promise; +} + +export function createPostgresDatabase(options: PostgresOptions): PostgresDatabase { + const pool = new pg.Pool({ + connectionString: options.connectionString, + max: options.max, + connectionTimeoutMillis: options.acquireTimeoutMs, + idleTimeoutMillis: options.idleTimeoutMs, + statement_timeout: options.queryTimeoutMs, + query_timeout: options.queryTimeoutMs, + idle_in_transaction_session_timeout: options.queryTimeoutMs, + application_name: 'librechat', + }); + // Idle-client errors must not become uncaught exceptions or expose connection details. + pool.on('error', () => undefined); + let closing = false; + let closePromise: Promise | undefined; + function connectionError(): Error { + return new Error('PostgreSQL connection unavailable'); + } + function isConnectionError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + const code = 'code' in error && typeof error.code === 'string' ? error.code : ''; + const severity = 'severity' in error ? error.severity : undefined; + return ( + severity === 'FATAL' || + severity === 'PANIC' || + code?.startsWith('08') === true || + ['57P01', '57P02', '57P03', 'ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'ECONNREFUSED'].includes( + code ?? '', + ) || + [ + 'Connection terminated unexpectedly', + 'Connection terminated', + 'Client has encountered a connection error and is not queryable', + 'Client was closed and is not queryable', + 'Query read timeout', + ].includes(error.message) + ); + } + function connect() { + if (closing || pool.waitingCount >= options.maxWaiting) { + return Promise.reject(new Error('PostgreSQL connection admission unavailable')); + } + // The callback installs protection before pg-pool's synchronous handoff returns. + return new Promise<{ + query: SqlExecutor['query']; + failure: Promise; + assertHealthy(): void; + discard(): void; + release(): void; + }>((resolve, reject) => { + pool.connect((error, client) => { + if (error || !client) { + reject(connectionError()); + return; + } + let failed: Error | undefined; + let released = false; + let discard = false; + let rejectFailure: (error: Error) => void; + const failure = new Promise((_, reject) => { + rejectFailure = reject; + }); + void failure.catch(() => undefined); + const onError = () => { + if (failed) return; + failed = connectionError(); + rejectFailure(failed); + }; + client.on('error', onError); + const assertHealthy = () => { + if (failed) throw failed; + if (released) throw connectionError(); + }; + resolve({ + failure, + assertHealthy, + discard() { + discard = true; + }, + async query(text: string, values?: unknown[]) { + assertHealthy(); + try { + const result = await client.query(text, values); + assertHealthy(); + return result; + } catch (error) { + if (isConnectionError(error)) onError(); + throw failed ?? error; + } + }, + release() { + if (released) return; + released = true; + try { + client.release(discard || Boolean(failed)); + } finally { + // release restores pg-pool's idle listener before ours is removed. + client.removeListener('error', onError); + } + }, + }); + }); + }).catch(() => { + throw connectionError(); + }); + } + const database: PostgresDatabase = { + query: (text, values) => database.session((session) => session.query(text, values)), + async transaction(operation: (sql: SqlExecutor) => Promise): Promise { + return database.session((session) => session.transaction(operation)); + }, + async session(operation: (sql: PostgresSession) => Promise): Promise { + const lease = await connect(); + let inTransaction = false; + const session: PostgresSession = { + query: lease.query, + async transaction(work: (sql: SqlExecutor) => Promise): Promise { + lease.assertHealthy(); + if (inTransaction) throw new Error('Nested PostgreSQL transactions are not supported'); + inTransaction = true; + try { + await lease.query('BEGIN'); + const value = await work(session); + await lease.query('COMMIT'); + return value; + } catch (error) { + // A lost COMMIT acknowledgement has an unknown outcome. Do not claim rollback. + lease.assertHealthy(); + try { + await lease.query('ROLLBACK'); + } catch { + lease.discard(); + } + lease.assertHealthy(); + throw error; + } finally { + inTransaction = false; + } + }, + }; + const work = Promise.resolve() + .then(() => { + lease.assertHealthy(); + return operation(session); + }) + .then((value) => { + lease.assertHealthy(); + return value; + }) + .finally(() => lease.release()); + // Report disconnects promptly, but retain ownership until the callback settles. + return Promise.race([work, lease.failure]); + }, + close() { + closing = true; + closePromise ??= pool.end(); + return closePromise; + }, + }; + return database; +} diff --git a/packages/data-schemas/src/postgres/postgres.helper.ts b/packages/data-schemas/src/postgres/postgres.helper.ts new file mode 100644 index 0000000000..2c523cf350 --- /dev/null +++ b/packages/data-schemas/src/postgres/postgres.helper.ts @@ -0,0 +1,51 @@ +import { randomBytes } from 'node:crypto'; +import type { PostgresDatabase, PostgresOptions } from './pool'; +import { createPostgresDatabase } from './pool'; + +export const postgresTestOptions: PostgresOptions = { + connectionString: process.env.PG_TEST_URL ?? '', + max: 4, + maxWaiting: 20, + acquireTimeoutMs: 1000, + queryTimeoutMs: 5000, + idleTimeoutMs: 1000, +}; + +/** Each spec/Stryker worker gets its own schema; network and constraints remain real. */ +export async function createPostgresTestDatabase(): Promise<{ + db: PostgresDatabase; + options: PostgresOptions; + schema: string; +}> { + const admin = createPostgresDatabase(postgresTestOptions); + const schema = `favorites_test_${randomBytes(8).toString('hex')}`; + try { + await admin.query(`CREATE SCHEMA ${schema}`); + const url = new URL(postgresTestOptions.connectionString); + url.searchParams.set('options', `-c search_path=${schema}`); + const options = { ...postgresTestOptions, connectionString: url.toString() }; + const database = createPostgresDatabase(options); + let closing: Promise | undefined; + const db: PostgresDatabase = { + ...database, + close() { + closing ??= (async () => { + try { + await database.close(); + } finally { + try { + await admin.query(`DROP SCHEMA ${schema} CASCADE`); + } finally { + await admin.close(); + } + } + })(); + return closing; + }, + }; + return { db, options, schema }; + } catch (error) { + await admin.close(); + throw error; + } +} diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.ts index 3c54e3f526..dc17862e9b 100644 --- a/packages/data-schemas/src/schema/user.ts +++ b/packages/data-schemas/src/schema/user.ts @@ -170,6 +170,10 @@ const userSchema: Schema = new Schema( type: Date, select: false, }, + agentTriggerDeletionAttemptId: { + type: String, + select: false, + }, subagentAdmissionFences: { type: [ { diff --git a/packages/data-schemas/src/types/favorite.ts b/packages/data-schemas/src/types/favorite.ts index ca8a01ebda..b9274950d2 100644 --- a/packages/data-schemas/src/types/favorite.ts +++ b/packages/data-schemas/src/types/favorite.ts @@ -33,3 +33,11 @@ export interface RemoveToolFavoriteResult { ok: boolean; removed: boolean; } + +export interface ToolFavoriteMethods { + getToolFavorites(userId: string): Promise; + addToolFavorite(params: ToolFavoriteParams): Promise; + removeToolFavorite(params: ToolFavoriteParams): Promise; + deleteUserToolFavorites(userId: string, deletionAttemptId?: string): Promise; + cancelUserToolFavoriteDeletion?(userId: string, deletionAttemptId: string): Promise; +} diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.ts index 9b0e8837b8..962b5faaa5 100644 --- a/packages/data-schemas/src/types/user.ts +++ b/packages/data-schemas/src/types/user.ts @@ -69,6 +69,8 @@ export interface IUser extends Document { termsAcceptedSource?: 'migration'; /** Internal fence that prevents agent-trigger admission during account deletion. */ agentTriggerDeletionStartedAt?: Date; + /** Durable identity shared by every offline recovery of this erasure attempt. */ + agentTriggerDeletionAttemptId?: string; /** Expiring fences closing subagent admission while bulk deletions drain. */ subagentAdmissionFences?: Array<{ token: string; diff --git a/scripts/quality/aggregate.mts b/scripts/quality/aggregate.mts new file mode 100644 index 0000000000..932dbf1702 --- /dev/null +++ b/scripts/quality/aggregate.mts @@ -0,0 +1,331 @@ +import { fileURLToPath } from 'node:url'; +import { join, resolve } from 'node:path'; +import { parseArgs, isDeepStrictEqual } from 'node:util'; +import { + appendFileSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs'; + +import { shardTargets } from './shard.mts'; + +interface Options { + head: string; + workspace: string; + count: number; +} + +type ObjectValue = Record; +const statuses = [ + 'Killed', + 'Survived', + 'Timeout', + 'NoCoverage', + 'CompileError', + 'RuntimeError', + 'Ignored', + 'Pending', +] as const; +type Status = (typeof statuses)[number]; + +function requireValue(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} +function object(value: unknown): ObjectValue { + requireValue( + value !== null && typeof value === 'object' && !Array.isArray(value), + 'Expected JSON object', + ); + return value as ObjectValue; +} +function array(value: unknown): unknown[] { + requireValue(Array.isArray(value), 'Expected JSON array'); + return value; +} +function text(value: unknown): string { + requireValue(typeof value === 'string' && value.length > 0, 'Expected nonempty string'); + return value; +} +function integer(value: unknown, minimum = 0): number { + requireValue( + Number.isSafeInteger(value) && typeof value === 'number' && value >= minimum, + 'Expected nonnegative integer', + ); + return value; +} +function path(value: unknown): string { + const file = text(value); + requireValue( + !file.includes('\\') && + !file.startsWith('/') && + file.split('/').every((part) => part && part !== '.' && part !== '..'), + 'Expected repository-relative file', + ); + return file; +} +function location(value: unknown): ObjectValue { + const span = object(value); + const start = object(span.start); + const end = object(span.end); + const line = integer(start.line, 1); + const column = integer(start.column, 1); + const endLine = integer(end.line, 1); + const endColumn = integer(end.column, 1); + requireValue(endLine > line || (endLine === line && endColumn >= column), 'Invalid location'); + return span; +} +function targets(value: unknown) { + const files = new Set(); + return array(value).map((entry) => { + const target = object(entry); + const file = path(target.file); + requireValue(!files.has(file), 'Duplicate target file'); + files.add(file); + const functions = array(target.functions); + requireValue(functions.length > 0 && array(target.mutate).length > 0, 'Empty mutation target'); + for (const fn of functions) { + const row = object(fn); + text(row.function); + integer(row.line, 1); + integer(row.column, 1); + location(row.location); + } + for (const pattern of array(target.mutate)) text(pattern); + return { file, functions, original: target }; + }); +} + +/** Validate complete file shards without changing their reports or local test/mutant IDs. */ +export function aggregate(inputs: readonly unknown[], options: Options) { + integer(options.count, 1); + text(options.head); + text(options.workspace); + requireValue(inputs.length === options.count, 'Wrong shard directory count'); + const shards = inputs + .map((input) => { + const row = object(input); + const selection = object(row.selection); + const shard = object(selection.shard); + const index = integer(shard.index, 1); + requireValue( + index <= options.count && shard.total === options.count, + 'Wrong shard index or count', + ); + requireValue(selection.head === options.head, 'Wrong selection head'); + requireValue(selection.workspace === options.workspace, 'Wrong selection workspace'); + text(selection.base); + const files = array(selection.files).map((entry) => path(object(entry).file)); + requireValue(new Set(files).size === files.length, 'Duplicate selected file'); + const all = targets(row.allFunctions); + requireValue( + all.every(({ file }) => files.includes(file)), + 'Target outside selected files', + ); + targets(row.functions); + const expected = shardTargets(all, { index, total: options.count }); + requireValue( + isDeepStrictEqual( + row.functions, + expected.map(({ original }) => original), + ), + 'Wrong file/function shard partition', + ); + return { + index, + selection, + all, + expected, + report: object(row.report), + progress: object(row.progress), + }; + }) + .sort((a, b) => a.index - b.index); + requireValue( + new Set(shards.map(({ index }) => index)).size === options.count, + 'Duplicate shard index', + ); + const first = shards[0]!; + const counts: Record = { + Killed: 0, + Survived: 0, + Timeout: 0, + NoCoverage: 0, + CompileError: 0, + RuntimeError: 0, + Ignored: 0, + Pending: 0, + }; + const identities = new Set(); + let staticTotal = 0; + for (const shard of shards) { + requireValue(shard.selection.base === first.selection.base, 'Inconsistent selection base'); + requireValue( + isDeepStrictEqual(shard.selection.files, first.selection.files), + 'Inconsistent original selected files', + ); + requireValue( + isDeepStrictEqual( + shard.all.map(({ original }) => original), + first.all.map(({ original }) => original), + ), + 'Inconsistent all-functions manifest', + ); + requireValue(shard.report.schemaVersion === '1.0', 'Unsupported mutation report schema'); + let total = 0; + let staticCount = 0; + const ids = new Set(); + for (const [file, entry] of Object.entries(object(shard.report.files))) { + path(file); + requireValue( + shard.expected.some((target) => target.file === file), + 'Report file outside shard targets', + ); + for (const value of array(object(entry).mutants)) { + const mutant = object(value); + const id = text(mutant.id); + requireValue(!ids.has(id), 'Duplicate local mutant ID'); + ids.add(id); + const span = location(mutant.location); + const mutator = text(mutant.mutatorName); + requireValue(typeof mutant.replacement === 'string', 'Invalid mutant replacement'); + requireValue( + mutant.static === undefined || typeof mutant.static === 'boolean', + 'Invalid static flag', + ); + const status = statuses.find((name) => name === mutant.status); + requireValue(status, 'Unknown mutant status'); + requireValue( + status !== 'RuntimeError' && status !== 'Pending', + 'Incomplete mutant outcome', + ); + const identity = JSON.stringify([ + file, + object(span.start).line, + object(span.start).column, + object(span.end).line, + object(span.end).column, + mutator, + mutant.replacement, + ]); + requireValue(!identities.has(identity), 'Duplicate mutation identity'); + identities.add(identity); + counts[status]++; + total++; + if (mutant.static === true) staticCount++; + } + } + const progress = shard.progress; + requireValue( + progress.complete === true && progress.completed === total && progress.total === total, + 'Incomplete mutation progress', + ); + requireValue( + progress.staticCompleted === staticCount && progress.staticTotal === staticCount, + 'Incomplete static mutation progress', + ); + staticTotal += staticCount; + } + const detected = counts.Killed + counts.Timeout; + const tested = detected + counts.Survived + counts.NoCoverage; + const total = Object.values(counts).reduce((sum, count) => sum + count, 0); + return { + head: options.head, + base: first.selection.base, + workspace: options.workspace, + shardCount: options.count, + complete: true, + files: first.selection.files, + functions: first.all.map(({ original }) => original), + total, + completed: total, + staticTotal, + staticCompleted: staticTotal, + counts, + score: tested ? (100 * detected) / tested : null, + shards: shards.map(({ index, report, progress }) => ({ index, report, progress })), + }; +} + +export function aggregateMarkdown(result: ReturnType): string { + const score = result.score === null ? 'N/A (no valid mutants)' : `${result.score.toFixed(2)}%`; + return [ + `## ${result.workspace}: mutation testing (advisory)`, + '', + `Complete: ${result.shardCount} shards, ${result.completed}/${result.total} mutants. Static mutations: ${result.staticCompleted}/${result.staticTotal}.`, + '', + `Mutation score: ${score}. Compile errors and ignored mutants are excluded from the score, not the report.`, + '', + '| Result | Mutants |', + '| --- | ---: |', + ...statuses.map((status) => `| ${status} | ${result.counts[status]} |`), + '', + 'Aggregate JSON retains each shard report, including source, test metadata, and unchanged local IDs.', + '', + ].join('\n'); +} + +function main() { + const { values } = parseArgs({ + options: { + directory: { type: 'string' }, + head: { type: 'string' }, + workspace: { type: 'string' }, + count: { type: 'string' }, + output: { type: 'string' }, + }, + }); + requireValue( + values.directory && values.head && values.workspace && values.count && values.output, + 'Required: --directory --head --workspace --count --output', + ); + for (const name of ['aggregate.json', 'summary.md']) { + rmSync(join(values.output, name), { force: true }); + } + requireValue(/^[1-9]\d*$/u.test(values.count), 'Invalid shard count'); + const entries = readdirSync(values.directory, { withFileTypes: true }); + requireValue( + entries.length === Number(values.count) && entries.every((entry) => entry.isDirectory()), + 'Wrong shard directory count', + ); + const inputs = entries.map((entry) => { + const read = (name: string): unknown => { + try { + return JSON.parse(readFileSync(join(values.directory!, entry.name, name), 'utf8')); + } catch { + throw new Error(`Missing or malformed shard ${name}`); + } + }; + return { + selection: read('selection.json'), + allFunctions: read('all-functions.json'), + functions: read('functions.json'), + report: read('mutation.json'), + progress: read('progress.json'), + }; + }); + const result = aggregate(inputs, { + head: values.head, + workspace: values.workspace, + count: Number(values.count), + }); + mkdirSync(values.output, { recursive: true }); + writeFileSync(join(values.output, 'aggregate.json'), JSON.stringify(result, null, 2)); + const markdown = aggregateMarkdown(result); + writeFileSync(join(values.output, 'summary.md'), markdown); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + console.log(markdown); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + main(); + } catch { + console.error( + 'Mutation aggregation failed: shard evidence is missing, inconsistent, malformed, or incomplete.', + ); + process.exitCode = 1; + } +} diff --git a/scripts/quality/aggregate.test.mts b/scripts/quality/aggregate.test.mts new file mode 100644 index 0000000000..1f7369ac31 --- /dev/null +++ b/scripts/quality/aggregate.test.mts @@ -0,0 +1,351 @@ +import { test } from 'node:test'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { strict as assert } from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; + +import { aggregate, aggregateMarkdown } from './aggregate.mts'; +import { shardTargets } from './shard.mts'; + +const options = { head: 'head-sha', workspace: 'api', count: 3 }; +const point = (line: number) => ({ line, column: 1 }); +const span = (line: number) => ({ start: point(line), end: point(line + 1) }); +const functions = Array.from({ length: 5 }, (_, index) => ({ + file: `packages/api/src/file${index}.ts`, + functions: Array.from({ length: [2, 4, 1, 3, 2][index] }, (_, fn) => fn * 2 + 1).map((line) => ({ + function: `f${line}`, + line, + column: 1, + location: span(line), + })), + mutate: [`packages/api/src/file${index}.ts:1:0-4:0`], +})); +const files = functions.map(({ file }) => ({ file, ranges: [{ start: 1, count: 4 }] })); +const outcomes = ['Killed', 'Survived', 'Timeout', 'NoCoverage', 'CompileError', 'Ignored']; +function fixtures() { + return Array.from({ length: 3 }, (_, shard) => { + const chosen = shardTargets(functions, { index: shard + 1, total: 3 }); + const reportFiles = Object.fromEntries( + chosen.map(({ file }, index) => [ + file, + { + source: 'function source() {}', + language: 'typescript', + mutants: [0, 1].map((position) => ({ + id: String(index * 2 + position), + location: span(position + 1), + mutatorName: 'BooleanLiteral', + replacement: 'false', + status: outcomes[(shard * 2 + position) % outcomes.length], + static: position === 1, + coveredBy: ['0'], + killedBy: position === 0 ? ['0'] : [], + statusReason: 'preserve reason', + })), + }, + ]), + ); + const total = chosen.length * 2; + return { + selection: { + head: options.head, + base: 'base-sha', + workspace: options.workspace, + files: structuredClone(files), + shard: { index: shard + 1, total: 3 }, + }, + allFunctions: structuredClone(functions), + functions: structuredClone([...chosen]), + report: { + schemaVersion: '1.0', + files: reportFiles, + testFiles: { 'tests/file.ts': { tests: [{ id: '0', name: 'test' }] } }, + framework: { name: 'Stryker' }, + }, + progress: { + complete: true, + completed: total, + total, + staticCompleted: chosen.length, + staticTotal: chosen.length, + }, + }; + }); +} + +test('aggregates every file, function, outcome, static mutant, and unchanged raw report', () => { + const inputs = fixtures(); + const snapshot = structuredClone(inputs); + const result = aggregate(inputs, options); + assert.equal(result.complete, true); + assert.equal(result.total, 10); + assert.equal(result.completed, 10); + assert.equal(result.staticTotal, 5); + assert.equal(result.staticCompleted, 5); + assert.deepEqual(result.functions, functions); + assert.deepEqual(result.files, files); + assert.deepEqual(result.counts, { + Killed: 2, + Survived: 2, + Timeout: 2, + NoCoverage: 2, + CompileError: 1, + RuntimeError: 0, + Ignored: 1, + Pending: 0, + }); + assert.equal(result.score, 50); + assert.deepEqual( + result.shards.map(({ report }) => report), + inputs.map(({ report }) => report), + ); + assert.deepEqual(inputs, snapshot); + assert.match(aggregateMarkdown(result), /CompileError \| 1/); + assert.match(aggregateMarkdown(result), /Static mutations: 5\/5/); +}); + +test('shard directory ordering does not affect results or identities', () => { + const inputs = fixtures(); + assert.deepEqual( + aggregate([inputs[2], inputs[0], inputs[1]], options), + aggregate(inputs, options), + ); +}); + +const invalid: [string, (inputs: ReturnType) => void][] = [ + [ + 'missing shard', + (inputs) => { + inputs.pop(); + }, + ], + [ + 'extra shard', + (inputs) => { + inputs.push(structuredClone(inputs[0]!)); + }, + ], + [ + 'duplicate shard index', + (inputs) => { + inputs[1]!.selection.shard.index = 1; + }, + ], + [ + 'wrong shard count', + (inputs) => { + inputs[0]!.selection.shard.total = 4; + }, + ], + [ + 'wrong head', + (inputs) => { + inputs[0]!.selection.head = 'other'; + }, + ], + [ + 'inconsistent base', + (inputs) => { + inputs[1]!.selection.base = 'other'; + }, + ], + [ + 'wrong workspace', + (inputs) => { + inputs[0]!.selection.workspace = 'client'; + }, + ], + [ + 'inconsistent original files', + (inputs) => { + inputs[0]!.selection.files.pop(); + }, + ], + [ + 'inconsistent full manifest', + (inputs) => { + inputs[1]!.allFunctions[0]!.functions[0]!.function = 'other'; + }, + ], + [ + 'dropped target file', + (inputs) => { + inputs[0]!.functions.pop(); + }, + ], + [ + 'dropped target function', + (inputs) => { + inputs[0]!.functions[0]!.functions.pop(); + }, + ], + [ + 'wrong round-robin partition', + (inputs) => { + inputs[0]!.functions.reverse(); + }, + ], + [ + 'out-of-shard report file', + (inputs) => { + inputs[0]!.report.files['packages/api/src/other.ts'] = structuredClone( + Object.values(inputs[0]!.report.files)[0]!, + ); + }, + ], + [ + 'runtime error', + (inputs) => { + Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'RuntimeError'; + }, + ], + [ + 'pending mutant', + (inputs) => { + Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Pending'; + }, + ], + [ + 'unknown status', + (inputs) => { + Object.values(inputs[0]!.report.files)[0]!.mutants[0]!.status = 'Unknown'; + }, + ], + [ + 'incomplete progress', + (inputs) => { + inputs[0]!.progress.complete = false; + }, + ], + [ + 'wrong completed count', + (inputs) => { + inputs[0]!.progress.completed--; + }, + ], + [ + 'wrong planned count', + (inputs) => { + inputs[0]!.progress.total++; + }, + ], + [ + 'wrong static completed count', + (inputs) => { + inputs[0]!.progress.staticCompleted--; + }, + ], + [ + 'wrong static total count', + (inputs) => { + inputs[0]!.progress.staticTotal++; + }, + ], + [ + 'duplicate mutant identity', + (inputs) => { + const mutants = Object.values(inputs[0]!.report.files)[0]!.mutants; + mutants[1] = { ...mutants[0]!, id: 'new' }; + }, + ], + [ + 'duplicate local ID', + (inputs) => { + Object.values(inputs[0]!.report.files)[0]!.mutants[1]!.id = '0'; + }, + ], +]; +for (const [name, change] of invalid) { + test(`fails closed: ${name}`, () => { + const inputs = fixtures(); + change(inputs); + assert.throws(() => aggregate(inputs, options)); + }); +} + +test('requires explicit complete evidence for zero-mutant shards', () => { + const inputs = fixtures(); + for (const input of inputs) { + input.allFunctions = []; + input.functions = []; + input.report.files = {}; + input.progress = { complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 }; + } + const result = aggregate(inputs, options); + assert.equal(result.total, 0); + assert.equal(result.score, null); + assert.deepEqual(result.functions, []); + const missing: unknown[] = structuredClone(inputs); + missing[0] = { ...inputs[0], progress: undefined }; + assert.throws(() => aggregate(missing, options)); +}); + +test('chosen function files may legitimately produce no mutants', () => { + const inputs = fixtures(); + inputs[0]!.report.files = {}; + inputs[0]!.progress = { + complete: true, + completed: 0, + total: 0, + staticCompleted: 0, + staticTotal: 0, + }; + assert.equal(aggregate(inputs, options).total, 6); +}); + +test('malformed evidence is rejected without trusting TypeScript casts', () => { + for (const bad of [null, [], {}, { selection: null }]) { + assert.throws(() => aggregate([bad, bad, bad], options)); + } + const inputs = fixtures(); + assert.throws(() => aggregate(inputs, { ...options, count: 0 })); + assert.throws(() => aggregate(inputs, { ...options, head: '' })); +}); + +test('CLI uses only built-ins, reads unmerged directories, and fails without a green output', (t) => { + const directory = mkdtempSync(join(tmpdir(), 'mutation-aggregate-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const input = join(directory, 'reports'); + const output = join(directory, 'output'); + for (const [index, shard] of fixtures().entries()) { + const folder = join(input, `artifact-${2 - index}`); + mkdirSync(folder, { recursive: true }); + for (const [name, value] of Object.entries({ + selection: shard.selection, + 'all-functions': shard.allFunctions, + functions: shard.functions, + mutation: shard.report, + progress: shard.progress, + })) { + writeFileSync(join(folder, `${name}.json`), JSON.stringify(value)); + } + } + const args = [ + 'scripts/quality/aggregate.mts', + '--directory', + input, + '--head', + options.head, + '--workspace', + options.workspace, + '--count', + '3', + '--output', + output, + ]; + const stepSummary = join(directory, 'step-summary.md'); + const result = spawnSync(process.execPath, args, { + encoding: 'utf8', + env: { ...process.env, GITHUB_STEP_SUMMARY: stepSummary }, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(readFileSync(join(output, 'aggregate.json'), 'utf8')).total, 10); + assert.match(readFileSync(join(output, 'summary.md'), 'utf8'), /50.00%/); + assert.equal(readFileSync(stepSummary, 'utf8'), readFileSync(join(output, 'summary.md'), 'utf8')); + writeFileSync(join(input, 'artifact-0', 'progress.json'), 'not-json-secret-payload'); + const failed = spawnSync(process.execPath, args, { encoding: 'utf8' }); + assert.equal(failed.status, 1); + assert.doesNotMatch(failed.stderr, /secret-payload/); + assert.throws(() => readFileSync(join(output, 'summary.md'))); +}); diff --git a/scripts/quality/run.mts b/scripts/quality/run.mts index 1a388b9f74..f83a9fa54f 100644 --- a/scripts/quality/run.mts +++ b/scripts/quality/run.mts @@ -14,6 +14,7 @@ import { import { mutationSelection, riskMarkdown, riskReport } from './risk.mts'; import { changes, git, WORKSPACES, workspaceFor } from './changes.mts'; +import { parseShard, shardTargets } from './shard.mts'; import { coverageArgs } from './jest.mts'; const { values } = parseArgs({ @@ -22,14 +23,17 @@ const { values } = parseArgs({ head: { type: 'string', default: 'HEAD' }, workspace: { type: 'string' }, check: { type: 'string' }, + shard: { type: 'string' }, }, }); const workspace = WORKSPACES.find(({ name }) => name === values.workspace); if (!workspace || !['risk', 'mutation'].includes(values.check ?? '')) { throw new Error( - 'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation', + 'Usage: node scripts/quality/run.mts --base ref --workspace name --check risk|mutation [--shard INDEX/COUNT]', ); } +const shard = parseShard(values.shard, values.check); +const shardLabel = shard ? ` (shard ${shard.index}/${shard.total})` : ''; const selection = changes(values.base, values.head); if (selection.head !== git(['rev-parse', 'HEAD']).trim()) { throw new Error('Check out the selected head before measuring its source'); @@ -38,14 +42,42 @@ const files = selection.files.filter(({ file }) => workspaceFor(file)?.name === const output = resolve('coverage/changes', workspace.name, values.check!); rmSync(output, { recursive: true, force: true }); mkdirSync(output, { recursive: true }); -writeFileSync(`${output}/selection.json`, JSON.stringify({ ...selection, files }, null, 2)); +writeFileSync( + `${output}/selection.json`, + JSON.stringify( + { ...selection, workspace: workspace.name, files, ...(shard ? { shard } : {}) }, + null, + 2, + ), +); const summary = (markdown: string) => { console.log(markdown); writeFileSync(`${output}/summary.md`, markdown); if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); }; +const emptyMutationReports = () => { + writeFileSync( + `${output}/mutation.json`, + JSON.stringify({ schemaVersion: '1.0', files: {} }, null, 2), + ); + writeFileSync( + `${output}/progress.json`, + JSON.stringify( + { complete: true, completed: 0, total: 0, staticCompleted: 0, staticTotal: 0 }, + null, + 2, + ), + ); +}; if (!files.length) { - summary(`## ${workspace.name}: ${values.check}\n\nNo changed source files.\n`); + if (shard) { + writeFileSync(`${output}/all-functions.json`, '[]'); + writeFileSync(`${output}/functions.json`, '[]'); + emptyMutationReports(); + } + summary( + `## ${workspace.name}: ${values.check}${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No changed source files.'}\n`, + ); process.exit(0); } // Test binaries and configuration belong to the measured head, not the tooling checkout. @@ -106,10 +138,17 @@ if (values.check === 'risk') { } else { process.env.NODE_ENV = env.NODE_ENV; process.env.BABEL_ENV = env.BABEL_ENV; - const targets = mutationSelection(files, selection.base); + const allTargets = mutationSelection(files, selection.base); + const targets = shardTargets(allTargets, shard); + if (shard) { + writeFileSync(`${output}/all-functions.json`, JSON.stringify(allTargets, null, 2)); + } writeFileSync(`${output}/functions.json`, JSON.stringify(targets, null, 2)); if (!targets.length) { - summary(`## ${workspace.name}: mutation testing\n\nNo added or edited functions.\n`); + if (shard) emptyMutationReports(); + summary( + `## ${workspace.name}: mutation testing${shardLabel}\n\n${shard ? 'No changed functions for shard.' : 'No added or edited functions.'}\n`, + ); process.exit(0); } const started = Date.now(); @@ -176,7 +215,7 @@ if (values.check === 'risk') { const score = tested ? `${((100 * detected) / tested).toFixed(2)}%` : 'N/A (no valid mutants)'; summary( [ - `## ${workspace.name}: mutation testing (advisory)`, + `## ${workspace.name}: mutation testing (advisory)${shardLabel}`, '', `Changed functions: ${targets.reduce((total, target) => total + target.functions.length, 0)} in ${targets.length} files. Static mutations are included. Duration: ${((Date.now() - started) / 60000).toFixed(1)} minutes.`, '', @@ -198,7 +237,7 @@ if (values.check === 'risk') { if (runtimeErrors) process.exitCode = 1; } catch (error) { summary( - `## ${workspace.name}: mutation testing\n\nStryker failed. This is not a mutation score. See the test log.\n`, + `## ${workspace.name}: mutation testing${shardLabel}\n\nStryker failed. This is not a mutation score. See the test log.\n`, ); throw error; } diff --git a/scripts/quality/shard.mts b/scripts/quality/shard.mts new file mode 100644 index 0000000000..564d4a6f9c --- /dev/null +++ b/scripts/quality/shard.mts @@ -0,0 +1,30 @@ +export interface Shard { + index: number; + total: number; +} + +export function parseShard( + value: string | undefined, + check: string | undefined, +): Shard | undefined { + if (value === undefined) return undefined; + if (check !== 'mutation') throw new Error('--shard is only allowed for mutation checks'); + if (!/^[1-9]\d*\/[1-9]\d*$/u.test(value)) { + throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT'); + } + const [index, total] = value.split('/').map(Number); + if (!Number.isSafeInteger(index) || !Number.isSafeInteger(total) || index > total) { + throw new Error('--shard must be INDEX/COUNT with positive integers and INDEX <= COUNT'); + } + return { index, total }; +} + +export function shardTargets( + targets: readonly T[], + shard?: Shard, +): readonly T[] { + if (!shard) return targets; + return [...targets] + .sort((a, b) => b.functions.length - a.functions.length || a.file.localeCompare(b.file)) + .filter((_target, index) => index % shard.total === shard.index - 1); +} diff --git a/scripts/quality/shard.test.mts b/scripts/quality/shard.test.mts new file mode 100644 index 0000000000..b58e313733 --- /dev/null +++ b/scripts/quality/shard.test.mts @@ -0,0 +1,234 @@ +import { test } from 'node:test'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { strict as assert } from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; + +import { parseShard, shardTargets } from './shard.mts'; +import { mutationTarget } from './risk.mts'; +import { git } from './changes.mts'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const runner = fileURLToPath(new URL('./run.mts', import.meta.url)); + +const targets = Object.freeze( + Array.from({ length: 8 }, (_value, index) => + Object.freeze( + mutationTarget( + `file-${index}.ts`, + 'const first = (ok) => ok ? true : false;\nconst second = () => false;', + [{ start: 1, count: 2 }], + ), + ), + ), +); + +for (const [index, expected] of [ + [1, [0, 3, 6]], + [2, [1, 4, 7]], + [3, [2, 5]], +] as const) { + test(`shard ${index}/3 retains whole files and their full function and mutation ranges`, () => { + const shard = parseShard(`${index}/3`, 'mutation'); + const chosen = shardTargets(targets, shard); + assert.deepEqual( + chosen, + expected.map((offset) => targets[offset]), + ); + for (const [position, offset] of expected.entries()) { + assert.equal(chosen[position], targets[offset]); + assert.equal(chosen[position].functions.length, 2); + assert.equal(chosen[position].mutate, targets[offset].mutate); + } + assert.deepEqual(shardTargets(targets, shard), chosen); + }); +} + +test('three shards contain every full target exactly once without mutating input order', () => { + const before = JSON.stringify(targets); + const union = [1, 2, 3].flatMap((index) => shardTargets(targets, { index, total: 3 })); + assert.equal(union.length, targets.length); + assert.equal(new Set(union).size, targets.length); + assert.deepEqual(new Set(union), new Set(targets)); + assert.deepEqual( + union.flatMap(({ mutate }) => mutate).sort(), + targets.flatMap(({ mutate }) => mutate).sort(), + ); + assert.equal(JSON.stringify(targets), before); +}); + +test('omitting --shard retains all original targets and allows risk checks', () => { + assert.equal(parseShard(undefined, 'mutation'), undefined); + assert.equal(parseShard(undefined, 'risk'), undefined); + assert.equal(shardTargets(targets), targets); + assert.deepEqual(shardTargets(targets, parseShard('1/1', 'mutation')), targets); +}); + +test('equal function counts use file order regardless of input order without mutating input', () => { + const reversed = Object.freeze([...targets].reverse()); + const before = JSON.stringify(reversed); + for (const index of [1, 2, 3]) { + const shard = parseShard(`${index}/3`, 'mutation'); + assert.deepEqual(shardTargets(reversed, shard), shardTargets(targets, shard)); + } + assert.equal(JSON.stringify(reversed), before); + assert.equal(shardTargets(reversed), reversed); + assert.deepEqual(shardTargets(reversed, parseShard('1/1', 'mutation')), targets); +}); + +test('function-weighted round robin spreads the known target file workload across three shards', () => { + const workload = Object.freeze( + ( + [ + ['pool.ts', 25], + ['import.ts', 19], + ['nativefavorite.ts', 13], + ['mongo.ts', 11], + ['migrations.ts', 9], + ['user.ts', 8], + ['backend.ts', 3], + ['favorites/mongo.ts', 1], + ['methods/createMethods.ts', 1], + ] as const + ).map(([file, count]) => + Object.freeze({ + file, + functions: Object.freeze(Array.from({ length: count }, (_value, index) => index)), + }), + ), + ); + const before = JSON.stringify(workload); + const expected = [ + ['pool.ts', 'mongo.ts', 'backend.ts'], + ['import.ts', 'migrations.ts', 'favorites/mongo.ts'], + ['nativefavorite.ts', 'user.ts', 'methods/createMethods.ts'], + ]; + const shuffled = Object.freeze([...workload].reverse()); + const shards = [1, 2, 3].map((index) => { + const shard = parseShard(`${index}/3`, 'mutation'); + const chosen = shardTargets(shuffled, shard); + assert.deepEqual( + chosen.map(({ file }) => file), + expected[index - 1], + ); + assert.deepEqual(chosen, shardTargets(workload, shard)); + return chosen; + }); + const union = shards.flat(); + assert.equal(union.length, workload.length); + assert.equal(new Set(union).size, workload.length); + assert.deepEqual(new Set(union), new Set(workload)); + assert.deepEqual( + shards.map((chosen) => chosen.reduce((sum, target) => sum + target.functions.length, 0)), + [39, 29, 22], + ); + assert.equal(JSON.stringify(workload), before); + assert.equal(shardTargets(shuffled), shuffled); +}); + +test('a shard beyond the number of selected target files is empty', () => { + assert.deepEqual(shardTargets(targets.slice(0, 1), parseShard('2/3', 'mutation')), []); + assert.deepEqual(shardTargets([], parseShard('1/3', 'mutation')), []); +}); + +for (const value of [ + '0/2', + '4/3', + 'foo', + '1/0', + '1.1/2', + '1/2/3', + '', + '-1/2', + '1/-2', + '01/2', + '1/02', + '+1/2', + ' 1/2', + '1/2 ', + '1e1/20', + '1/9007199254740992', +]) { + test(`invalid shard ${JSON.stringify(value)} is rejected`, () => { + assert.throws(() => parseShard(value, 'mutation'), /--shard must be INDEX\/COUNT/u); + }); +} + +test('risk checks reject --shard before attempting source selection', () => { + assert.throws(() => parseShard('1/3', 'risk'), /only allowed for mutation/u); + const result = spawnSync( + process.execPath, + [ + runner, + '--workspace', + 'data-schemas', + '--check', + 'risk', + '--shard', + '1/3', + '--base', + 'nonexistent-shard-test-ref', + ], + { cwd: root, encoding: 'utf8', timeout: 15000 }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /--shard is only allowed for mutation checks/u); + assert.doesNotMatch(result.stderr, /Not a valid object name/u); +}); + +test('empty sharded CLI runs pass with truthful metadata; unsharded output stays unchanged', (t) => { + const directory = mkdtempSync(join(tmpdir(), 'quality-shard-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const env = { + ...process.env, + GIT_DIR: git(['rev-parse', '--absolute-git-dir'], root).trim(), + GIT_WORK_TREE: directory, + GITHUB_STEP_SUMMARY: '', + }; + const output = join(directory, 'coverage/changes/data-schemas/mutation'); + for (const shard of ['2/3', undefined]) { + const result = spawnSync( + process.execPath, + [ + runner, + '--workspace', + 'data-schemas', + '--check', + 'mutation', + '--base', + 'HEAD', + ...(shard ? ['--shard', shard] : []), + ], + { cwd: directory, env, encoding: 'utf8', timeout: 15000 }, + ); + assert.equal(result.status, 0, `${result.error ?? ''}\n${result.stdout}\n${result.stderr}`); + const selection = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8')); + assert.equal(selection.workspace, 'data-schemas'); + assert.deepEqual(selection.files, []); + if (shard) { + assert.deepEqual(selection.shard, { index: 2, total: 3 }); + assert.deepEqual(JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')), { + schemaVersion: '1.0', + files: {}, + }); + assert.deepEqual(JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')), { + complete: true, + completed: 0, + total: 0, + staticCompleted: 0, + staticTotal: 0, + }); + assert.deepEqual(JSON.parse(readFileSync(join(output, 'all-functions.json'), 'utf8')), []); + assert.deepEqual(JSON.parse(readFileSync(join(output, 'functions.json'), 'utf8')), []); + assert.match(result.stdout, /mutation \(shard 2\/3\)/u); + assert.match(result.stdout, /No changed functions for shard\./u); + } else { + assert.equal(selection.shard, undefined); + assert.equal(existsSync(join(output, 'all-functions.json')), false); + assert.equal(existsSync(join(output, 'functions.json')), false); + assert.equal(result.stdout, '## data-schemas: mutation\n\nNo changed source files.\n\n'); + } + } +}); diff --git a/scripts/quality/workflow.test.mts b/scripts/quality/workflow.test.mts index 017944f03c..95af5e458d 100644 --- a/scripts/quality/workflow.test.mts +++ b/scripts/quality/workflow.test.mts @@ -26,8 +26,15 @@ interface Step { run?: string; with?: Record; 'working-directory'?: string; + if?: string; + env?: Record; } interface Job { + name?: string; + needs?: string[]; + if?: string; + 'timeout-minutes'?: number | string; + strategy?: { matrix: { report?: { check: string; shard: number }[]; workspace: string } }; defaults?: { run: { 'working-directory': string } }; steps: Step[]; } @@ -84,13 +91,60 @@ test('workflow separates tooling, source, installs and artifacts without changin ], ); assert.equal(runner['working-directory'], undefined); - assert.equal( - runner.run, - 'node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK"', + assert.match(runner.run!, /args=\(\)/); + assert.match(runner.run!, /--shard "\$QUALITY_SHARD"/); + assert.match(runner.run!, /--base "\$QUALITY_BASE" --head "\$QUALITY_HEAD"/); + assert.match( + runner.run!, + /--workspace "\$QUALITY_WORKSPACE" --check "\$QUALITY_CHECK" "\$\{args\[@\]\}"/, ); assert.equal( report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path, - 'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.check }}/', + 'source/coverage/changes/${{ matrix.workspace }}/${{ matrix.report.check }}/', + ); +}); + +test('mutation shards retain the fixed budget and aggregate all evidence before the existing gate passes', () => { + assert.equal(report['timeout-minutes'], "${{ matrix.report.check == 'mutation' && 240 || 45 }}"); + assert.deepEqual(report.strategy?.matrix.report, [ + { check: 'risk', shard: 0 }, + { check: 'mutation', shard: 1 }, + { check: 'mutation', shard: 2 }, + { check: 'mutation', shard: 3 }, + ]); + assert.equal(runner.env?.QUALITY_CHECK, '${{ matrix.report.check }}'); + assert.equal( + runner.env?.QUALITY_SHARD, + "${{ matrix.report.shard && format('{0}/3', matrix.report.shard) || '' }}", + ); + const aggregate = workflow.jobs.mutation; + assert.equal(aggregate.name, 'Mutation testing (${{ matrix.workspace }})'); + assert.deepEqual(aggregate.needs, ['select', 'reports']); + assert.match(aggregate.if!, /^always\(\)/); + assert.equal(checkouts(aggregate)[0].with?.ref, '${{ github.sha }}'); + assert.equal( + aggregate.steps.some(({ run }) => run?.includes('npm ci')), + false, + ); + const download = aggregate.steps.find(({ uses }) => uses === 'actions/download-artifact@v7'); + assert.equal( + download?.with?.pattern, + 'change-mutation-${{ matrix.workspace }}-shard-*-${{ env.QUALITY_HEAD }}', + ); + assert.equal(download?.with?.['merge-multiple'], undefined); + const verify = aggregate.steps.findIndex(({ name }) => name === 'Verify all mutation shards'); + const requireSuccess = aggregate.steps.findIndex( + ({ name }) => name === 'Require successful report jobs', + ); + assert.ok(verify >= 0 && requireSuccess > verify); + assert.match(aggregate.steps[verify].run!, /--head "\$QUALITY_HEAD"/); + assert.match(aggregate.steps[verify].run!, /--count 3/); + assert.equal(aggregate.steps[requireSuccess].run, 'test "$REPORT_RESULT" = success'); + const upload = aggregate.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6'); + assert.equal(upload?.if, 'always()'); + assert.equal( + upload?.with?.name, + 'change-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}', ); }); @@ -189,6 +243,7 @@ for (const olderHead of [true, false]) { ...process.env, QUALITY_HEAD: head, QUALITY_WORKSPACE: 'data-schemas', + QUALITY_SHARD: '', GITHUB_OUTPUT: '', GITHUB_STEP_SUMMARY: '', }; @@ -249,6 +304,51 @@ for (const olderHead of [true, false]) { const progress = JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8')); assert.equal(progress.complete, true); assert.ok(progress.completed > 0); + const unsharded = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')); + const identify = (report: { files: Record }) => + Object.entries(report.files) + .flatMap(([file, { mutants }]) => + mutants.map((value) => { + const mutant = value as { + location: unknown; + mutatorName: string; + replacement: string; + }; + return JSON.stringify([ + file, + mutant.location, + mutant.mutatorName, + mutant.replacement, + ]); + }), + ) + .sort(); + const shardReports = join(directory, 'mutation-reports'); + const selectedMutants: string[] = []; + for (let index = 1; index <= 3; index++) { + run(runner.run!, source, { + QUALITY_BASE: base, + QUALITY_CHECK: 'mutation', + QUALITY_SHARD: `${index}/3`, + }); + const shard = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8')); + assert.deepEqual(shard.shard, { index, total: 3 }); + assert.equal(shard.workspace, 'data-schemas'); + const mutation = JSON.parse(readFileSync(join(output, 'mutation.json'), 'utf8')); + selectedMutants.push(...identify(mutation)); + cpSync(output, join(shardReports, `shard-${index}`), { recursive: true }); + } + assert.deepEqual(selectedMutants.sort(), identify(unsharded)); + const aggregateOutput = join(directory, 'aggregate'); + run( + `node ../tooling/scripts/quality/aggregate.mts --directory "${shardReports}" --head "${head}" --workspace data-schemas --count 3 --output "${aggregateOutput}"`, + source, + {}, + ); + const aggregate = JSON.parse(readFileSync(join(aggregateOutput, 'aggregate.json'), 'utf8')); + assert.equal(aggregate.complete, true); + assert.equal(aggregate.head, head); + assert.equal(aggregate.total, selectedMutants.length); } assert.equal(git(['rev-parse', 'HEAD'], source).trim(), head); assert.equal(git(['diff', 'HEAD', '--', file], source), '');