fix: finish scheduled chat lifecycle hardening

api/server/controllers/UserController.js

@@ -594,6 +594,10 @@ const quiesceInteractiveGenerations = async (user) => {
     user.id,
     user.tenantId,
   );
+  // A writer whose replica temporarily loses the primary lease store promotes its
+  // hold into Mongo before the primary TTL can expire. Read that fallback before the
+  // active scan for the same lease-to-job handoff ordering used above.
+  const earlyFallbackOwnerWork = await db.countUserFinalizationFallbackLeases(user.id);
   const activeJobIds = await GenerationJobManager.getActiveJobIdsForUser(user.id, user.tenantId);
   for (const streamId of activeJobIds ?? []) {
     // FENCE BEFORE THE CAS. An abort transitions the shared job to terminal, which
@@ -659,6 +663,16 @@ const quiesceInteractiveGenerations = async (user) => {
     );
     return false;
   }
+  const fallbackOwnerWork =
+    earlyFallbackOwnerWork > 0
+      ? earlyFallbackOwnerWork
+      : await db.countUserFinalizationFallbackLeases(user.id);
+  if (fallbackOwnerWork > 0) {
+    logger.info(
+      `[quiesceInteractiveGenerations] ${fallbackOwnerWork} fallback finalization(s) still landing for ${user.id}`,
+    );
+    return false;
+  }
   const fences = await db.getUserAbortFences(user.id);
   if (fences.length === 0) {
     return true;

api/server/controllers/UserController.spec.js

@@ -32,6 +32,7 @@ jest.mock('~/models', () => {
     addUserAbortFence: jest.fn().mockResolvedValue(undefined),
     clearUserAbortFence: jest.fn().mockResolvedValue(undefined),
     getUserAbortFences: jest.fn().mockResolvedValue([]),
+    countUserFinalizationFallbackLeases: jest.fn().mockResolvedValue(0),
     deleteAllUserSessions: jest.fn().mockResolvedValue(undefined),
     deleteAllSharedLinks: jest.fn().mockResolvedValue(undefined),
     deleteAllAgentApiKeys: jest.fn().mockResolvedValue(undefined),

api/server/controllers/__tests__/deleteUser.spec.js

@@ -97,6 +97,7 @@ jest.mock('~/models', () => ({
   addUserAbortFence: jest.fn().mockResolvedValue(undefined),
   clearUserAbortFence: jest.fn().mockResolvedValue(undefined),
   getUserAbortFences: jest.fn().mockResolvedValue([]),
+  countUserFinalizationFallbackLeases: jest.fn().mockResolvedValue(0),
   deleteTransactions: jest.fn(),
   deleteBalances: jest.fn(),
   deleteAllAgentApiKeys: jest.fn(),
@@ -423,6 +424,20 @@ describe('deleteUserController - interactive generation quiesce', () => {
     expect(mockDeleteUserById).not.toHaveBeenCalled();
   });
 
+  it('defers while a Mongo fallback finalization lease is still live', async () => {
+    const req = { user: { id: 'user1', _id: 'user1', email: 'a@b.com' }, body: {} };
+    const res = createRes();
+    const db = require('~/models');
+    mockGetUserById.mockResolvedValue({ _id: 'user1', twoFactorEnabled: false });
+    db.countUserFinalizationFallbackLeases.mockResolvedValueOnce(1);
+
+    await deleteUserController(req, res);
+
+    expect(res.status).toHaveBeenCalledWith(503);
+    expect(mockDeleteMessages).not.toHaveBeenCalled();
+    expect(mockDeleteUserById).not.toHaveBeenCalled();
+  });
+
   it('fences an abort whose signal never left this replica and re-signals it', async () => {
     const req = { user: { id: 'user1', _id: 'user1', email: 'a@b.com' }, body: {} };
     const res = createRes();
@@ -558,6 +573,10 @@ describe('deleteUserController - interactive generation quiesce', () => {
     expect(GenerationJobManager.countUserFinalizations.mock.invocationCallOrder[0]).toBeLessThan(
       GenerationJobManager.getActiveJobIdsForUser.mock.invocationCallOrder[0],
     );
+    const db = require('~/models');
+    expect(db.countUserFinalizationFallbackLeases.mock.invocationCallOrder[0]).toBeLessThan(
+      GenerationJobManager.getActiveJobIdsForUser.mock.invocationCallOrder[0],
+    );
   });
 
   it('defers on an admission lease observed before the job scan', async () => {

api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js

@@ -2025,6 +2025,7 @@ describe('ResumableAgentController resume metadata', () => {
         error: 'Attached resources could not be restored',
       }),
       1000,
+      { preserveForReconcile: false },
     );
   });
 

api/server/experimental.js

@@ -53,9 +53,14 @@ const {
   markEraseAttempted,
   getActiveRunsForSchedule,
   recordRunOutcome,
+  renewUserFinalizationFallbackLease,
+  clearUserFinalizationFallbackLease,
+  addUserAbortFence,
+  clearUserAbortFence,
 } = require('~/models');
 const { checkMigrations } = require('./services/start/migration');
 const { configureGenerationStreams } = require('@librechat/api');
+const { createUserFinalizationFallbackStore } = require('./services/UserFinalizationFallback');
 const initializeMCPs = require('./services/initializeMCPs');
 const configureSocialLogins = require('./socialLogins');
 const createSpaFallback = require('./utils/fallback');
@@ -423,7 +428,14 @@ if (cluster.isMaster) {
     // store even with USE_REDIS_STREAMS — making the multiworker scheduler private,
     // so cross-worker aborts and orphan recovery could not work. Shared with
     // api/server/index.js so both topologies initialize identically.
-    configureGenerationStreams();
+    configureGenerationStreams(
+      createUserFinalizationFallbackStore({
+        renewUserFinalizationFallbackLease,
+        clearUserFinalizationFallbackLease,
+        addUserAbortFence,
+        clearUserAbortFence,
+      }),
+    );
     // Same teardown discipline as api/server/index.js: stop active generations and
     // close their SSE streams while the HTTP server drains, then release stream
     // resources. Without this a killed worker leaves its Redis jobs `running` until

api/server/index.js

@@ -50,6 +50,10 @@ const {
   sweepOrphanedPreviews,
   getRoleByName,
   seedDatabase,
+  renewUserFinalizationFallbackLease,
+  clearUserFinalizationFallbackLease,
+  addUserAbortFence,
+  clearUserAbortFence,
 } = require('~/models');
 const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
 const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');
@@ -127,7 +131,14 @@ const rejectScheduleWritesUntilReady = (req, res, next) => {
 const configureGenerationStreams = () => {
   // Shared with the clustered entrypoint (experimental.js) so both topologies get the
   // same stream services; returns whether the resulting store is genuinely shared.
-  const isShared = configureSharedGenerationStreams();
+  const isShared = configureSharedGenerationStreams(
+    createUserFinalizationFallbackStore({
+      renewUserFinalizationFallbackLease,
+      clearUserFinalizationFallbackLease,
+      addUserAbortFence,
+      clearUserAbortFence,
+    }),
+  );
   // Stop active generations and close their SSE streams while the HTTP server drains.
   registerShutdownTask(
     'generation job manager prepare',

api/server/services/UserFinalizationFallback.js

@@ -0,0 +1,39 @@
+const { tenantStorage, runAsSystem } = require('@librechat/data-schemas');
+
+/**
+ * Generation lifecycle callbacks run from timers and Redis handlers, where there
+ * is no request-scoped tenant context. Re-establish the exact user's tenant for
+ * tenant-isolated User writes; base users deliberately use the explicit system
+ * context rather than inheriting an unrelated request context.
+ */
+const runForUserTenant = (userId, tenantId, operation) => {
+  const run = async () => operation();
+  return tenantId ? tenantStorage.run({ tenantId, userId }, run) : runAsSystem(run);
+};
+
+/**
+ * Adapts User model methods to the process-level generation lifecycle store.
+ * Keeping this adapter shared ensures the standard and clustered entrypoints
+ * apply identical strict-tenant behavior.
+ */
+const createUserFinalizationFallbackStore = ({
+  renewUserFinalizationFallbackLease,
+  clearUserFinalizationFallbackLease,
+  addUserAbortFence,
+  clearUserAbortFence,
+}) => ({
+  renew: (userId, tenantId, safeLeaseKey, expiresAt) =>
+    runForUserTenant(userId, tenantId, () =>
+      renewUserFinalizationFallbackLease(userId, safeLeaseKey, expiresAt),
+    ),
+  clear: (userId, tenantId, safeLeaseKey) =>
+    runForUserTenant(userId, tenantId, () =>
+      clearUserFinalizationFallbackLease(userId, safeLeaseKey),
+    ),
+  retainAbortDelivery: (userId, tenantId, streamId) =>
+    runForUserTenant(userId, tenantId, () => addUserAbortFence(userId, streamId)),
+  clearAbortDelivery: (userId, tenantId, streamId) =>
+    runForUserTenant(userId, tenantId, () => clearUserAbortFence(userId, streamId)),
+});
+
+module.exports = { createUserFinalizationFallbackStore };

api/server/services/UserFinalizationFallback.spec.js

@@ -0,0 +1,71 @@
+const {
+  getTenantId,
+  getUserId,
+  tenantStorage,
+  SYSTEM_TENANT_ID,
+} = require('@librechat/data-schemas');
+const { createUserFinalizationFallbackStore } = require('./UserFinalizationFallback');
+
+describe('UserFinalizationFallback startup adapter', () => {
+  const makeStrictUserMethod = (contexts) =>
+    jest.fn(async () => {
+      const tenantId = getTenantId();
+      if (!tenantId) {
+        throw new Error('[TenantIsolation] Query attempted without tenant context in strict mode');
+      }
+      contexts.push({ tenantId, userId: getUserId() });
+    });
+
+  test('establishes tenant context for every timer callback with no request context', async () => {
+    const contexts = [];
+    const renewUserFinalizationFallbackLease = makeStrictUserMethod(contexts);
+    const clearUserFinalizationFallbackLease = makeStrictUserMethod(contexts);
+    const addUserAbortFence = makeStrictUserMethod(contexts);
+    const clearUserAbortFence = makeStrictUserMethod(contexts);
+    const store = createUserFinalizationFallbackStore({
+      renewUserFinalizationFallbackLease,
+      clearUserFinalizationFallbackLease,
+      addUserAbortFence,
+      clearUserAbortFence,
+    });
+
+    expect(tenantStorage.getStore()).toBeUndefined();
+
+    await store.renew('user-1', 'tenant-a', 'fallback_1', new Date());
+    await store.clear('user-1', 'tenant-a', 'fallback_1');
+    await store.retainAbortDelivery('user-1', 'tenant-a', 'stream-1');
+    await store.clearAbortDelivery('user-1', 'tenant-a', 'stream-1');
+
+    expect(contexts).toEqual([
+      { tenantId: 'tenant-a', userId: 'user-1' },
+      { tenantId: 'tenant-a', userId: 'user-1' },
+      { tenantId: 'tenant-a', userId: 'user-1' },
+      { tenantId: 'tenant-a', userId: 'user-1' },
+    ]);
+    expect(renewUserFinalizationFallbackLease).toHaveBeenCalledWith(
+      'user-1',
+      'fallback_1',
+      expect.any(Date),
+    );
+    expect(clearUserFinalizationFallbackLease).toHaveBeenCalledWith('user-1', 'fallback_1');
+    expect(addUserAbortFence).toHaveBeenCalledWith('user-1', 'stream-1');
+    expect(clearUserAbortFence).toHaveBeenCalledWith('user-1', 'stream-1');
+  });
+
+  test('uses explicit system context for tenantless users instead of ambient request context', async () => {
+    const contexts = [];
+    const renewUserFinalizationFallbackLease = makeStrictUserMethod(contexts);
+    const store = createUserFinalizationFallbackStore({
+      renewUserFinalizationFallbackLease,
+      clearUserFinalizationFallbackLease: jest.fn(),
+      addUserAbortFence: jest.fn(),
+      clearUserAbortFence: jest.fn(),
+    });
+
+    await tenantStorage.run({ tenantId: 'unrelated-tenant', userId: 'other-user' }, async () =>
+      store.renew('base-user', undefined, 'fallback_base', new Date()),
+    );
+
+    expect(contexts).toEqual([{ tenantId: SYSTEM_TENANT_ID, userId: 'other-user' }]);
+  });
+});

api/strategies/openIdJwtStrategy.spec.js

@@ -532,6 +532,7 @@ describe('openIdJwtStrategy – auth user document cache', () => {
       mockAuthUserDocCacheStore,
       'auth-user-doc-key',
       expect.objectContaining({ id: 'tenant-b-user', tenantId: 'tenant-b' }),
+      expect.objectContaining({ readAt: expect.any(Number) }),
     );
   });
 

client/src/components/SidePanel/Schedules/ScheduleCard.tsx

@@ -6,6 +6,7 @@ import { Play, Trash, Pencil, Ellipsis } from 'lucide-react';
 import { PermissionTypes, Permissions } from 'librechat-data-provider';
 import {
   Label,
+  Chip,
   Switch,
   Spinner,
   OGDialog,
@@ -22,8 +23,8 @@ import {
   useRunScheduleNowMutation,
 } from '~/data-provider';
 import { useLocalize, useHasAccess } from '~/hooks';
-import { getMessageTimestamp, cn } from '~/utils';
 import { useAgentsMapContext } from '~/Providers';
+import { getMessageTimestamp } from '~/utils';
 import ScheduleDialog from './ScheduleDialog';
 import { describeCadence } from './cadence';
 
@@ -31,20 +32,16 @@ interface ScheduleCardProps {
   schedule: TSchedule;
 }
 
-const CHIP_BASE = 'rounded-full px-2 py-0.5 text-xs font-medium';
-const GREEN_CHIP = 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300';
-const RED_CHIP = 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300';
-const AMBER_CHIP = 'bg-amber-100 text-amber-800 dark:bg-amber-900 dark:text-amber-300';
-const NEUTRAL_CHIP = 'bg-surface-tertiary text-text-secondary';
+type StatusTone = 'neutral' | 'success' | 'warning' | 'error';
 
-const STATUS_CHIPS: Record<ScheduleRunStatus, { label: TranslationKeys; className: string }> = {
-  success: { label: 'com_ui_schedule_last_run', className: GREEN_CHIP },
-  error: { label: 'com_ui_schedule_last_run_failed', className: RED_CHIP },
-  interrupted: { label: 'com_ui_schedule_last_run_failed', className: RED_CHIP },
-  requires_action: { label: 'com_ui_schedule_needs_approval', className: AMBER_CHIP },
-  started: { label: 'com_ui_schedule_run_started', className: NEUTRAL_CHIP },
-  skipped_overlap: { label: 'com_ui_schedule_run_skipped', className: NEUTRAL_CHIP },
-  skipped_balance: { label: 'com_ui_schedule_run_skipped', className: NEUTRAL_CHIP },
+const STATUS_CHIPS: Record<ScheduleRunStatus, { label: TranslationKeys; tone: StatusTone }> = {
+  success: { label: 'com_ui_schedule_last_run', tone: 'success' },
+  error: { label: 'com_ui_schedule_last_run_failed', tone: 'error' },
+  interrupted: { label: 'com_ui_schedule_last_run_failed', tone: 'error' },
+  requires_action: { label: 'com_ui_schedule_needs_approval', tone: 'warning' },
+  started: { label: 'com_ui_schedule_run_started', tone: 'neutral' },
+  skipped_overlap: { label: 'com_ui_schedule_run_skipped', tone: 'neutral' },
+  skipped_balance: { label: 'com_ui_schedule_run_skipped', tone: 'neutral' },
 };
 
 const DISABLED_REASON_LABELS: Record<ScheduleDisabledReason, TranslationKeys> = {
@@ -139,6 +136,7 @@ export default function ScheduleCard({ schedule }: ScheduleCardProps) {
         label: localize('com_ui_schedule_run_now'),
         onClick: handleRunNow,
         hideOnClick: false,
+        disabled: runSchedule.isLoading,
         icon: runSchedule.isLoading ? (
           <Spinner className="size-4" />
         ) : (
@@ -173,7 +171,7 @@ export default function ScheduleCard({ schedule }: ScheduleCardProps) {
   return (
     <div
       data-testid="schedule-card"
-      className="rounded-lg border border-border-light bg-transparent px-3 py-2.5 hover:bg-surface-secondary"
+      className="rounded-lg border border-border-light bg-transparent px-3 py-2.5 transition-colors duration-theme-fast hover:bg-surface-secondary"
     >
       <div className="flex items-center gap-2">
         <span className="min-w-0 flex-1 truncate text-sm font-semibold text-text-primary">
@@ -185,7 +183,7 @@ export default function ScheduleCard({ schedule }: ScheduleCardProps) {
               checked={schedule.enabled}
               onCheckedChange={handleToggle}
               disabled={updateSchedule.isLoading}
-              aria-label={localize('com_ui_schedule_enabled')}
+              aria-label={`${localize('com_ui_schedule_enabled')}: ${schedule.name}`}
               className="shrink-0"
             />
             <DropdownPopup
@@ -199,7 +197,7 @@ export default function ScheduleCard({ schedule }: ScheduleCardProps) {
               trigger={
                 <Ariakit.MenuButton
                   id={`schedule-menu-${schedule.id}`}
-                  aria-label={localize('com_ui_schedule_options')}
+                  aria-label={`${localize('com_ui_schedule_options')}: ${schedule.name}`}
                   className="inline-flex size-7 shrink-0 items-center justify-center rounded-md text-text-secondary hover:bg-surface-tertiary hover:text-text-primary focus:outline-none focus-visible:ring-2 focus-visible:ring-border-heavy"
                 >
                   <Ellipsis className="size-4" aria-hidden={true} />
@@ -221,20 +219,16 @@ export default function ScheduleCard({ schedule }: ScheduleCardProps) {
             (lastRunConvoId != null && lastRunConvoId !== '' ? (
               <button
                 type="button"
-                className={cn(CHIP_BASE, statusChip.className, 'hover:underline')}
+                className="rounded-full hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-text-primary"
                 onClick={() => navigate(`/c/${lastRunConvoId}`)}
               >
-                {localize(statusChip.label)}
+                <Chip tone={statusChip.tone}>{localize(statusChip.label)}</Chip>
               </button>
             ) : (
-              <span className={cn(CHIP_BASE, statusChip.className)}>
-                {localize(statusChip.label)}
-              </span>
+              <Chip tone={statusChip.tone}>{localize(statusChip.label)}</Chip>
             ))}
           {schedule.disabledReason != null && (
-            <span className={cn(CHIP_BASE, RED_CHIP)}>
-              {localize(DISABLED_REASON_LABELS[schedule.disabledReason])}
-            </span>
+            <Chip tone="error">{localize(DISABLED_REASON_LABELS[schedule.disabledReason])}</Chip>
           )}
         </div>
       )}
@@ -267,7 +261,7 @@ export default function ScheduleCard({ schedule }: ScheduleCardProps) {
           selection={{
             selectHandler: confirmDelete,
             selectClasses:
-              'bg-red-700 dark:bg-red-600 hover:bg-red-800 dark:hover:bg-red-800 text-white',
+              'bg-surface-destructive text-text-on-status hover:bg-surface-destructive-hover',
             selectText: localize('com_ui_delete'),
           }}
         />

client/src/components/SidePanel/Schedules/ScheduleCardSkeleton.tsx

@@ -0,0 +1,24 @@
+import { Skeleton } from '@librechat/client';
+
+/** Mirrors the schedule card's title controls, agent, cadence, and next-run rows. */
+export default function ScheduleCardSkeleton({ count = 5 }: { count?: number }) {
+  return (
+    <div className="flex flex-col gap-2" aria-hidden="true">
+      {Array.from({ length: count }, (_, index) => (
+        <div
+          key={index}
+          className="rounded-lg border border-border-light bg-transparent px-3 py-2.5"
+        >
+          <div className="flex items-center gap-2">
+            <Skeleton className="h-4 min-w-0 flex-1 rounded" />
+            <Skeleton className="h-5 w-9 shrink-0 rounded-full" />
+            <Skeleton className="size-7 shrink-0 rounded-md" />
+          </div>
+          <Skeleton className="mt-1 h-3 w-2/5 rounded" />
+          <Skeleton className="mt-2 h-4 w-3/4 rounded" />
+          <Skeleton className="mt-1 h-3 w-1/2 rounded" />
+        </div>
+      ))}
+    </div>
+  );
+}

client/src/components/SidePanel/Schedules/ScheduleDialog.tsx

@@ -7,7 +7,9 @@ import {
   Input,
   Label,
   Button,
+  FieldMessage,
   Spinner,
+  Textarea,
   Dropdown,
   OGDialog,
   ControlCombobox,
@@ -123,9 +125,10 @@ export default function ScheduleDialog({
     register,
     watch,
     handleSubmit,
-    formState: { dirtyFields },
+    formState: { dirtyFields, errors },
   } = useForm<ScheduleFormValues>({
     defaultValues: getDefaultValues(schedule),
+    mode: 'onChange',
   });
   /** The revision the form defaults were built from, captured at the same instant.
    *  The `schedule` prop keeps refreshing while the dialog is open (the schedules
@@ -133,9 +136,6 @@ export default function ScheduleDialog({
    *  refreshed revision onto a cadence rebuilt from this stale snapshot — exactly
    *  the overwrite the fence exists to refuse. */
   const openedConfigRevision = useRef(schedule?.configRevision);
-  const name = watch('name');
-  const prompt = watch('prompt');
-  const agentId = watch('agent_id');
   const frequency = watch('frequency');
   const hour12 = watch('hour12');
   const minute = watch('minute');
@@ -316,7 +316,6 @@ export default function ScheduleDialog({
     resolvePreservedWeeklyDays(frequency),
   );
   const summary = `${describeCadence(summaryCadence, localize, locale)} · ${timezone}`;
-  const canSubmit = name.trim().length > 0 && prompt.trim().length > 0 && agentId.length > 0;
 
   return (
     <OGDialog open={open} onOpenChange={onOpenChange} triggerRef={triggerRef}>
@@ -333,28 +332,34 @@ export default function ScheduleDialog({
               <Input
                 id="schedule-name"
                 className="w-full"
-                {...register('name', { required: true })}
+                aria-invalid={errors.name != null}
+                aria-describedby="schedule-name-message"
+                {...register('name', { required: localize('com_ui_field_required') })}
               />
+              <FieldMessage id="schedule-name-message" message={errors.name?.message} />
             </div>
             <div className="space-y-2">
               <Label htmlFor="schedule-prompt" className="text-sm font-medium text-text-primary">
                 {localize('com_ui_prompt')}
               </Label>
-              <textarea
+              <Textarea
                 id="schedule-prompt"
                 rows={4}
-                className="min-h-[100px] w-full resize-none rounded-lg border border-border-light bg-transparent px-3 py-2 text-sm text-text-primary focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-border-heavy"
-                {...register('prompt', { required: true })}
+                className="min-h-[100px] w-full resize-none"
+                aria-invalid={errors.prompt != null}
+                aria-describedby="schedule-prompt-message"
+                {...register('prompt', { required: localize('com_ui_field_required') })}
               />
+              <FieldMessage id="schedule-prompt-message" message={errors.prompt?.message} />
             </div>
             <div className="space-y-2">
-              <Label className="text-sm font-medium text-text-primary">
+              <Label htmlFor="schedule-agent" className="text-sm font-medium text-text-primary">
                 {localize('com_ui_agent')}
               </Label>
               <Controller
                 name="agent_id"
                 control={control}
-                rules={{ required: true }}
+                rules={{ required: localize('com_ui_field_required') }}
                 render={({ field }) => (
                   <ControlCombobox
                     selectedValue={field.value}
@@ -364,8 +369,12 @@ export default function ScheduleDialog({
                     selectPlaceholder={localize('com_ui_select_agent')}
                     searchPlaceholder={localize('com_agents_search_name')}
                     setValue={field.onChange}
+                    onBlur={field.onBlur}
                     items={agentItems}
                     ariaLabel={localize('com_ui_agent')}
+                    ariaInvalid={errors.agent_id != null}
+                    ariaDescribedBy="schedule-agent-message"
+                    selectId="schedule-agent"
                     isCollapsed={false}
                     showCarat={true}
                     containerClassName="px-0"
@@ -373,6 +382,7 @@ export default function ScheduleDialog({
                   />
                 )}
               />
+              <FieldMessage id="schedule-agent-message" message={errors.agent_id?.message} />
               <p className="text-xs text-text-secondary">
                 {localize('com_ui_schedule_target_new_chat')}
               </p>
@@ -386,7 +396,7 @@ export default function ScheduleDialog({
                 control={control}
                 render={({ field }) => (
                   <div
-                    className="flex gap-1"
+                    className="grid grid-cols-2 gap-1 sm:grid-cols-4"
                     role="group"
                     aria-label={localize('com_ui_schedule_frequency')}
                   >
@@ -437,7 +447,12 @@ export default function ScheduleDialog({
               <Label className="text-sm font-medium text-text-primary">
                 {localize('com_ui_schedule_time')}
               </Label>
-              <div className="flex gap-2">
+              <div
+                className={cn(
+                  'grid gap-2',
+                  frequency === 'hourly' ? 'grid-cols-1' : 'grid-cols-2 sm:grid-cols-3',
+                )}
+              >
                 {frequency !== 'hourly' && (
                   <Controller
                     name="hour12"
@@ -447,7 +462,7 @@ export default function ScheduleDialog({
                         value={String(field.value)}
                         onChange={(value) => field.onChange(Number(value))}
                         options={hourOptions}
-                        className="flex-1"
+                        className="w-full"
                         ariaLabel={localize('com_ui_schedule_hour')}
                         testId="schedule-hour-select"
                       />
@@ -462,7 +477,7 @@ export default function ScheduleDialog({
                       value={String(field.value)}
                       onChange={(value) => field.onChange(Number(value))}
                       options={minuteOptions}
-                      className="flex-1"
+                      className="w-full"
                       ariaLabel={localize('com_ui_schedule_minute')}
                       testId="schedule-minute-select"
                     />
@@ -477,7 +492,7 @@ export default function ScheduleDialog({
                         value={field.value}
                         onChange={field.onChange}
                         options={meridiemOptions}
-                        className="flex-1"
+                        className="w-full"
                         ariaLabel={localize('com_ui_schedule_meridiem')}
                         testId="schedule-meridiem-select"
                       />
@@ -486,7 +501,7 @@ export default function ScheduleDialog({
                 )}
               </div>
             </div>
-            <p className="text-sm text-text-secondary">{summary}</p>
+            <p className="break-words text-sm text-text-secondary">{summary}</p>
           </div>
         }
         buttons={
@@ -494,8 +509,7 @@ export default function ScheduleDialog({
             type="button"
             variant="submit"
             onClick={handleSubmit(onSubmit)}
-            disabled={isLoading || !canSubmit}
-            className="text-white"
+            disabled={isLoading}
             aria-label={localize(schedule ? 'com_ui_save' : 'com_ui_create')}
           >
             {isLoading ? (

client/src/components/SidePanel/Schedules/SchedulePanel.tsx

@@ -1,7 +1,9 @@
-import { useState } from 'react';
+import { useState, type ReactNode } from 'react';
 import { Plus } from 'lucide-react';
-import { Button, Spinner, TooltipAnchor } from '@librechat/client';
+import { Button, TooltipAnchor } from '@librechat/client';
 import { PermissionTypes, Permissions } from 'librechat-data-provider';
+import { PanelContent, PanelFooter } from '~/components/ui';
+import ScheduleCardSkeleton from './ScheduleCardSkeleton';
 import { useSchedulesQuery } from '~/data-provider';
 import { useLocalize, useHasAccess } from '~/hooks';
 import ScheduleDialog from './ScheduleDialog';
@@ -17,38 +19,47 @@ export default function SchedulePanel() {
     permission: Permissions.CREATE,
   });
 
-  if (isLoading) {
-    return (
-      <div className="flex h-full w-full items-center justify-center p-4">
-        <Spinner />
-      </div>
-    );
-  }
+  const schedules = data?.schedules ?? [];
+  const maxPerUser = data?.limits.maxPerUser;
+  const atLimit = maxPerUser !== undefined && schedules.length >= maxPerUser;
+  let panelContent: ReactNode;
 
-  // A failed query leaves `data` undefined, which the empty-state branch below would
-  // render as "no scheduled chats yet" — telling the user their schedules are gone when
-  // the request merely failed. `maxPerUser` is unknown too, so the create button would
-  // stay enabled and any create would 4xx against a limit we cannot see.
   if (isError) {
-    return (
-      <div className="flex h-full w-full flex-col items-center justify-center gap-2 p-4">
+    panelContent = (
+      <div className="flex min-h-40 flex-col items-center justify-center gap-2 p-4">
         <p className="text-center text-sm text-text-secondary">
           {localize('com_ui_schedules_error')}
         </p>
-        <Button variant="outline" size="sm" onClick={() => refetch()}>
+        <Button type="button" variant="outline" size="sm" onClick={() => refetch()}>
           {localize('com_ui_retry')}
         </Button>
       </div>
     );
+  } else if (schedules.length === 0) {
+    panelContent = (
+      <div className="rounded-lg border border-dashed border-border-light p-4 text-center">
+        <p className="text-sm text-text-secondary">{localize('com_ui_schedules_empty')}</p>
+      </div>
+    );
+  } else {
+    panelContent = (
+      <div className="space-y-2" role="list" aria-label={localize('com_ui_schedules')}>
+        {schedules.map((schedule) => (
+          <div key={schedule.id} role="listitem">
+            <ScheduleCard schedule={schedule} />
+          </div>
+        ))}
+      </div>
+    );
   }
 
-  const schedules = data?.schedules ?? [];
-  const maxPerUser = data?.limits.maxPerUser;
-  const atLimit = maxPerUser !== undefined && schedules.length >= maxPerUser;
-
   return (
-    <div className="flex h-auto w-full flex-col px-3 pb-3 pt-2">
-      <div role="region" aria-label={localize('com_ui_schedules')} className="space-y-2">
+    <div
+      role="region"
+      aria-label={localize('com_ui_schedules')}
+      className="flex h-full w-full flex-col overflow-hidden pt-2"
+    >
+      <div className="shrink-0 px-3 pb-2">
         <div className="flex items-center justify-between gap-2">
           <span className="text-sm font-medium text-text-primary">
             {localize('com_ui_schedules')}
@@ -63,7 +74,7 @@ export default function SchedulePanel() {
                   size="icon"
                   className="size-9 shrink-0 bg-transparent"
                   aria-label={localize('com_ui_schedule_new')}
-                  disabled={atLimit}
+                  disabled={atLimit || isError || isLoading}
                   onClick={() => setCreateOpen(true)}
                 >
                   <Plus className="size-4" aria-hidden="true" />
@@ -72,24 +83,23 @@ export default function SchedulePanel() {
             />
           )}
         </div>
-        {schedules.length === 0 ? (
-          <div className="rounded-lg border border-dashed border-border-light p-4 text-center">
-            <p className="text-sm text-text-secondary">{localize('com_ui_schedules_empty')}</p>
-          </div>
-        ) : (
-          <div className="space-y-2">
-            {schedules.map((schedule) => (
-              <ScheduleCard key={schedule.id} schedule={schedule} />
-            ))}
-          </div>
-        )}
-        {maxPerUser !== undefined && (
+      </div>
+
+      <PanelContent isLoading={isLoading} skeleton={<ScheduleCardSkeleton />} className="px-3 pb-3">
+        {/* A failed query must not masquerade as an empty list or enable creation
+            against limits that could not be loaded. */}
+        {panelContent}
+      </PanelContent>
+
+      {!isLoading && !isError && maxPerUser !== undefined && (
+        <PanelFooter className="justify-start">
           <p className="text-xs text-text-secondary">
             {localize('com_ui_schedules_used', { used: schedules.length, max: maxPerUser })}
           </p>
-        )}
-        {createOpen && <ScheduleDialog open={createOpen} onOpenChange={setCreateOpen} />}
-      </div>
+        </PanelFooter>
+      )}
+
+      {createOpen && <ScheduleDialog open={createOpen} onOpenChange={setCreateOpen} />}
     </div>
   );
 }

package-lock.json

@@ -23269,6 +23269,25 @@
       "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz",
       "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="
     },
+    "node_modules/croner": {
+      "version": "10.0.1",
+      "resolved": "https://registry.npmjs.org/croner/-/croner-10.0.1.tgz",
+      "integrity": "sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==",
+      "funding": [
+        {
+          "type": "other",
+          "url": "https://paypal.me/hexagonpp"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/hexagon"
+        }
+      ],
+      "license": "MIT",
+      "engines": {
+        "node": ">=18.0"
+      }
+    },
     "node_modules/cross-env": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
@@ -42822,6 +42841,7 @@
         "@langchain/langgraph-checkpoint": "^1.1.2",
         "@langchain/langgraph-checkpoint-mongodb": "^1.4.0",
         "cluster-key-slot": "^1.1.2",
+        "croner": "^10.0.1",
         "proxy-from-env": "^2.1.0"
       },
       "devDependencies": {

packages/api/src/schedules/FOLLOWUPS.md

@@ -1,138 +1,34 @@
 # Scheduled chats: deferred scope
 
-v1 ships an **autonomous, single-process** scheduler, experimental and default-off
-(`interface.schedules`). Two capabilities were deliberately cut to get a correct core;
-both were fully built and are recoverable from git history rather than greenfield.
+Scheduled chats remain experimental and default-off.
 
-## Why they were cut
+## Must-have product follow-ups
 
-Both existed to reconcile **two independent durable state machines** — `GenerationJob`
-(`running → requires_action → running → terminal`) and `ScheduleRun` (`started →
-requires_action → started → terminal`). Every lifecycle event had to project onto both
-with the right fence token, and the token lived on the row while the producer lived in
-the controller. That impedance mismatch, not any single bug, is what kept generating
-drift: three review rounds, and the worst finding in each was resume-related.
+- Add one-time and monthly schedules; current cadences are hourly, daily, weekdays, and weekly.
+- Add non-agent chat targets and existing-conversation continuation; current schedules require
+  `agent_id` and always target `new`.
 
-## 1. HITL resume of scheduled runs
+## Client/API parity
 
-**v1 behavior:** a scheduled run that pauses for approval frees its capacity slot and is
-handed off. It becomes an ordinary paused conversation the user resumes in the chat UI;
-the scheduler observes the outcome but does not reserve, lease, or fence the resume.
-Abandoned pauses are reaped by the reconciler.
+- Return `minIntervalMinutes` in schedule limits and surface it in the dialog.
+- Add active-run and run-history discovery. The list response currently projects only `lastRun`,
+  which is written on pause, skip, or terminal outcomes, so the client's `started` chip is
+  unreachable.
+- Add multi-day weekly, timezone, and attachment controls. The API supports these, but the dialog
+  does not.
 
-**What the fast-follow restores** (see `c188a0f92` for the exact removal):
+## Runtime scope
 
-- `resumeSeq` epoch + the pause CAS
-- the durable resume lease: `acquireResumeLease` / `markResumeClaimed` /
-  `commitResumeLease` / `releaseResumeLease`, with `resumeHolder` / `resumeExpiresAt` /
-  `resumeClaimedAt` / `resumeAdopted`
-- adopt-vs-promote discrimination (a committed run must never be demoted by a late
-  duplicate attempt)
-- the lease-recovery reconciler branch (post-claim rolls forward, pre-claim rolls back)
-- the resume-side capacity reservation
+- The standard server supports multi-replica scheduling only with a confirmed Redis-backed
+  `GenerationJobManager`; in-memory mode requires `SCHEDULES_SINGLE_PROCESS`. The legacy
+  `experimental.js` clustered entrypoint does not arm the engine and rejects schedule writes.
+- Scheduled HITL resumes are durably fenced, but user-driven resumes do not reserve scheduler
+  `fireConcurrency` capacity.
+- Deferred account deletion blocks new remote API admission but cannot yet drain already-admitted
+  OpenAI-compatible or Responses requests; see
+  [#14594](https://github.com/danny-avila/LibreChat/issues/14594).
 
-**Build it differently this time:**
+## Test debt
 
-1. Apply the single-seam pattern from `6c2f4c1d9` FIRST. Derive the pause epoch inside
-   `recordRunOutcome` from the row, exactly as the config fence now is. Do not add
-   `expectResumeSeq` back as a caller-supplied parameter — that is what went inert.
-2. Integration-test through real entry points (initial pause vs fast resume; duplicate
-   submit after commit; approval-claim failure; crash mid-reconstruction). Leaf tests
-   that hand-supply tokens pass for a broken system.
-3. Watch the epoch-0 trap: a never-resumed row has NO `resumeSeq`, and Mongo equality on
-   `0` does not match a missing field (`$in: [0, null]`).
-
-## 2. Clustered / multi-worker scheduling
-
-**v1 behavior:** single-process only. `experimental.js` does not arm the engine and
-refuses schedule writes with `501 SCHEDULES_NOT_SUPPORTED` (reads stay open).
-
-**What the fast-follow restores** (see `f1c3f1d06`):
-
-- `isJobStoreShared` and the `clustered` plumbing
-- cross-worker abort delivery for delete / account-deletion quiescing
-- the `jobStoreShared` gates around orphan and abandoned-pause reaping
-
-**Prerequisite:** a *confirmed* shared stream store, not an assumed one. `USE_REDIS`
-is the wrong signal; the check must be the live `GenerationJobManager.isRedis` read
-AFTER `configureGenerationStreams` (a Redis config can fall back to in-memory).
-
-**Known v1 residual this deferral leaves:** the clustered entrypoint's erasure sweep
-only erases DRAINED rows — it deliberately never settles a run, because settling
-requires job-store visibility a process-local store cannot give. A deleting schedule
-whose abort owner died therefore stays hidden-but-retained until the run-row TTL
-(~90 days) reclaims its runs. Bounded, reachable only via owner-death on a topology
-v1 does not schedule on, and fixed by this fast-follow's reconciliation — not by
-teaching the sweep to settle rows it cannot verify.
-
-**Note:** the shared stream-service initialization (`86255b8c7`) was KEPT in v1. It fixed
-a real bug — clustered workers never configured their stream services at all — and is
-correct independent of scheduling.
-
-## Account deletion: deferred completion
-
-If the quiesce drain cannot be CONFIRMED, `deleteUserController` responds 202 and leaves
-the deletion barrier up instead of destroying data a still-live generation might rewrite.
-
-This leaves **no scheduler debt** — quiesce marks every one of the user's schedules
-`deleting`, and the reconciler already erases those and their runs once drained. Only the
-account document remains, with `deletionRequestedAt` set, so nothing new can be
-scheduled. `markUserDeleting` is idempotent, so re-issuing DELETE completes the cascade.
-
-A platform-level worker that automatically finishes such accounts (query:
-`getUsersPendingDeletion`) is out of scope for the scheduler and left to follow-up.
-
-## Also deferred (tracked separately)
-
-- Client active-job discovery. There is no per-schedule run-status surface today, and
-  `ScheduleCard`'s `STATUS_CHIPS.started` is dead code: `lastRun` is only written on
-  terminal/pause/skip, so its status can never be `started`.
-- `minIntervalMinutes` in the wire type + dialog gating; the server enforces a floor the
-  client neither surfaces nor explains.
-
-## Account deletion: no barrier at AUTHENTICATION (out of scope, pre-existing)
-
-`deletionRequestedAt` is a SCHEDULING barrier. It is consulted by the engine's claims,
-the fire dispatch boundary (`isOwnerDeleting`), and every schedule write — so no new
-scheduled work is admitted once it is up. It is **not** consulted by
-`api/strategies/jwtStrategy.js` or `openIdJwtStrategy.js`, so a user whose deletion has
-begun still authenticates, and an already-issued access token keeps working until it
-expires (15 minutes by default). A request admitted that way can write during the
-cascade and leave rows behind for an account the operator believes is erased.
-
-This predates the scheduler: on `dev` the account-deletion path has no barrier at all,
-so the same window exists and is strictly wider. Closing it means refusing admission in
-the auth strategies (and/or invalidating the user's sessions) — a platform-level change
-affecting every route, which is why it is not folded into the scheduler PR.
-
-Related: `setCachedAuthUserDoc` writes its cache entry and reverse index in two steps,
-so an invalidation interleaving between them can miss the entry it never indexed. Bounded
-by the entry's own TTL, and it cannot weaken the barrier itself (every barrier check
-reads Mongo directly, never the auth cache), but it belongs with the work above.
-
-## Balance: mid-generation insufficiency is classified as `skipped_balance` (resolved)
-
-`isOutOfBalance` can only pre-skip a balance a fire has ALREADY exhausted; the true cost
-of a run is unknowable before its payload is built, so an owner whose credits sit between
-zero and the run's cost passes the pre-check and fails mid-generation (the loopback POST
-was accepted; `checkBalance` throws inside the generation).
-
-That refusal is structured (`{ type: 'token_balance', ... }`), so the controller's catch
-now settles the run as `skipped_balance` instead of `error`: `recordRunOutcome` takes the
-same guarded terminal transition but walks the consecutive-balance-skip streak (toward
-`insufficient_balance`) rather than the failure streak (toward `too_many_failures`). The
-pre-skip itself remains inexact by nature; only the classification was wrong.
-
-## Resume re-pause hand-off has a narrower version of the drain race (open)
-
-The `started`-row-plus-paused-job drain gate closes the ORIGINAL pause hand-off: every
-write in the request controller's pause branch happens while the row is still `started`,
-and quiesce defers exactly that state. A resumed run that pauses AGAIN has no such
-marker — the row is `requires_action` throughout the resume, so a drain observing the
-re-paused job can settle the row while resume.js's re-pause branch is still persisting
-the segment content. The window is one branch, not the whole turn (the resume claim keeps
-the job `running` — not settleable — until the re-interrupt), and the writes are
-unfinished-marked segment saves rather than settlement-ordered outcomes. Closing it
-cleanly needs a hand-off marker the row can carry across pause generations (e.g., the
-pending action's createdAt fenced against the drain's observation), which touches the
-resume claim protocol — deferred until the shape is needed elsewhere.
+- Add Playwright coverage for creating a schedule through the UI. The execution E2E currently
+  seeds schedules through the API.

packages/api/src/stream/__tests__/RedisEventTransport.spec.ts

@@ -157,7 +157,7 @@ describe('RedisEventTransport', () => {
     transport.destroy();
   });
 
-  it('publishes the abort acknowledgement only AFTER the pre-ACK fence resolves', async () => {
+  it('awaits the owner lease callback before stopping and acknowledging the generation', async () => {
     const mockPublisher = createMockPublisher();
     const mockSubscriber = createMockSubscriber();
     const transport = new RedisEventTransport(
@@ -165,16 +165,20 @@ describe('RedisEventTransport', () => {
       mockSubscriber as unknown as Redis,
     );
     const streamId = 'abort-ack-fence';
-    // The stopping side releases its own lease the moment this ACK arrives; the
-    // owner's abort-catch persistence is still ahead. If the ACK can outrun the
-    // owner-lifecycle lease registration, a deletion quiesce lands in the gap.
-    let releaseFence!: () => void;
-    const fencePending = new Promise<void>((resolve) => {
-      releaseFence = resolve;
+    // The callback owns the provider trip, so it must first make the owner lease
+    // durable. Awaiting only a pre-ACK hook is too late: the controller can unwind
+    // and release while that hook is still registering the lease.
+    let releaseLease!: () => void;
+    const leasePending = new Promise<void>((resolve) => {
+      releaseLease = resolve;
     });
-    const fence = jest.fn(() => fencePending);
-    transport.beforeAbortAcknowledged = fence;
-    await transport.onAbort(streamId, () => true);
+    const provider = new AbortController();
+    const callback = jest.fn(async () => {
+      await leasePending;
+      provider.abort();
+      return true;
+    });
+    await transport.onAbort(streamId, callback);
     const messageHandler = getMessageHandler(mockSubscriber);
 
     messageHandler(
@@ -184,17 +188,19 @@ describe('RedisEventTransport', () => {
     await Promise.resolve();
     await Promise.resolve();
 
-    expect(fence).toHaveBeenCalledWith(streamId, 7);
-    // Neither the durable ack proof nor the ack publication may precede the fence.
+    expect(callback).toHaveBeenCalledWith(7);
+    expect(provider.signal.aborted).toBe(false);
+    // Neither the provider trip nor its acknowledgement may precede the lease.
     expect(mockPublisher.set).not.toHaveBeenCalled();
     const ackPublishes = mockPublisher.publish.mock.calls.filter(([, payload]) =>
       String(payload).includes('abort_ack'),
     );
     expect(ackPublishes).toHaveLength(0);
 
-    releaseFence();
+    releaseLease();
     await new Promise((resolve) => setImmediate(resolve));
 
+    expect(provider.signal.aborted).toBe(true);
     expect(mockPublisher.set).toHaveBeenCalledWith(
       expect.stringContaining('abort-ack'),
       '1',
@@ -204,7 +210,7 @@ describe('RedisEventTransport', () => {
     transport.destroy();
   });
 
-  it('SUPPRESSES the abort acknowledgement when the pre-ACK fence rejects', async () => {
+  it('suppresses the abort acknowledgement when the owner lease callback rejects', async () => {
     const mockPublisher = createMockPublisher();
     const mockSubscriber = createMockSubscriber();
     const transport = new RedisEventTransport(
@@ -215,10 +221,9 @@ describe('RedisEventTransport', () => {
     // FAIL CLOSED: an ACK without a durable owner lease lets the stopping side
     // release its lease into an unfenced gap. The stopping side stays retryable and
     // every resignal re-drives this handler until the fence holds.
-    transport.beforeAbortAcknowledged = jest.fn(async () => {
+    await transport.onAbort(streamId, async () => {
       throw new Error('owner lease unavailable');
     });
-    await transport.onAbort(streamId, () => true);
     const messageHandler = getMessageHandler(mockSubscriber);
 
     messageHandler(
@@ -885,7 +890,7 @@ describe('RedisEventTransport', () => {
     await manager.destroy();
   });
 
-  it('resubscribes a replacement after an unexposed registration loses initialization', async () => {
+  it('keeps one channel subscription across an unexposed replacement race', async () => {
     const mockPublisher = createMockPublisher();
     const mockSubscriber = createMockSubscriber();
     let signalSubscriptionStarted: (() => void) | undefined;
@@ -922,16 +927,19 @@ describe('RedisEventTransport', () => {
     );
     const replacement = await replacementCreation;
 
-    expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(1);
-    expect(mockSubscriber.subscribe).toHaveBeenCalledTimes(2);
+    // The safely-aborted predecessor retains its registration until the
+    // successor's listener is active, so there is no unsubscribe/resubscribe gap.
+    expect(mockSubscriber.unsubscribe).not.toHaveBeenCalled();
+    expect(mockSubscriber.subscribe).toHaveBeenCalledTimes(1);
 
     getMessageHandler(mockSubscriber)(
       `stream:{${streamId}}:events`,
       JSON.stringify({ type: 'abort', generationId: replacement.createdAt }),
     );
+    await new Promise((resolve) => setImmediate(resolve));
 
     expect(replacement.abortController.signal.aborted).toBe(true);
-    expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(2);
+    expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(1);
 
     await manager.destroy();
   });
@@ -961,6 +969,7 @@ describe('RedisEventTransport', () => {
       `stream:{${streamId}}:events`,
       JSON.stringify({ type: 'abort' }),
     );
+    await new Promise((resolve) => setImmediate(resolve));
 
     expect(job.abortController.signal.aborted).toBe(true);
     expect(mockSubscriber.unsubscribe).toHaveBeenCalledWith(`stream:{${streamId}}:events`);

packages/api/src/stream/__tests__/abortSignalHonesty.spec.ts

@@ -187,35 +187,141 @@ describe('abort signal honesty across replicas', () => {
     expect(result.success).toBe(true);
     expect(result.signalDelivered).toBe(false);
     expect(result.signalPublished).toBe(false);
-    // The job is terminal (invisible to active-set scans) and a user Stop writes no
-    // durable abort fence — this lease is the ONLY thing deferring a deletion
-    // quiesce while the remote owner keeps generating.
+    // The job is terminal (invisible to active-set scans). This live-process lease
+    // complements the durable user-document delivery fence while the remote owner
+    // keeps generating.
     await new Promise((resolve) => setImmediate(resolve));
     await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(1);
   });
 
-  it('renews the fence on each undelivered resignal attempt', async () => {
+  it('heartbeats an undelivered Stop past the lease TTL without a client retry', async () => {
     const { owner, peer, peerTransport, jobStore } = await makeManagers();
     await owner.createJob('conv-9', 'user-1', 'conv-9');
-    Object.assign(peerTransport, {
-      emitAbort: jest.fn((): void => undefined),
-      emitAbortConfirmed: jest.fn(async () => false),
-    });
-    await peer.abortJob('conv-9');
+    const emitAbortConfirmed = jest.fn(async () => false);
+    Object.assign(peerTransport, { emitAbortConfirmed });
     const register = jest.spyOn(jobStore, 'registerUserFinalization');
 
-    const resignal = await peer.resignalAbort('conv-9');
+    jest.useFakeTimers({ now: Date.now() });
+    try {
+      const result = await peer.abortJob('conv-9');
+      expect(result).toMatchObject({
+        success: true,
+        signalDelivered: false,
+        signalPublished: false,
+      });
+      const stopRenewals = () => register.mock.calls.filter((call) => call[4] === 'stop');
+      expect(stopRenewals()).toHaveLength(1);
 
-    expect(resignal).toEqual({ delivered: false, published: false });
-    // The 5-minute lease is never self-renewing: each retry heartbeats a fresh one
-    // so the quiesce keeps deferring for as long as delivery is still being driven.
-    expect(register).toHaveBeenCalledWith(
+      // No resignal is issued. The manager that observed failed delivery owns the
+      // liveness duty and must refresh the five-minute marker autonomously.
+      await jest.advanceTimersByTimeAsync(6 * 60_000);
+      expect(stopRenewals().length).toBeGreaterThan(1);
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(1);
+      // The heartbeat probes settlement; it does not depend on another publication.
+      expect(emitAbortConfirmed).not.toHaveBeenCalled();
+    } finally {
+      await Promise.allSettled([owner.destroy(), peer.destroy()]);
+      jest.useRealTimers();
+    }
+  });
+
+  it('persists undelivered Stop recovery evidence before the CAS and keeps it through shutdown', async () => {
+    const { GenerationJobManagerClass } = await import('../GenerationJobManager');
+    const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+    const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 0 });
+    jest.spyOn(jobStore, 'destroy').mockResolvedValue();
+    const owner = new GenerationJobManagerClass();
+    owner.configure({
+      jobStore,
+      eventTransport: new InMemoryEventTransport(),
+      isRedis: false,
+    });
+    owner.initialize();
+    await owner.createJob('conv-durable-stop-recovery', 'user-1', 'conv-durable-stop-recovery');
+
+    const retainAbortDelivery = jest.fn(async () => undefined);
+    const clearAbortDelivery = jest.fn(async () => undefined);
+    const peerTransport = Object.assign(new InMemoryEventTransport(), {
+      emitAbortConfirmed: jest.fn(async () => false),
+    });
+    const peer = new GenerationJobManagerClass();
+    peer.configure({
+      jobStore,
+      eventTransport: peerTransport,
+      isRedis: true,
+      userFinalizationFallback: {
+        renew: jest.fn(async () => undefined),
+        clear: jest.fn(async () => undefined),
+        retainAbortDelivery,
+        clearAbortDelivery,
+      },
+    });
+    peer.initialize();
+    const transition = jest.spyOn(jobStore, 'transitionStatusAndDrainSteers');
+
+    const result = await peer.abortJob('conv-durable-stop-recovery');
+
+    expect(result).toMatchObject({
+      success: true,
+      signalDelivered: false,
+      signalPublished: false,
+    });
+    expect(retainAbortDelivery).toHaveBeenCalledWith(
       'user-1',
-      'conv-9',
       undefined,
-      expect.any(Number),
-      expect.any(String),
+      'conv-durable-stop-recovery',
     );
+    expect(retainAbortDelivery.mock.invocationCallOrder[0]).toBeLessThan(
+      transition.mock.invocationCallOrder[0],
+    );
+    expect(clearAbortDelivery).not.toHaveBeenCalled();
+
+    await peer.destroy();
+    // TTL-backed manager markers may now age out, but the user-document recovery
+    // record must survive so the deletion sweep can enumerate and re-signal it.
+    expect(clearAbortDelivery).not.toHaveBeenCalled();
+    await owner.destroy();
+  });
+
+  it('heartbeats the abort contender while required persistence exceeds the lease TTL', async () => {
+    const { owner, jobStore } = await makeManagers();
+    await owner.createJob('conv-slow-stop-persistence', 'user-1', 'conv-slow-stop-persistence');
+    const register = jest.spyOn(jobStore, 'registerUserFinalization');
+    let persistenceStarted!: () => void;
+    const started = new Promise<void>((resolve) => {
+      persistenceStarted = resolve;
+    });
+    let finishPersistence!: () => void;
+    const persistenceGate = new Promise<void>((resolve) => {
+      finishPersistence = resolve;
+    });
+
+    jest.useFakeTimers({ now: Date.now() });
+    try {
+      const aborting = owner.abortJob('conv-slow-stop-persistence', {
+        beforePublish: async () => {
+          persistenceStarted();
+          await persistenceGate;
+        },
+      });
+      await started;
+      const contenderLease = register.mock.calls.find(
+        (call) => call[4] !== 'owner' && call[4] !== 'stop',
+      )?.[4];
+      expect(contenderLease).toEqual(expect.any(String));
+
+      await jest.advanceTimersByTimeAsync(6 * 60_000);
+      expect(
+        register.mock.calls.filter((call) => call[4] === contenderLease).length,
+      ).toBeGreaterThan(1);
+
+      finishPersistence();
+      await expect(aborting).resolves.toMatchObject({ success: true });
+    } finally {
+      await owner.destroy();
+      jest.useRealTimers();
+    }
   });
 
   it('a losing second Stop releases only its own lease, never the winning one', async () => {
@@ -327,14 +433,144 @@ describe('abort signal honesty across replicas', () => {
       await jest.advanceTimersByTimeAsync(6 * 60 * 1000);
       await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(1);
 
-      hold.release();
-      await jest.advanceTimersByTimeAsync(1);
+      await hold.release();
       await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
     } finally {
       jest.useRealTimers();
     }
   });
 
+  it('promotes a stalled primary heartbeat into Mongo before the primary TTL expires', async () => {
+    const { GenerationJobManagerClass } = await import('../GenerationJobManager');
+    const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+    const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 0 });
+    const fallback = {
+      renew: jest.fn(
+        async (
+          _userId: string,
+          _tenantId: string | undefined,
+          _leaseKey: string,
+          _expiresAt: Date,
+        ) => undefined,
+      ),
+      clear: jest.fn(
+        async (_userId: string, _tenantId: string | undefined, _leaseKey: string) => undefined,
+      ),
+    };
+    const manager = new GenerationJobManagerClass();
+    manager.configure({
+      jobStore,
+      eventTransport: new InMemoryEventTransport(),
+      isRedis: true,
+      userFinalizationFallback: fallback,
+    });
+    manager.initialize();
+
+    jest.useFakeTimers({ now: Date.now() });
+    try {
+      const hold = await manager.holdUserFinalization(
+        'user-1',
+        'conv-primary-stall',
+        undefined,
+        1000,
+        'held',
+      );
+      const realRegister = jobStore.registerUserFinalization.bind(jobStore);
+      let releasePrimary!: () => void;
+      const primaryPending = new Promise<void>((resolve) => {
+        releasePrimary = resolve;
+      });
+      jest.spyOn(jobStore, 'registerUserFinalization').mockImplementationOnce(async (...args) => {
+        await primaryPending;
+        await realRegister(...args);
+      });
+
+      // The renewal starts at one minute. Five seconds later it is promoted to
+      // Mongo, leaving almost four minutes on the last confirmed primary TTL.
+      jest.advanceTimersByTime(60_000);
+      await Promise.resolve();
+      await jest.advanceTimersByTimeAsync(5_000);
+      expect(fallback.renew).toHaveBeenCalledWith(
+        'user-1',
+        undefined,
+        expect.stringMatching(/^fallback_[A-Za-z0-9_-]+$/),
+        expect.any(Date),
+      );
+
+      // Release serializes behind the late primary command, then clears both
+      // stores so that command cannot resurrect a marker behind cleanup.
+      const releasing = hold.release();
+      releasePrimary();
+      await releasing;
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
+      const fallbackKey = fallback.renew.mock.calls[0][2];
+      expect(fallback.clear).toHaveBeenCalledWith('user-1', undefined, fallbackKey);
+      await jest.advanceTimersByTimeAsync(6 * 60_000);
+      expect(fallback.renew).toHaveBeenCalledTimes(1);
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
+    } finally {
+      await manager.destroy();
+      jest.useRealTimers();
+    }
+  });
+
+  it('keeps a promoted fallback alive after the primary heartbeat recovers', async () => {
+    const { GenerationJobManagerClass } = await import('../GenerationJobManager');
+    const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+    const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 0 });
+    const fallback = {
+      renew: jest.fn(
+        async (
+          _userId: string,
+          _tenantId: string | undefined,
+          _leaseKey: string,
+          _expiresAt: Date,
+        ) => undefined,
+      ),
+      clear: jest.fn(
+        async (_userId: string, _tenantId: string | undefined, _leaseKey: string) => undefined,
+      ),
+    };
+    const manager = new GenerationJobManagerClass();
+    manager.configure({
+      jobStore,
+      eventTransport: new InMemoryEventTransport(),
+      isRedis: true,
+      userFinalizationFallback: fallback,
+    });
+    manager.initialize();
+
+    jest.useFakeTimers({ now: Date.now() });
+    try {
+      const hold = await manager.holdUserFinalization(
+        'user-1',
+        'conv-primary-recovers',
+        undefined,
+        1000,
+        'held',
+      );
+      jest
+        .spyOn(jobStore, 'registerUserFinalization')
+        .mockRejectedValueOnce(new Error('primary temporarily unavailable'));
+
+      await jest.advanceTimersByTimeAsync(60_000);
+      expect(fallback.renew).toHaveBeenCalledTimes(1);
+      await jest.advanceTimersByTimeAsync(60_000);
+      // Primary recovery does not clear the fallback mid-hold. It is renewed
+      // until the persistence owner explicitly releases both leases.
+      expect(fallback.renew).toHaveBeenCalledTimes(2);
+
+      await hold.release();
+      expect(fallback.clear).toHaveBeenCalledTimes(1);
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
+    } finally {
+      await manager.destroy();
+      jest.useRealTimers();
+    }
+  });
+
   it('retains a fence when the abort transition commits but loses its reply', async () => {
     const { owner, peer, jobStore } = await makeManagers();
     await owner.createJob('conv-14', 'user-1', 'conv-14');
@@ -415,10 +651,18 @@ describe('abort signal honesty across replicas', () => {
 
     jest.useFakeTimers({ now: Date.now() });
     try {
-      // The transport hook is how the owner replica acquires the lease.
+      // The abort callback routes through this same serialized acquisition before
+      // it trips the provider. Invoke it directly here to isolate heartbeat TTL.
       await (
-        transport as { beforeAbortAcknowledged?: (s: string, g: number) => Promise<void> }
-      ).beforeAbortAcknowledged?.('conv-17', createdAt);
+        manager as unknown as {
+          acquireOwnerLease: (
+            userId: string,
+            streamId: string,
+            tenantId: string | undefined,
+            createdAt: number,
+          ) => Promise<void>;
+        }
+      ).acquireOwnerLease('user-1', 'conv-17', undefined, createdAt);
       await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(1);
 
       // Owner-side persistence can legitimately outlive the five-minute store TTL.
@@ -435,7 +679,126 @@ describe('abort signal honesty across replicas', () => {
     }
   });
 
-  it('retries failed-handoff retention on the heartbeat instead of degrading to one-shot', async () => {
+  it('does not resurrect an owner lease when release races its initial registration', async () => {
+    const { owner, jobStore } = await makeManagers();
+    await owner.createJob('conv-owner-release-race', 'user-1', 'conv-owner-release-race');
+    const createdAt = (await jobStore.getJob('conv-owner-release-race'))!.createdAt;
+    const realRegister = jobStore.registerUserFinalization.bind(jobStore);
+    let finishRegistration!: () => void;
+    const registrationPending = new Promise<void>((resolve) => {
+      finishRegistration = resolve;
+    });
+    jest.spyOn(jobStore, 'registerUserFinalization').mockImplementation(async (...args) => {
+      if (args[4] === 'owner') {
+        await registrationPending;
+      }
+      return realRegister(...args);
+    });
+    const privateOwner = owner as unknown as {
+      acquireOwnerLease: (
+        userId: string,
+        streamId: string,
+        tenantId: string | undefined,
+        generationId: number,
+      ) => Promise<void>;
+    };
+
+    const acquiring = privateOwner.acquireOwnerLease(
+      'user-1',
+      'conv-owner-release-race',
+      undefined,
+      createdAt,
+    );
+    await Promise.resolve();
+    const releasing = owner.releaseOwnerLease(
+      'user-1',
+      'conv-owner-release-race',
+      undefined,
+      createdAt,
+    );
+    finishRegistration();
+    await acquiring.catch(() => undefined);
+    await releasing;
+
+    await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
+  });
+
+  it('does not trip a local replacement provider without a durable owner lease', async () => {
+    const { owner, jobStore } = await makeManagers();
+    const predecessor = await owner.createJob(
+      'conv-local-replacement-fence',
+      'user-1',
+      'conv-local-replacement-fence',
+    );
+    jest
+      .spyOn(jobStore, 'registerUserFinalization')
+      .mockRejectedValue(new Error('marker store unavailable'));
+
+    await expect(
+      owner.createJob('conv-local-replacement-fence', 'user-1', 'conv-local-replacement-fence'),
+    ).rejects.toThrow();
+
+    expect(predecessor.abortController.signal.aborted).toBe(false);
+    await expect(jobStore.getJob('conv-local-replacement-fence')).resolves.toMatchObject({
+      status: 'running',
+      userId: 'user-1',
+    });
+  });
+
+  it('does not trip a locally-owned Stop without a durable owner lease', async () => {
+    const { owner, jobStore } = await makeManagers();
+    const job = await owner.createJob('conv-local-stop-fence', 'user-1', 'conv-local-stop-fence');
+    const realRegister = jobStore.registerUserFinalization.bind(jobStore);
+    jest.spyOn(jobStore, 'registerUserFinalization').mockImplementation(async (...args) => {
+      if (args[4] === 'owner') {
+        throw new Error('marker store unavailable');
+      }
+      return realRegister(...args);
+    });
+
+    const result = await owner.abortJob('conv-local-stop-fence');
+
+    expect(result).toMatchObject({
+      success: true,
+      signalDelivered: false,
+      signalPublished: false,
+    });
+    expect(job.abortController.signal.aborted).toBe(false);
+    await expect(jobStore.getJob('conv-local-stop-fence')).resolves.toMatchObject({
+      status: 'aborted',
+      userId: 'user-1',
+    });
+  });
+
+  it('keeps a failed remote replacement visible until retention is durable', async () => {
+    const { GenerationJobManagerClass } = await import('../GenerationJobManager');
+    const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+    const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 0 });
+    const owner = new GenerationJobManagerClass();
+    owner.configure({ jobStore, eventTransport: new InMemoryEventTransport(), isRedis: false });
+    owner.initialize();
+    const peer = new GenerationJobManagerClass();
+    const peerTransport = new InMemoryEventTransport();
+    Object.assign(peerTransport, { emitAbortConfirmed: jest.fn(async () => false) });
+    peer.configure({ jobStore, eventTransport: peerTransport, isRedis: true });
+    peer.initialize();
+    await owner.createJob('conv-remote-retention-fence', 'user-1', 'conv-remote-retention-fence');
+    jest
+      .spyOn(jobStore, 'registerUserFinalization')
+      .mockRejectedValue(new Error('marker store unavailable'));
+
+    await expect(
+      peer.createJob('conv-remote-retention-fence', 'user-1', 'conv-remote-retention-fence'),
+    ).rejects.toThrow();
+
+    await expect(jobStore.getJob('conv-remote-retention-fence')).resolves.toMatchObject({
+      status: 'running',
+      userId: 'user-1',
+    });
+  });
+
+  it('heartbeats a durably-established failed-handoff retention marker', async () => {
     const { GenerationJobManagerClass } = await import('../GenerationJobManager');
     const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
     const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
@@ -450,28 +813,92 @@ describe('abort signal honesty across replicas', () => {
     peer.initialize();
     await owner.createJob('conv-18', 'user-1', 'conv-18');
 
-    // Retention registration fails at first — the retainer must keep trying on its
-    // heartbeat, not silently give up while the predecessor is undiscoverable.
-    const realRegister = jobStore.registerUserFinalization.bind(jobStore);
-    let failures = 0;
-    jest.spyOn(jobStore, 'registerUserFinalization').mockImplementation(async (...args) => {
-      if (args[4] === 'owner' && failures++ < 1) {
-        throw new Error('marker store blip');
-      }
-      return realRegister(...args);
-    });
+    // The first marker must already be durable before createJob may reject and
+    // release its caller-side admission lease. The heartbeat preserves that proof.
     jest.useFakeTimers({ now: Date.now() });
     try {
       await expect(peer.createJob('conv-18', 'user-1', 'conv-18')).rejects.toThrow(
         'predecessor handoff',
       );
-      await jest.advanceTimersByTimeAsync(61_000);
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBeGreaterThanOrEqual(1);
+      await jest.advanceTimersByTimeAsync(6 * 60_000);
       await expect(jobStore.countUserFinalizations('user-1')).resolves.toBeGreaterThanOrEqual(1);
     } finally {
       jest.useRealTimers();
     }
   });
 
+  it('stands down a failed-handoff retainer after natural owner settlement is proven', async () => {
+    const { GenerationJobManagerClass } = await import('../GenerationJobManager');
+    const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+    const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 0 });
+    const settlementProofs = new Set<string>();
+    const proofKey = (streamId: string, generationId: number) => `${streamId}:${generationId}`;
+    const recordGenerationSettlement = jest.fn(async (streamId: string, generationId: number) => {
+      settlementProofs.add(proofKey(streamId, generationId));
+      return true;
+    });
+    const ownerTransport = new InMemoryEventTransport();
+    Object.assign(ownerTransport, { recordGenerationSettlement });
+    const owner = new GenerationJobManagerClass();
+    owner.configure({ jobStore, eventTransport: ownerTransport, isRedis: true });
+    owner.initialize();
+
+    const hasGenerationSettlement = jest.fn(async (streamId: string, generationId: number) =>
+      settlementProofs.has(proofKey(streamId, generationId)),
+    );
+    const peerTransport = new InMemoryEventTransport();
+    Object.assign(peerTransport, {
+      emitAbortConfirmed: jest.fn(async () => false),
+      hasAbortAcknowledgement: jest.fn(async () => false),
+      hasGenerationSettlement,
+    });
+    const peer = new GenerationJobManagerClass();
+    peer.configure({ jobStore, eventTransport: peerTransport, isRedis: true });
+    peer.initialize();
+
+    await owner.createJob('conv-natural-settlement', 'user-1', 'conv-natural-settlement');
+    const predecessor = await jobStore.getJob('conv-natural-settlement');
+    const register = jest.spyOn(jobStore, 'registerUserFinalization');
+
+    jest.useFakeTimers({ now: Date.now() });
+    try {
+      await expect(
+        peer.createJob('conv-natural-settlement', 'user-1', 'conv-natural-settlement'),
+      ).rejects.toThrow('predecessor handoff');
+      const ownerRegistrations = () => register.mock.calls.filter((call) => call[4] === 'owner');
+      expect(ownerRegistrations()).toHaveLength(1);
+
+      // A predecessor may finish naturally after the handoff publication failed,
+      // so there is no abort ACK. Its settlement path writes the separate,
+      // short-lived exact-generation proof before releasing the shared owner field.
+      await owner.releaseOwnerLease(
+        'user-1',
+        'conv-natural-settlement',
+        undefined,
+        predecessor!.createdAt,
+      );
+      expect(recordGenerationSettlement).toHaveBeenCalledWith(
+        'conv-natural-settlement',
+        predecessor!.createdAt,
+      );
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
+
+      const registrationsAtSettlement = ownerRegistrations().length;
+      await jest.advanceTimersByTimeAsync(2 * 60_000);
+      expect(hasGenerationSettlement).toHaveBeenCalledWith(
+        'conv-natural-settlement',
+        predecessor!.createdAt,
+      );
+      expect(ownerRegistrations()).toHaveLength(registrationsAtSettlement);
+      await expect(jobStore.countUserFinalizations('user-1')).resolves.toBe(0);
+    } finally {
+      await Promise.allSettled([owner.destroy(), peer.destroy()]);
+      jest.useRealTimers();
+    }
+  });
+
   it('reports republication as succeeded when the owner acknowledges', async () => {
     const { owner, peer, peerTransport, jobStore } = await makeManagers();
     await owner.createJob('conv-4', 'user-1', 'conv-4');

packages/api/src/stream/__tests__/idempotencyClaim.spec.ts

@@ -771,7 +771,10 @@ describe('GenerationJobManager start-generation claim', () => {
         event: 'on_message_delta',
         data: { delta: 'stale provider output' },
       });
-      await Promise.resolve();
+      // The stale append continuation now awaits durable owner-lease acquisition
+      // before it may trip the provider. Flush that async handoff without advancing
+      // the later reconnect-grace timers.
+      await jest.advanceTimersByTimeAsync(0);
 
       expect(predecessor.abortController.signal.aborted).toBe(true);
       expect(onError).not.toHaveBeenCalled();
@@ -819,8 +822,7 @@ describe('GenerationJobManager start-generation claim', () => {
         event: 'on_message_delta',
         data: { delta: 'uncoordinated provider output' },
       });
-      await Promise.resolve();
-      await Promise.resolve();
+      await jest.advanceTimersByTimeAsync(0);
 
       expect(job.abortController.signal.aborted).toBe(true);
       expect(onError).not.toHaveBeenCalled();

packages/api/src/stream/__tests__/startup.spec.ts

@@ -1337,7 +1337,9 @@ describe('GenerationJobManager startup telemetry', () => {
 
       expect(job.abortController.signal.aborted).toBe(true);
       expect(manager.getRuntimeStats().runtimeStateSize).toBe(1);
-      expect(jest.getTimerCount()).toBe(timerCountBeforeFence + 1);
+      // The fenced-retirement grace and its owner-side persistence lease each
+      // carry one timer while the controller catch is still outstanding.
+      expect(jest.getTimerCount()).toBe(timerCountBeforeFence + 2);
 
       await jest.advanceTimersByTimeAsync(REDIS_ABORT_TERMINAL_GRACE_MS);
       await lookupStarted;
@@ -1355,7 +1357,9 @@ describe('GenerationJobManager startup telemetry', () => {
       expect(getJob).toHaveBeenCalledTimes(1);
       expect(onError).not.toHaveBeenCalled();
       expect(onAllSubscribersLeft).not.toHaveBeenCalled();
-      expect(jest.getTimerCount()).toBe(timerCountBeforeFence);
+      // Subscriber detachment cancels only the retirement timer. The owner
+      // heartbeat remains until controller settlement or manager destruction.
+      expect(jest.getTimerCount()).toBe(timerCountBeforeFence + 1);
 
       releaseLookup?.();
       await jest.advanceTimersByTimeAsync(0);
@@ -3701,6 +3705,138 @@ describe('GenerationJobManager startup telemetry', () => {
     subscription?.unsubscribe();
   });
 
+  it('does not hang shutdown or clear evidence behind a stalled primary renewal', async () => {
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+    jest.spyOn(jobStore, 'destroy').mockResolvedValue();
+    const clearPrimary = jest.spyOn(jobStore, 'clearUserFinalization');
+    const fallback = {
+      renew: jest.fn(
+        async (
+          _userId: string,
+          _tenantId: string | undefined,
+          _leaseKey: string,
+          _expiresAt: Date,
+        ) => undefined,
+      ),
+      clear: jest.fn(
+        async (_userId: string, _tenantId: string | undefined, _leaseKey: string) => undefined,
+      ),
+    };
+    const manager = new GenerationJobManagerClass();
+    manager.configure({
+      jobStore,
+      eventTransport: new InMemoryEventTransport(),
+      isRedis: true,
+      userFinalizationFallback: fallback,
+    });
+    manager.initialize();
+
+    jest.useFakeTimers({ now: Date.now() });
+    try {
+      await manager.holdUserFinalization(
+        'user-stalled-shutdown',
+        'stream-stalled-shutdown',
+        undefined,
+        123,
+        'held',
+      );
+      const neverSettles = new Promise<void>(() => undefined);
+      jest.spyOn(jobStore, 'registerUserFinalization').mockImplementationOnce(() => neverSettles);
+
+      // The first renewal never replies. Its five-second deadline promotes the
+      // holder to fallback storage while retaining the ambiguous primary command.
+      jest.advanceTimersByTime(60_000);
+      await Promise.resolve();
+      await jest.advanceTimersByTimeAsync(5_000);
+      expect(fallback.renew).toHaveBeenCalledTimes(1);
+
+      let destroySettled = false;
+      const destroying = manager.destroy().then(() => {
+        destroySettled = true;
+      });
+      await jest.advanceTimersByTimeAsync(1);
+
+      // Teardown cannot serialize behind a command that may never reply. It also
+      // cannot clear either copy: the late primary command could otherwise recreate
+      // its marker after shutdown, and detached persistence may still be landing.
+      expect(destroySettled).toBe(true);
+      await destroying;
+      expect(clearPrimary).not.toHaveBeenCalled();
+      expect(fallback.clear).not.toHaveBeenCalled();
+      await expect(jobStore.countUserFinalizations('user-stalled-shutdown')).resolves.toBe(1);
+    } finally {
+      jest.useRealTimers();
+    }
+  });
+
+  it('does not clear a shared owner marker when only its failed-handoff retainer shuts down', async () => {
+    const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+    jest.spyOn(jobStore, 'destroy').mockResolvedValue();
+    const owner = new GenerationJobManagerClass();
+    owner.configure({
+      jobStore,
+      eventTransport: new InMemoryEventTransport(),
+      isRedis: false,
+    });
+    owner.initialize();
+    const retainerTransport = Object.assign(new InMemoryEventTransport(), {
+      hasAbortAcknowledgement: jest.fn(async () => false),
+      hasGenerationSettlement: jest.fn(async () => false),
+    });
+    const retainer = new GenerationJobManagerClass();
+    retainer.configure({ jobStore, eventTransport: retainerTransport, isRedis: true });
+    retainer.initialize();
+    const generationId = 456;
+    const ownerInternals = owner as unknown as {
+      acquireOwnerLease: (
+        userId: string,
+        streamId: string,
+        tenantId: string | undefined,
+        createdAt: number,
+      ) => Promise<void>;
+    };
+    const retainerInternals = retainer as unknown as {
+      retainPredecessorOwnerLease: (
+        userId: string,
+        streamId: string,
+        tenantId: string | undefined,
+        createdAt: number,
+      ) => Promise<void>;
+    };
+
+    await ownerInternals.acquireOwnerLease(
+      'user-shared-owner',
+      'stream-shared-owner',
+      undefined,
+      generationId,
+    );
+    await retainerInternals.retainPredecessorOwnerLease(
+      'user-shared-owner',
+      'stream-shared-owner',
+      undefined,
+      generationId,
+    );
+    await expect(jobStore.countUserFinalizations('user-shared-owner')).resolves.toBe(1);
+    const clearOwnerMarker = jest.spyOn(jobStore, 'clearUserFinalization');
+
+    await retainer.destroy();
+
+    // Both replicas renew the same deterministic `owner` field. The retainer owns
+    // only its local heartbeat; shutdown must leave the shared marker for the real
+    // owner whose persistence may still be in flight.
+    expect(clearOwnerMarker).not.toHaveBeenCalled();
+    await expect(jobStore.countUserFinalizations('user-shared-owner')).resolves.toBe(1);
+
+    await owner.releaseOwnerLease(
+      'user-shared-owner',
+      'stream-shared-owner',
+      undefined,
+      generationId,
+    );
+    await expect(jobStore.countUserFinalizations('user-shared-owner')).resolves.toBe(0);
+    await owner.destroy();
+  });
+
   it('drains replaced services without destroying the newly configured services', async () => {
     const replacedJobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
     const replacedEventTransport = new InMemoryEventTransport();

packages/api/src/stream/implementations/RedisEventTransport.ts

@@ -101,7 +101,7 @@ interface ReorderBuffer {
 }
 
 interface AbortRegistration {
-  callback: (generationId?: number) => void | boolean;
+  callback: (generationId?: number) => void | boolean | Promise<void | boolean>;
 }
 
 interface AbortAckWaiter {
@@ -258,8 +258,6 @@ interface StreamSubscribers {
  * ```
  */
 export class RedisEventTransport implements IEventTransport {
-  /** Awaited before an owned abort is acknowledged; see IEventTransport. */
-  beforeAbortAcknowledged?: (streamId: string, generationId: number) => Promise<void>;
   /** Redis client for publishing events */
   private publisher: Redis | Cluster;
   /** Redis client for subscribing to events (separate connection required) */
@@ -929,41 +927,7 @@ export class RedisEventTransport implements IEventTransport {
     }
 
     if (message.type === EventTypes.ABORT) {
-      let ownerAcknowledged = false;
-      for (const registration of streamState.abortCallbacks) {
-        try {
-          if (message.generationId == null) {
-            ownerAcknowledged = registration.callback() === true || ownerAcknowledged;
-          } else {
-            ownerAcknowledged =
-              registration.callback(message.generationId) === true || ownerAcknowledged;
-          }
-        } catch (err) {
-          logger.error(`[RedisEventTransport] Error in abort callback:`, err);
-        }
-      }
-      if (ownerAcknowledged && message.abortRequestId != null && message.generationId != null) {
-        const generationId = message.generationId;
-        const abortRequestId = message.abortRequestId;
-        void (async () => {
-          // The owner-lifecycle lease must be durable BEFORE the acknowledgement:
-          // the stopping side releases its own lease the moment this ACK arrives,
-          // and the owner's abort-catch persistence is still ahead. FAIL CLOSED — a
-          // rejected fence suppresses the acknowledgement entirely; the stopping
-          // side stays retryable and its retention lease keeps the user fenced,
-          // while every resignal re-drives this handler until the fence holds.
-          try {
-            await this.beforeAbortAcknowledged?.(streamId, generationId);
-          } catch (error) {
-            logger.error(
-              `[RedisEventTransport] Suppressing abort acknowledgement for ${streamId} — owner fence failed:`,
-              error,
-            );
-            return;
-          }
-          await this.publishAbortAcknowledgement(streamId, generationId, abortRequestId);
-        })();
-      }
+      void this.deliverAbortMessage(streamId, streamState, message);
     }
 
     if (message.type === EventTypes.PREEMPT && message.preempt != null) {
@@ -977,6 +941,41 @@ export class RedisEventTransport implements IEventTransport {
     }
   }
 
+  /** The callback owns both the lifecycle-fence acquisition and the provider
+   * trip. Await it as one operation: a separate pre-ACK hook is too late because
+   * the controller can unwind and release its lease immediately after the trip. */
+  private async deliverAbortMessage(
+    streamId: string,
+    streamState: StreamSubscribers,
+    message: PubSubMessage,
+  ): Promise<void> {
+    let ownerAcknowledged = false;
+    for (const registration of [...streamState.abortCallbacks]) {
+      try {
+        const result =
+          message.generationId == null
+            ? await registration.callback()
+            : await registration.callback(message.generationId);
+        ownerAcknowledged = result === true || ownerAcknowledged;
+      } catch (error) {
+        logger.error(
+          `[RedisEventTransport] Suppressing abort acknowledgement for ${streamId} — owner callback failed:`,
+          error,
+        );
+      }
+    }
+    if (!ownerAcknowledged || message.abortRequestId == null || message.generationId == null) {
+      return;
+    }
+    await this.publishAbortAcknowledgement(
+      streamId,
+      message.generationId,
+      message.abortRequestId,
+    ).catch((error) => {
+      logger.error(`[RedisEventTransport] Failed to publish abort acknowledgement:`, error);
+    });
+  }
+
   private detachStreamSubscribers(streamId: string, state: StreamSubscribers): void {
     this.resetReorderBuffer(streamId);
 
@@ -1404,7 +1403,7 @@ export class RedisEventTransport implements IEventTransport {
    */
   async onAbort(
     streamId: string,
-    callback: (generationId?: number) => void | boolean,
+    callback: (generationId?: number) => void | boolean | Promise<void | boolean>,
   ): Promise<() => void> {
     const channel = CHANNELS.events(streamId);
     const state = this.getOrCreateStreamState(streamId);

packages/api/src/stream/interfaces/IJobStore.ts

@@ -1472,6 +1472,14 @@ export interface IEventTransport {
    *  retainer stops renewing without clearing the shared owner field. */
   hasAbortAcknowledgement?(streamId: string, generationId: number): Promise<boolean>;
 
+  /** Short-lived exact-generation proof that the owner finished every external
+   * persistence write. Failed-handoff retainers use it when a predecessor ends
+   * naturally and therefore never emits an abort acknowledgement. */
+  recordGenerationSettlement?(streamId: string, generationId: number): Promise<boolean>;
+
+  /** Whether exact-generation owner settlement proof is still available. */
+  hasGenerationSettlement?(streamId: string, generationId: number): Promise<boolean>;
+
   /** Publish a predecessor DONE only while the current job's opaque creation
    * attempt still carries that predecessor in its durable receipt chain. */
   emitReplacedDoneConfirmed?(
@@ -1491,8 +1499,9 @@ export interface IEventTransport {
    */
   onAbort?(
     streamId: string,
-    /** Return true only when this replica owns and stopped the tagged generation. */
-    callback: (generationId?: number) => void | boolean,
+    /** Resolve true only after this replica has durably fenced and stopped the
+     * tagged generation. A rejection suppresses the acknowledgement. */
+    callback: (generationId?: number) => void | boolean | Promise<void | boolean>,
   ): void | (() => void) | Promise<void | (() => void)>;
 
   /**

packages/api/src/stream/startup.ts

@@ -2,6 +2,26 @@ import { GenerationJobManager } from './GenerationJobManager';
 import { createStreamServices } from './createStreamServices';
 import { isEnabled } from '../utils/common';
 
+export interface UserFinalizationFallbackStore {
+  renew(
+    userId: string,
+    tenantId: string | undefined,
+    safeLeaseKey: string,
+    expiresAt: Date,
+  ): Promise<void>;
+  clear(userId: string, tenantId: string | undefined, safeLeaseKey: string): Promise<void>;
+  retainAbortDelivery?(
+    userId: string,
+    tenantId: string | undefined,
+    streamId: string,
+  ): Promise<void>;
+  clearAbortDelivery?(
+    userId: string,
+    tenantId: string | undefined,
+    streamId: string,
+  ): Promise<void>;
+}
+
 /**
  * Configures and initializes the generation stream services, and reports whether the
  * resulting job store is SHARED across processes.
@@ -22,11 +42,14 @@ import { isEnabled } from '../utils/common';
  * processes. Callers in a clustered topology MUST fail closed when this is false
  * rather than assume sharing.
  */
-export function configureGenerationStreams(): boolean {
+export function configureGenerationStreams(
+  userFinalizationFallback?: UserFinalizationFallbackStore,
+): boolean {
   const streamServices = createStreamServices();
   GenerationJobManager.configure({
     ...streamServices,
     cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),
+    userFinalizationFallback,
   });
   GenerationJobManager.initialize();
   // Read AFTER configure/initialize so this reflects the store actually in use,

packages/client/src/components/ControlCombobox.tsx

@@ -14,7 +14,10 @@ interface ControlComboboxProps {
   displayValue?: string;
   items: OptionWithIcon[];
   setValue: (value: string) => void;
+  onBlur?: React.FocusEventHandler<HTMLButtonElement>;
   ariaLabel: string;
+  ariaInvalid?: boolean;
+  ariaDescribedBy?: string;
   searchPlaceholder?: string;
   selectPlaceholder?: string;
   isCollapsed: boolean;
@@ -46,7 +49,10 @@ function ControlCombobox({
   displayValue,
   items,
   setValue,
+  onBlur,
   ariaLabel,
+  ariaInvalid,
+  ariaDescribedBy,
   searchPlaceholder,
   selectPlaceholder,
   containerClassName,
@@ -145,6 +151,9 @@ function ControlCombobox({
         store={select}
         id={selectId}
         disabled={disabled}
+        onBlur={onBlur}
+        aria-invalid={ariaInvalid || undefined}
+        aria-describedby={ariaDescribedBy}
         className={cn(
           'flex items-center justify-center gap-2 rounded-full bg-surface-secondary',
           'text-text-primary hover:bg-surface-tertiary',

packages/data-schemas/src/methods/user.finalizationFallback.spec.ts

@@ -0,0 +1,235 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
+
+import type { IUser } from '~/types';
+
+import { createUserMethods } from './user';
+import userSchema from '~/schema/user';
+
+let mongoServer: MongoMemoryServer;
+let User: mongoose.Model<IUser>;
+let methods: ReturnType<typeof createUserMethods>;
+const originalAuthUserCacheMode = process.env.AUTH_USER_CACHE_MODE;
+
+beforeAll(async () => {
+  mongoServer = await MongoMemoryServer.create();
+  await mongoose.connect(mongoServer.getUri());
+  User = mongoose.models.User || mongoose.model<IUser>('User', userSchema);
+  methods = createUserMethods(mongoose);
+});
+
+afterAll(async () => {
+  await mongoose.disconnect();
+  await mongoServer.stop();
+});
+
+beforeEach(async () => {
+  if (originalAuthUserCacheMode === undefined) {
+    delete process.env.AUTH_USER_CACHE_MODE;
+  } else {
+    process.env.AUTH_USER_CACHE_MODE = originalAuthUserCacheMode;
+  }
+  await mongoose.connection.dropDatabase();
+});
+
+afterEach(() => {
+  if (originalAuthUserCacheMode === undefined) {
+    delete process.env.AUTH_USER_CACHE_MODE;
+  } else {
+    process.env.AUTH_USER_CACHE_MODE = originalAuthUserCacheMode;
+  }
+});
+
+describe('user finalization fallback leases', () => {
+  test('renews one lease and counts only leases live at the requested time', async () => {
+    const user = await User.create({
+      name: 'Fallback Lease User',
+      email: 'fallback-lease@example.com',
+      provider: 'local',
+    });
+    const userId = user._id.toString();
+    const firstExpiry = new Date('2026-08-16T12:05:00.000Z');
+    const renewedExpiry = new Date('2026-08-16T12:10:00.000Z');
+
+    await methods.renewUserFinalizationFallbackLease(userId, 'lease_01-safe', firstExpiry);
+    await methods.renewUserFinalizationFallbackLease(userId, 'lease_01-safe', renewedExpiry);
+
+    await expect(
+      methods.countUserFinalizationFallbackLeases(userId, new Date('2026-08-16T12:05:00.000Z')),
+    ).resolves.toBe(1);
+    await expect(
+      methods.countUserFinalizationFallbackLeases(userId, new Date('2026-08-16T12:10:00.000Z')),
+    ).resolves.toBe(0);
+  });
+
+  test('clears only the requested lease', async () => {
+    const user = await User.create({
+      name: 'Fallback Lease Clear User',
+      email: 'fallback-lease-clear@example.com',
+      provider: 'local',
+    });
+    const userId = user._id.toString();
+    const expiresAt = new Date('2026-08-16T12:05:00.000Z');
+    const asOf = new Date('2026-08-16T12:00:00.000Z');
+    await methods.renewUserFinalizationFallbackLease(userId, 'lease_a', expiresAt);
+    await methods.renewUserFinalizationFallbackLease(userId, 'lease_b', expiresAt);
+
+    await methods.clearUserFinalizationFallbackLease(userId, 'lease_a');
+
+    await expect(methods.countUserFinalizationFallbackLeases(userId, asOf)).resolves.toBe(1);
+    await methods.clearUserFinalizationFallbackLease(userId, 'lease_b');
+    await expect(methods.countUserFinalizationFallbackLeases(userId, asOf)).resolves.toBe(0);
+  });
+
+  test('prunes expired leases with an expiry fence and invalidates the auth cache', async () => {
+    process.env.AUTH_USER_CACHE_MODE = 'on';
+    const user = await User.create({
+      name: 'Fallback Lease Prune User',
+      email: 'fallback-lease-prune@example.com',
+      provider: 'openid',
+    });
+    const userId = user._id.toString();
+    const expiredAt = new Date('2026-08-16T11:59:00.000Z');
+    const liveUntil = new Date('2026-08-16T12:10:00.000Z');
+    const asOf = new Date('2026-08-16T12:00:00.000Z');
+    await methods.renewUserFinalizationFallbackLease(userId, 'lease_expired', expiredAt);
+    await methods.renewUserFinalizationFallbackLease(userId, 'lease_live', liveUntil);
+
+    const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`;
+    const cache = {
+      get: jest.fn().mockResolvedValue([]),
+      delete: jest.fn().mockResolvedValue(true),
+    };
+    const getCache = jest.fn().mockReturnValue(cache);
+    const methodsWithCache = createUserMethods(mongoose, { getCache });
+    const updateOne = jest.spyOn(User, 'updateOne');
+
+    await expect(methodsWithCache.countUserFinalizationFallbackLeases(userId, asOf)).resolves.toBe(
+      1,
+    );
+
+    expect(updateOne).toHaveBeenCalledWith(
+      { _id: userId, 'finalizationFallbackLeases.lease_expired': expiredAt },
+      { $unset: { 'finalizationFallbackLeases.lease_expired': '' } },
+      { timestamps: false },
+    );
+    updateOne.mockRestore();
+    const stored = await User.findById(userId).select('finalizationFallbackLeases');
+    const storedLeases = stored?.finalizationFallbackLeases;
+    expect(storedLeases).toBeInstanceOf(Map);
+    if (!(storedLeases instanceof Map)) {
+      throw new Error('Expected finalization fallback leases to hydrate as a Map');
+    }
+    expect(storedLeases.has('lease_expired')).toBe(false);
+    expect(storedLeases.get('lease_live')).toEqual(liveUntil);
+    expect(getCache).toHaveBeenCalledTimes(1);
+    expect(getCache).toHaveBeenCalledWith(CacheKeys.AUTH_USER_DOC);
+    expect(cache.get).toHaveBeenCalledWith(indexKey);
+    expect(cache.delete).toHaveBeenCalledWith(indexKey);
+  });
+
+  test('fails closed when an expired-lease cleanup loses its compare-and-set', async () => {
+    const user = await User.create({
+      name: 'Fallback Lease Cleanup Race User',
+      email: 'fallback-lease-cleanup-race@example.com',
+      provider: 'local',
+    });
+    const userId = user._id.toString();
+    await methods.renewUserFinalizationFallbackLease(
+      userId,
+      'lease_raced',
+      new Date('2026-08-16T11:59:00.000Z'),
+    );
+    const updateOne = jest.spyOn(User, 'updateOne').mockResolvedValueOnce({
+      acknowledged: true,
+      matchedCount: 0,
+      modifiedCount: 0,
+      upsertedCount: 0,
+      upsertedId: null,
+    });
+
+    await expect(
+      methods.countUserFinalizationFallbackLeases(userId, new Date('2026-08-16T12:00:00.000Z')),
+    ).resolves.toBe(1);
+
+    updateOne.mockRestore();
+  });
+
+  test('fails closed when the lease document cannot be read', async () => {
+    jest.spyOn(User, 'findById').mockImplementationOnce(() => {
+      throw new Error('mongo read failed');
+    });
+
+    await expect(
+      methods.countUserFinalizationFallbackLeases(new mongoose.Types.ObjectId().toString()),
+    ).resolves.toBe(1);
+  });
+
+  test('rejects keys that are unsafe in a Mongo map path', async () => {
+    const user = await User.create({
+      name: 'Fallback Lease Key User',
+      email: 'fallback-lease-key@example.com',
+      provider: 'local',
+    });
+    const userId = user._id.toString();
+    const expiresAt = new Date(Date.now() + 60_000);
+
+    await expect(
+      methods.renewUserFinalizationFallbackLease(userId, 'unsafe.key', expiresAt),
+    ).rejects.toThrow(/safe opaque key/);
+    await expect(methods.clearUserFinalizationFallbackLease(userId, '$unsafe')).rejects.toThrow(
+      /safe opaque key/,
+    );
+    await expect(
+      methods.renewUserFinalizationFallbackLease(userId, '__proto__', expiresAt),
+    ).rejects.toThrow(/safe opaque key/);
+    await expect(methods.countUserFinalizationFallbackLeases(userId)).resolves.toBe(0);
+  });
+
+  test('fails renewal when the user no longer exists', async () => {
+    await expect(
+      methods.renewUserFinalizationFallbackLease(
+        new mongoose.Types.ObjectId().toString(),
+        'lease_missing_user',
+        new Date(Date.now() + 60_000),
+      ),
+    ).rejects.toThrow(/missing user/);
+  });
+
+  test('invalidates cached auth user documents after renew and clear mutations', async () => {
+    process.env.AUTH_USER_CACHE_MODE = 'on';
+    const user = await User.create({
+      name: 'Fallback Lease Cache User',
+      email: 'fallback-lease-cache@example.com',
+      provider: 'openid',
+    });
+    const userId = user._id.toString();
+    const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`;
+    const cache = {
+      get: jest.fn().mockResolvedValue([]),
+      delete: jest.fn().mockResolvedValue(true),
+    };
+    const getCache = jest.fn().mockReturnValue(cache);
+    const methodsWithCache = createUserMethods(mongoose, {
+      getCache,
+    });
+
+    await methodsWithCache.renewUserFinalizationFallbackLease(
+      userId,
+      'lease_cache',
+      new Date(Date.now() + 60_000),
+    );
+    await methodsWithCache.clearUserFinalizationFallbackLease(userId, 'lease_cache');
+
+    expect(getCache).toHaveBeenCalledTimes(2);
+    expect(getCache).toHaveBeenNthCalledWith(1, CacheKeys.AUTH_USER_DOC);
+    expect(getCache).toHaveBeenNthCalledWith(2, CacheKeys.AUTH_USER_DOC);
+    expect(cache.get).toHaveBeenCalledTimes(2);
+    expect(cache.get).toHaveBeenNthCalledWith(1, indexKey);
+    expect(cache.get).toHaveBeenNthCalledWith(2, indexKey);
+    expect(cache.delete).toHaveBeenCalledTimes(2);
+    expect(cache.delete).toHaveBeenNthCalledWith(1, indexKey);
+    expect(cache.delete).toHaveBeenNthCalledWith(2, indexKey);
+  });
+});

packages/data-schemas/src/methods/user.ts

@@ -16,6 +16,13 @@ import { signPayload } from '~/crypto';
  *  cannot see the abort fences must never conclude settlement. */
 export const UNREADABLE_ABORT_FENCE = '__unreadable__';
 
+const SAFE_FINALIZATION_FALLBACK_LEASE_KEY = /^[A-Za-z0-9_-]{1,128}$/;
+const RESERVED_FINALIZATION_FALLBACK_LEASE_KEYS = new Set([
+  '__proto__',
+  'constructor',
+  'prototype',
+]);
+
 /** Default JWT session expiry: 15 minutes in milliseconds */
 export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15;
 /** Minimum age before an explicitly offline operator may recover an abandoned deletion fence. */
@@ -155,6 +162,13 @@ export function createUserMethods(
   addUserAbortFence: (userId: string, streamId: string) => Promise<void>;
   clearUserAbortFence: (userId: string, streamId: string) => Promise<void>;
   getUserAbortFences: (userId: string) => Promise<string[]>;
+  renewUserFinalizationFallbackLease: (
+    userId: string,
+    safeLeaseKey: string,
+    expiresAt: Date,
+  ) => Promise<void>;
+  clearUserFinalizationFallbackLease: (userId: string, safeLeaseKey: string) => Promise<void>;
+  countUserFinalizationFallbackLeases: (userId: string, asOf?: Date) => Promise<number>;
   updateUserPlugins: (
     userId: string,
     plugins: string[] | undefined,
@@ -521,6 +535,119 @@ export function createUserMethods(
     }
   }
 
+  function isSafeFinalizationFallbackLeaseKey(safeLeaseKey: string): boolean {
+    return (
+      SAFE_FINALIZATION_FALLBACK_LEASE_KEY.test(safeLeaseKey) &&
+      !RESERVED_FINALIZATION_FALLBACK_LEASE_KEYS.has(safeLeaseKey)
+    );
+  }
+
+  function assertSafeFinalizationFallbackLeaseKey(safeLeaseKey: string): void {
+    if (!isSafeFinalizationFallbackLeaseKey(safeLeaseKey)) {
+      throw new Error('Finalization fallback lease key must be a safe opaque key');
+    }
+  }
+
+  /** Renews one Mongo-backed fallback lease. This mutation throws if the user no
+   *  longer exists: a holder must never mistake a missing deletion barrier for a
+   *  successfully renewed lease. */
+  async function renewUserFinalizationFallbackLease(
+    userId: string,
+    safeLeaseKey: string,
+    expiresAt: Date,
+  ): Promise<void> {
+    assertSafeFinalizationFallbackLeaseKey(safeLeaseKey);
+    if (!(expiresAt instanceof Date) || !Number.isFinite(expiresAt.getTime())) {
+      throw new Error('Finalization fallback lease expiry must be a valid date');
+    }
+    const User = mongoose.models.User;
+    const result = await User.updateOne(
+      { _id: userId },
+      { $set: { [`finalizationFallbackLeases.${safeLeaseKey}`]: expiresAt } },
+      { timestamps: false },
+    );
+    if (result.matchedCount !== 1) {
+      throw new Error('Cannot renew finalization fallback lease for a missing user');
+    }
+    await invalidateAuthUserDocCache(userId);
+  }
+
+  /** Clears one fallback lease without disturbing contenders on the same user. */
+  async function clearUserFinalizationFallbackLease(
+    userId: string,
+    safeLeaseKey: string,
+  ): Promise<void> {
+    assertSafeFinalizationFallbackLeaseKey(safeLeaseKey);
+    const User = mongoose.models.User;
+    await User.updateOne(
+      { _id: userId },
+      { $unset: { [`finalizationFallbackLeases.${safeLeaseKey}`]: '' } },
+      { timestamps: false },
+    );
+    await invalidateAuthUserDocCache(userId);
+  }
+
+  /** Counts leases whose expiry is strictly after `asOf` and opportunistically
+   *  removes expired entries. Cleanup is fenced on every observed expiry so it can
+   *  never erase a lease renewed after this read. A malformed entry or read/cleanup
+   *  failure counts as live: deletion may defer unnecessarily, but can never treat
+   *  unreadable finalization work as settled. */
+  async function countUserFinalizationFallbackLeases(
+    userId: string,
+    asOf: Date = new Date(),
+  ): Promise<number> {
+    if (!(asOf instanceof Date) || !Number.isFinite(asOf.getTime())) {
+      return 1;
+    }
+    const User = mongoose.models.User;
+    try {
+      const user = await User.findById(userId)
+        .select('finalizationFallbackLeases')
+        .lean<Pick<IUser, 'finalizationFallbackLeases'>>();
+      const leases = user?.finalizationFallbackLeases;
+      if (leases == null) {
+        return 0;
+      }
+      const entries: Array<[string, Date]> =
+        leases instanceof Map ? [...leases.entries()] : Object.entries(leases);
+      const expiredEntries: Array<[string, Date]> = [];
+      let liveCount = 0;
+      for (const [safeLeaseKey, expiresAt] of entries) {
+        const timestamp = expiresAt instanceof Date ? expiresAt.getTime() : Number.NaN;
+        if (!Number.isFinite(timestamp) || timestamp > asOf.getTime()) {
+          liveCount += 1;
+          continue;
+        }
+        if (isSafeFinalizationFallbackLeaseKey(safeLeaseKey)) {
+          expiredEntries.push([safeLeaseKey, expiresAt]);
+        }
+      }
+      if (expiredEntries.length > 0) {
+        const expectedExpiries: Record<string, Date> = {};
+        const unsetExpiries: Record<string, ''> = {};
+        for (const [safeLeaseKey, expiresAt] of expiredEntries) {
+          const path = `finalizationFallbackLeases.${safeLeaseKey}`;
+          expectedExpiries[path] = expiresAt;
+          unsetExpiries[path] = '';
+        }
+        const cleanup = await User.updateOne(
+          { _id: userId, ...expectedExpiries },
+          { $unset: unsetExpiries },
+          { timestamps: false },
+        );
+        if (cleanup.modifiedCount === 0) {
+          // A concurrent renew/clear changed the snapshot. Defer once rather than
+          // return a stale zero that could hide the newly renewed lease.
+          return Math.max(liveCount, 1);
+        }
+        await invalidateAuthUserDocCache(userId);
+      }
+      return liveCount;
+    } catch {
+      return 1;
+    }
+  }
+
   /** Rotation stamp for the deferred-deletion sweep window. Bookkeeping only, but it
    *  still mutates user documents, so cached copies are dropped (best-effort — these
    *  users are already refused at authentication behind the barrier). */
@@ -942,6 +1069,9 @@ export function createUserMethods(
     addUserAbortFence,
     clearUserAbortFence,
     getUserAbortFences,
+    renewUserFinalizationFallbackLease,
+    clearUserFinalizationFallbackLease,
+    countUserFinalizationFallbackLeases,
     updateUserPlugins,
     toggleUserMemories,
     updateUserStatefulCodeEnvironment,

packages/data-schemas/src/schema/user.ts

@@ -201,6 +201,13 @@ const userSchema: Schema<IUser> = new Schema<IUser>(
       type: [String],
       default: undefined,
     },
+    /** Expiring failover leases for persistence that remains live while its primary
+     *  finalization lease store is unavailable. Absent on unaffected users. */
+    finalizationFallbackLeases: {
+      type: Map,
+      of: Date,
+      default: undefined,
+    },
     /** Field for external source identification (for consistency with TPrincipal schema) */
     idOnTheSource: {
       type: String,

packages/data-schemas/src/types/user.ts

@@ -71,6 +71,9 @@ export interface IUser extends Document {
   /** Stream ids whose deletion-side abort has not been acknowledged; durable and
    *  TTL-free so a publication outage can never be mistaken for settlement. */
   deletionAbortFences?: string[];
+  /** Expiring Mongo-backed leases used when the primary finalization lease store
+   *  cannot be renewed. Keys are pre-sanitized opaque lease identities. */
+  finalizationFallbackLeases?: Map<string, Date> | Record<string, Date>;
   /** Field for external source identification (for consistency with TPrincipal schema) */
   idOnTheSource?: string;
   tenantId?: string;