diff --git a/packages/api/src/schedules/engine.ts b/packages/api/src/schedules/engine.ts index 1e3926b16b..130b927962 100644 --- a/packages/api/src/schedules/engine.ts +++ b/packages/api/src/schedules/engine.ts @@ -211,12 +211,20 @@ export function startScheduleEngine(deps: ScheduleEngineDeps): ScheduleEngine { * owner's context via `runInTenantContext`. */ async function runTick(): Promise { - // Do NOT gate claims on the BASE config's `enabled`: schedules can be enabled - // per user/role/tenant even when the base config disables them, so gating here - // would silently never fire those users' occurrences. The fire path re-resolves - // the OWNER's limits and skips ('disabled') any occurrence whose owner has the - // feature off, so an owner-scoped disable is still honored. The base config only - // supplies the per-tick claim budget (a global throttle). + // GLOBAL kill switch: stop claiming entirely. This is the operator's hard stop + // (SCHEDULES_DISABLED, or `interface.schedules: false` in the BASE config), which + // no principal override can widen — distinct from per-principal availability below. + // Deliberately gates CLAIMS only; reconcile() is never gated, because in-flight + // runs must still be settled while the feature is off or they strand `started` + // rows and leak capacity forever. + if (await deps.isGloballyDisabled()) { + return 0; + } + // Do NOT gate claims on the per-principal `enabled`: schedules can be enabled per + // user/role/tenant, so gating here would silently never fire those users' + // occurrences. The fire path re-resolves the OWNER's limits and skips ('disabled') + // any occurrence whose owner has the feature off, so an owner-scoped disable is + // still honored. The base config only supplies the per-tick claim budget. const limits = await deps.getLimits(); let fired = 0; // Cap on ACTIVE scheduled runs, not just per-tick starts: the loopback chat diff --git a/packages/api/src/schedules/service.spec.ts b/packages/api/src/schedules/service.spec.ts index cb6f7977f1..6dd861981b 100644 --- a/packages/api/src/schedules/service.spec.ts +++ b/packages/api/src/schedules/service.spec.ts @@ -16,6 +16,7 @@ type ActiveRun = { scheduleId: string; scheduledFor: Date; conversationId?: stri function makeService( getActiveRunsForUser: jest.Mock, [string]>, + getAppConfig?: SchedulesServiceDeps['getAppConfig'], ): ReturnType { const methods = { disableUserSchedulesForDeletion: jest.fn(async () => undefined), @@ -24,7 +25,7 @@ function makeService( }; const deps = { methods, - getAppConfig: jest.fn(async () => ({})), + getAppConfig: getAppConfig ?? jest.fn(async () => ({})), findUserById: jest.fn(async () => null), findBalance: jest.fn(async () => null), upsertBalance: jest.fn(async () => null), @@ -91,3 +92,53 @@ describe('quiesceUserSchedules drain wait', () => { expect(getActive).toHaveBeenCalledTimes(1); }); }); + +describe('global kill switch', () => { + const noRuns = () => jest.fn, [string]>().mockResolvedValue([]); + + afterEach(() => { + delete process.env.SCHEDULES_DISABLED; + }); + + it('is off by default', async () => { + const service = makeService(noRuns()); + expect(await service.engineDeps.isGloballyDisabled()).toBe(false); + }); + + it('trips on the SCHEDULES_DISABLED env lever without reading config', async () => { + process.env.SCHEDULES_DISABLED = 'true'; + // Throwing getAppConfig proves the env lever works even when the config plane is + // unhealthy — the case where a config-dependent kill switch would fail. + const getAppConfig = jest.fn(async () => { + throw new Error('config plane down'); + }) as unknown as SchedulesServiceDeps['getAppConfig']; + const service = makeService(noRuns(), getAppConfig); + expect(await service.engineDeps.isGloballyDisabled()).toBe(true); + expect(getAppConfig).not.toHaveBeenCalled(); + }); + + it('trips on `interface.schedules: false` read from the BASE config only', async () => { + const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) => + options?.baseOnly === true + ? { interfaceConfig: { schedules: false } } + : // A principal-merged view that re-enables must NOT be consulted: the global + // stop is base-only so no role/user/tenant override can widen past it. + { interfaceConfig: { schedules: true } }, + ) as unknown as SchedulesServiceDeps['getAppConfig']; + const service = makeService(noRuns(), getAppConfig); + expect(await service.engineDeps.isGloballyDisabled()).toBe(true); + expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true }); + }); + + it('does not trip when only a principal-merged config disables it', async () => { + // Per-principal availability is NOT the global stop; the engine keeps claiming so + // other principals still fire, and the fire path skips this owner's occurrences. + const getAppConfig = jest.fn(async (options?: { baseOnly?: boolean }) => + options?.baseOnly === true + ? { interfaceConfig: { schedules: true } } + : { interfaceConfig: { schedules: false } }, + ) as unknown as SchedulesServiceDeps['getAppConfig']; + const service = makeService(noRuns(), getAppConfig); + expect(await service.engineDeps.isGloballyDisabled()).toBe(false); + }); +}); diff --git a/packages/api/src/schedules/service.ts b/packages/api/src/schedules/service.ts index e3700cacf2..5f482f178d 100644 --- a/packages/api/src/schedules/service.ts +++ b/packages/api/src/schedules/service.ts @@ -25,6 +25,7 @@ import { DEFAULT_SCHEDULE_LIMITS } from './types'; import { getBalanceConfig } from '../app/config'; import { startScheduleEngine } from './engine'; import { withCapacitySlot } from './capacity'; +import { isEnabled } from '../utils/common'; /** Recordable terminal/paused run outcome, as accepted by `recordRunOutcome`. */ type ScheduleRunOutcomeStatus = Parameters[0]['status']; @@ -369,6 +370,19 @@ export function createSchedulesService(deps: SchedulesServiceDeps): SchedulesSer // Counted in system scope so the cap is GLOBAL — a per-owner (tenant-scoped) // count would let multiple tenants collectively exceed fireConcurrency. countActiveRunsGlobal: () => runAsSystem(() => methods.countActiveRuns()), + isGloballyDisabled: async () => { + // Env first: an incident lever that must work even if the DB/config plane is the + // thing failing (a kill switch that needs a healthy DB is the one that fails when + // you need it). + if (isEnabled(process.env.SCHEDULES_DISABLED)) { + return true; + } + // BASE config only: DB principal overrides can narrow availability but must never + // widen past an operator's global stop, so `schedules: false` in librechat.yaml is + // genuinely non-overridable rather than emergent from the override filters. + const base = await deps.getAppConfig({ baseOnly: true }); + return base?.interfaceConfig?.schedules === false; + }, // Occupancy is read in SYSTEM scope so the cap is global across tenants (the // owner's tenant context would only see its own runs); the claim itself stays in // the caller's context so the inserted row keeps correct tenant ownership. diff --git a/packages/api/src/schedules/types.ts b/packages/api/src/schedules/types.ts index 7f0821f02f..bd81ec796a 100644 --- a/packages/api/src/schedules/types.ts +++ b/packages/api/src/schedules/types.ts @@ -97,6 +97,14 @@ export interface ScheduleEngineDeps { clearReconciledJob: (conversationId: string, identity: JobIdentity) => Promise; /** Global in-flight scheduled-run count (system tenant scope) for the fire cap. */ countActiveRunsGlobal: () => Promise; + /** + * The GLOBAL kill switch, deliberately distinct from per-principal availability. + * True when scheduling is stopped for the whole deployment: the SCHEDULES_DISABLED + * env lever (works even when the config plane is unhealthy), or `interface.schedules: + * false` in the BASE config — read base-only so no role/user/tenant override can + * re-enable it. Checked once per engine tick, so the uncached read is negligible. + */ + isGloballyDisabled: () => Promise; /** * Runs `claim` against the lowest free GLOBAL capacity slot, retrying the next slot * when the unique partial index rejects a collision. Enforces fireConcurrency in the