diff --git a/packages/api/src/app/shutdown.spec.ts b/packages/api/src/app/shutdown.spec.ts index 16f2de5c51..9e0e76f199 100644 --- a/packages/api/src/app/shutdown.spec.ts +++ b/packages/api/src/app/shutdown.spec.ts @@ -347,4 +347,63 @@ describe('setupGracefulShutdown', () => { expect(calls).toEqual(['async-done']); expect(exitSpy).toHaveBeenCalledWith(0); }); + it('disarms the force-exit timer when shutdown state is reset', async () => { + jest.useFakeTimers(); + try { + // A drain that never settles: the server close callback is never invoked, + // so `shutdown` stays awaiting and never reaches its own `clearTimeout`. + jest.spyOn(server, 'close').mockImplementation(() => server); + setupGracefulShutdown(server); + triggerSignal('SIGTERM'); + await Promise.resolve(); + + // The safety net is armed and would exit the process on its own. + __resetShutdownStateForTests(); + jest.advanceTimersByTime(120_000); + + // Without the reset clearing it, this timer fires long after the suite + // that armed it has finished, killing the run with code 1. + expect(exitSpy).not.toHaveBeenCalledWith(1); + } finally { + jest.useRealTimers(); + } + }); + + it("keeps a later shutdown's safety net when an earlier drain settles late", async () => { + // `setImmediate` stays real so the first shutdown's continuation can actually + // reach its `finally`; only the force-exit timer is faked. + jest.useFakeTimers({ doNotFake: ['setImmediate'] }); + try { + let releaseFirstClose: (() => void) | undefined; + jest.spyOn(server, 'close').mockImplementation((cb?: (err?: Error) => void) => { + if (cb) { + releaseFirstClose = () => cb(); + } + return server; + }); + setupGracefulShutdown(server); + triggerSignal('SIGTERM'); + await flush(); + + // A second shutdown arms its own net after the first is reset away. + __resetShutdownStateForTests(); + const secondServer = http.createServer(); + Object.defineProperty(secondServer, 'listening', { value: true, configurable: true }); + jest.spyOn(secondServer, 'close').mockImplementation(() => secondServer); + setupGracefulShutdown(secondServer); + triggerSignal('SIGTERM'); + await flush(); + + // The first drain settles only now; its `finally` must not disarm the second. + releaseFirstClose?.(); + await flush(); + await flush(); + await flush(); + + jest.advanceTimersByTime(120_000); + expect(exitSpy).toHaveBeenCalledWith(1); + } finally { + jest.useRealTimers(); + } + }); }); diff --git a/packages/api/src/app/shutdown.ts b/packages/api/src/app/shutdown.ts index e8662296d5..656936af3f 100644 --- a/packages/api/src/app/shutdown.ts +++ b/packages/api/src/app/shutdown.ts @@ -23,6 +23,7 @@ const tasks: ShutdownTask[] = []; let nextRegistrationOrder = 0; let isShuttingDown = false; let httpServer: Server | null = null; +let forceExitTimer: NodeJS.Timeout | null = null; /** * Register a cleanup task for graceful shutdown. Post-drain is the default phase. @@ -73,6 +74,11 @@ export function __resetShutdownStateForTests(): void { nextRegistrationOrder = 0; isShuttingDown = false; httpServer = null; + /** A drain that never settles leaves this armed. It is `unref`'d, so it does + * not hold the process open — but it does fire if anything else keeps the + * process alive past the timeout, exiting a suite that had long since moved + * on with code 1 and no attributable failure. */ + clearForceExitTimer(); } async function runShutdownTasks(phase: ShutdownPhase): Promise { @@ -93,6 +99,13 @@ async function runShutdownTasks(phase: ShutdownPhase): Promise { } } +function clearForceExitTimer(): void { + if (forceExitTimer) { + clearTimeout(forceExitTimer); + forceExitTimer = null; + } +} + async function shutdown(signal: NodeJS.Signals): Promise { if (isShuttingDown) { return; @@ -100,24 +113,33 @@ async function shutdown(signal: NodeJS.Signals): Promise { isShuttingDown = true; logger.info(`Received ${signal}, draining HTTP server...`); + /** Owned locally so a late `finally` from a superseded drain cannot clear the + * safety net belonging to a shutdown that started after it. */ const forceExit = setTimeout(() => { logger.warn(`Graceful shutdown exceeded ${SHUTDOWN_TIMEOUT_MS}ms, forcing exit`); process.exit(1); }, SHUTDOWN_TIMEOUT_MS); forceExit.unref(); + forceExitTimer = forceExit; let exitCode = 0; - const serverClosePromise = closeHttpServer().catch((err) => { - logger.error('Error closing HTTP server during graceful shutdown:', err); - exitCode = 1; - }); + try { + const serverClosePromise = closeHttpServer().catch((err) => { + logger.error('Error closing HTTP server during graceful shutdown:', err); + exitCode = 1; + }); - await runShutdownTasks('pre-drain'); - await serverClosePromise; - await runShutdownTasks('post-drain'); + await runShutdownTasks('pre-drain'); + await serverClosePromise; + await runShutdownTasks('post-drain'); + } finally { + clearTimeout(forceExit); + if (forceExitTimer === forceExit) { + forceExitTimer = null; + } + } - clearTimeout(forceExit); logger.info('Graceful shutdown complete, exiting'); process.exit(exitCode); }