From 9e5f6f8cedf4d37240ab13dfe0cfff8df4ed27c8 Mon Sep 17 00:00:00 2001 From: Lia Date: Sun, 20 Sep 2026 04:27:16 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=A5=A2=20fix:=20Exclude=20Prompt=20Inputs?= =?UTF-8?q?=20From=20Terminal=20Events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Terminal events carried prompt-building inputs into every place a FINAL is kept: the runtime cache, the durable Redis job hash, Pub/Sub publication and late/cross-replica replay. Neither terminal producer projected the payload, so a 706 KiB attachment-heavy FINAL was stored and published at full size. sanitizeMessageForTransmit removed fileContext and files[].text from requestMessage at the controller, but never image_urls, and responseMessage was not sanitized at all. projectTerminalEvent is one shared, schema-aware projection applied at both GenerationJobManager.emitDone() and publishTerminalClaim(), and on read where a stored record may predate it. It excludes fileContext, image_urls and embedded file bodies from requestMessage, responseMessage and every runMessages entry, while preserving message content, attachment references and display metadata, and the terminal/reconciliation protocol fields. publishTerminalClaim compared publicationEvent !== finalEvent to decide persistenceFailed. Projection allocates a new object, so that identity check now compares against the projected intendedEvent; a successfully persisted FINAL would otherwise have been reported as a persistence failure. --- .../api/src/stream/GenerationJobManager.ts | 36 +- .../__tests__/terminalProjection.spec.ts | 272 +++++++++++++ ...jectionBoundary.stream_integration.spec.ts | 362 ++++++++++++++++++ packages/api/src/stream/index.ts | 10 + packages/api/src/stream/terminalProjection.ts | 174 +++++++++ packages/api/src/utils/message.ts | 12 +- 6 files changed, 856 insertions(+), 10 deletions(-) create mode 100644 packages/api/src/stream/__tests__/terminalProjection.spec.ts create mode 100644 packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts create mode 100644 packages/api/src/stream/terminalProjection.ts diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts index a547193cc0..b0329cdb92 100644 --- a/packages/api/src/stream/GenerationJobManager.ts +++ b/packages/api/src/stream/GenerationJobManager.ts @@ -62,6 +62,7 @@ import { STEER_QUEUE_MAX_DEPTH, } from './interfaces/IJobStore'; import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery'; +import { projectTerminalEvent } from './terminalProjection'; import { assertJobStoreV2 } from './jobStoreCapabilities'; /** @@ -3012,7 +3013,10 @@ class GenerationJobManagerClass { let finalEvent: t.ServerSentEvent | undefined; if (jobData.finalEvent) { try { - finalEvent = JSON.parse(jobData.finalEvent) as t.ServerSentEvent; + /** Records written before projection shipped, or by an older replica, + * are projected on read so replay never re-delivers or re-caches an + * oversized payload. */ + finalEvent = projectTerminalEvent(JSON.parse(jobData.finalEvent) as t.ServerSentEvent); } catch { // Ignore parse errors } @@ -3959,7 +3963,13 @@ class GenerationJobManagerClass { conversationId: claim.conversationId, status: claim.status, }); - const desiredEvent = finalEvent ?? reconcileEvent; + /** The caller's event still carries prompt-building inputs, so everything + * this method stores, publishes or caches uses the projected payload. The + * projection allocates a new object whenever it excludes anything, so + * `intendedEvent` — not the caller's reference — is what the success + * bookkeeping below compares identity against. */ + const intendedEvent = finalEvent == null ? null : projectTerminalEvent(finalEvent); + const desiredEvent = intendedEvent ?? reconcileEvent; let publicationEvent: t.ServerSentEvent | null = null; let durable = false; @@ -3979,7 +3989,12 @@ class GenerationJobManagerClass { settledJob.terminalPersistencePending !== true && settledJob.finalEvent ) { - publicationEvent = JSON.parse(settledJob.finalEvent) as t.ServerSentEvent; + /** Written by whichever side won the CAS, possibly a replica that + * predates projection. Project on read so a legacy oversized record + * is not republished unchanged. */ + publicationEvent = projectTerminalEvent( + JSON.parse(settledJob.finalEvent) as t.ServerSentEvent, + ); durable = true; } } @@ -4000,9 +4015,9 @@ class GenerationJobManagerClass { runtime.finalEvent = publicationEvent; } const persistenceFailed = - finalEvent == null || + intendedEvent == null || !durable || - publicationEvent !== finalEvent || + publicationEvent !== intendedEvent || ('reconcile' in publicationEvent && publicationEvent.reconcile === true); try { @@ -5419,7 +5434,11 @@ class GenerationJobManagerClass { let finalEvent = runtime.finalEvent; if (!finalEvent && terminalJob.finalEvent) { try { - finalEvent = JSON.parse(terminalJob.finalEvent) as t.ServerSentEvent; + /** Same mixed-deployment concern as the cross-replica runtime: a + * stored record may predate projection. */ + finalEvent = projectTerminalEvent( + JSON.parse(terminalJob.finalEvent) as t.ServerSentEvent, + ); } catch (err) { logger.warn( `[GenerationJobManager] Failed to parse stored final event for ${streamId}:`, @@ -8742,9 +8761,12 @@ class GenerationJobManagerClass { */ async emitDone( streamId: string, - event: t.ServerSentEvent, + rawEvent: t.ServerSentEvent, expectedCreatedAt?: number, ): Promise { + /** Exclude prompt-building inputs before this event reaches the runtime + * cache, the durable job hash or the transport. */ + const event = projectTerminalEvent(rawEvent); const runtime = this.runtimeState.get(streamId); const generationId = expectedCreatedAt ?? runtime?.createdAt; const matchingRuntime = diff --git a/packages/api/src/stream/__tests__/terminalProjection.spec.ts b/packages/api/src/stream/__tests__/terminalProjection.spec.ts new file mode 100644 index 0000000000..e58acac3dc --- /dev/null +++ b/packages/api/src/stream/__tests__/terminalProjection.spec.ts @@ -0,0 +1,272 @@ +import type { FinalEvent, ServerSentEvent } from '~/types/events'; +import { projectTerminalEvent } from '../terminalProjection'; + +const byteLength = (value: unknown): number => Buffer.byteLength(JSON.stringify(value), 'utf8'); + +/** `FinalMessageFields` carries an index signature, so nested collections read + * back as `unknown`; these keep the assertions readable without `any`. */ +const entries = (value: unknown): Array> => + (value ?? []) as Array>; +const filesOf = (message: unknown): Array> => + entries((message as { files?: unknown } | null | undefined)?.files); +const attachmentsOf = (message: unknown): Array> => + entries((message as { attachments?: unknown } | null | undefined)?.attachments); + +/** A terminal event shaped like the one `request.js` builds, with the transient + * prompt-building inputs still attached. */ +function buildAttachmentHeavyFinal(bodyChars: number): FinalEvent { + const body = 'D'.repeat(bodyChars); + const base64 = 'A'.repeat(bodyChars); + return { + final: true, + terminalStatus: 'complete', + conversation: { conversationId: 'conv-1', title: 'Report' }, + title: 'Report', + requestMessage: { + messageId: 'um-1', + parentMessageId: 'root', + conversationId: 'conv-1', + text: 'Summarize the attachment', + sender: 'User', + isCreatedByUser: true, + fileContext: body, + image_urls: [{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64}` } }], + files: [ + { + file_id: 'f-1', + filename: 'report.pdf', + filepath: '/uploads/report.pdf', + type: 'application/pdf', + text: body, + _id: 'mongo-id', + __v: 0, + }, + ], + }, + responseMessage: { + messageId: 'rm-1', + parentMessageId: 'um-1', + conversationId: 'conv-1', + sender: 'AI', + isCreatedByUser: false, + content: [{ type: 'text', text: 'The report concludes that latency improved.' }], + attachments: [ + { file_id: 'a-1', filename: 'chart.png', filepath: '/out/chart.png', text: body }, + ], + }, + }; +} + +describe('projectTerminalEvent', () => { + describe('exclusion of transient prompt data', () => { + it('excludes fileContext, image_urls and embedded file bodies', () => { + const projected = projectTerminalEvent(buildAttachmentHeavyFinal(64)); + + expect(projected.requestMessage).not.toHaveProperty('fileContext'); + expect(projected.requestMessage).not.toHaveProperty('image_urls'); + expect(filesOf(projected.requestMessage)[0]).not.toHaveProperty('text'); + expect(attachmentsOf(projected.responseMessage)[0]).not.toHaveProperty('text'); + }); + + it('excludes storage bookkeeping from nested file entries', () => { + const projected = projectTerminalEvent(buildAttachmentHeavyFinal(64)); + const file = filesOf(projected.requestMessage)[0]; + + expect(file).not.toHaveProperty('_id'); + expect(file).not.toHaveProperty('__v'); + }); + + it('excludes transient fields from every runMessages entry', () => { + const event: FinalEvent = { + final: true, + runMessages: [ + { messageId: 'r-1', text: 'kept', fileContext: 'X'.repeat(128) }, + { + messageId: 'r-2', + text: 'also kept', + image_urls: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,AAA' } }], + files: [{ file_id: 'f-2', filename: 'a.pdf', text: 'Y'.repeat(128) }], + }, + ], + }; + + const projected = projectTerminalEvent(event); + + expect(projected.runMessages?.[0]).not.toHaveProperty('fileContext'); + expect(projected.runMessages?.[0]?.text).toBe('kept'); + expect(projected.runMessages?.[1]).not.toHaveProperty('image_urls'); + expect(projected.runMessages?.[1]?.text).toBe('also kept'); + expect(filesOf(projected.runMessages?.[1])[0]).not.toHaveProperty('text'); + }); + }); + + describe('preservation of authoritative final state', () => { + it('keeps legitimate message content and structured response content', () => { + const projected = projectTerminalEvent(buildAttachmentHeavyFinal(64)); + + expect(projected.requestMessage?.text).toBe('Summarize the attachment'); + expect(projected.responseMessage?.content).toEqual([ + { type: 'text', text: 'The report concludes that latency improved.' }, + ]); + }); + + it('keeps attachment references and display metadata needed to render FINAL', () => { + const projected = projectTerminalEvent(buildAttachmentHeavyFinal(64)); + + expect(filesOf(projected.requestMessage)[0]).toEqual({ + file_id: 'f-1', + filename: 'report.pdf', + filepath: '/uploads/report.pdf', + type: 'application/pdf', + }); + expect(attachmentsOf(projected.responseMessage)[0]).toEqual({ + file_id: 'a-1', + filename: 'chart.png', + filepath: '/out/chart.png', + }); + }); + + it('keeps the terminal and reconciliation protocol fields', () => { + const event: FinalEvent = { + final: true, + reconcile: true, + reconcileReason: 'generation_replaced', + terminalStatus: 'aborted', + generationCreatedAt: 1234, + aborted: true, + earlyAbort: true, + title: 'T', + conversation: { conversationId: 'conv-1' }, + pendingSteers: [{ id: 's-1' } as never], + error: { message: 'boom' }, + requestMessage: { messageId: 'um-1', fileContext: 'drop me' }, + }; + + const projected = projectTerminalEvent(event); + + expect(projected).toMatchObject({ + final: true, + reconcile: true, + reconcileReason: 'generation_replaced', + terminalStatus: 'aborted', + generationCreatedAt: 1234, + aborted: true, + earlyAbort: true, + title: 'T', + conversation: { conversationId: 'conv-1' }, + error: { message: 'boom' }, + }); + expect(projected.pendingSteers).toEqual(event.pendingSteers); + expect(projected.requestMessage).not.toHaveProperty('fileContext'); + }); + + it('does not remove a legitimate text field from a message or content part', () => { + const event: FinalEvent = { + final: true, + responseMessage: { + messageId: 'rm-1', + text: 'top level answer text', + content: [{ type: 'text', text: { value: 'nested structured text' } }], + }, + }; + + const projected = projectTerminalEvent(event); + + expect(projected.responseMessage?.text).toBe('top level answer text'); + expect(projected.responseMessage?.content).toEqual([ + { type: 'text', text: { value: 'nested structured text' } }, + ]); + }); + }); + + describe('optional and null semantics', () => { + it('preserves an explicit null message slot', () => { + const event: FinalEvent = { + final: true, + earlyAbort: true, + requestMessage: null, + responseMessage: null, + }; + + const projected = projectTerminalEvent(event); + + expect(projected.requestMessage).toBeNull(); + expect(projected.responseMessage).toBeNull(); + }); + + it('leaves an absent slot absent rather than inventing it', () => { + const projected = projectTerminalEvent({ final: true } as FinalEvent); + + expect('requestMessage' in projected).toBe(false); + expect('responseMessage' in projected).toBe(false); + expect('runMessages' in projected).toBe(false); + }); + }); + + describe('input ownership and idempotence', () => { + it('does not mutate the input event or its nested objects', () => { + const event = buildAttachmentHeavyFinal(64); + const snapshot = structuredClone(event); + + projectTerminalEvent(event); + + expect(event).toEqual(snapshot); + expect(event.requestMessage).toHaveProperty('fileContext'); + expect(event.requestMessage).toHaveProperty('image_urls'); + expect(filesOf(event.requestMessage)[0]).toHaveProperty('text'); + }); + + it('is idempotent', () => { + const once = projectTerminalEvent(buildAttachmentHeavyFinal(64)); + const twice = projectTerminalEvent(once); + + expect(twice).toEqual(once); + }); + + it('returns the identical reference for an already-safe event', () => { + const safe: FinalEvent = { + final: true, + requestMessage: { messageId: 'um-1', text: 'clean', files: [{ file_id: 'f-1' }] }, + responseMessage: { messageId: 'rm-1', content: [{ type: 'text', text: 'answer' }] }, + }; + + expect(projectTerminalEvent(safe)).toBe(safe); + }); + + it('returns non-terminal events unchanged', () => { + const chunk: ServerSentEvent = { event: 'on_message_delta', data: { delta: 'x' } }; + const created: ServerSentEvent = { + created: true, + message: { messageId: 'um-1', sender: 'User', isCreatedByUser: true }, + streamId: 's-1', + }; + + expect(projectTerminalEvent(chunk)).toBe(chunk); + expect(projectTerminalEvent(created)).toBe(created); + }); + }); + + describe('payload scaling', () => { + it('does not scale with excluded attachment bodies', () => { + const small = byteLength(projectTerminalEvent(buildAttachmentHeavyFinal(1_000))); + const large = byteLength(projectTerminalEvent(buildAttachmentHeavyFinal(200_000))); + + expect(byteLength(buildAttachmentHeavyFinal(200_000))).toBeGreaterThan(400_000); + expect(large).toBe(small); + }); + + it('does not truncate legitimate response content to satisfy a size bound', () => { + const answer = 'Legitimate answer sentence. '.repeat(5_000); + const event: FinalEvent = { + final: true, + requestMessage: { messageId: 'um-1', fileContext: 'D'.repeat(200_000) }, + responseMessage: { messageId: 'rm-1', content: [{ type: 'text', text: answer }] }, + }; + + const projected = projectTerminalEvent(event); + + expect((projected.responseMessage?.content as Array<{ text: string }>)[0]?.text).toBe(answer); + expect(byteLength(projected)).toBeGreaterThan(100_000); + }); + }); +}); diff --git a/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts b/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts new file mode 100644 index 0000000000..074de6bd62 --- /dev/null +++ b/packages/api/src/stream/__tests__/terminalProjectionBoundary.stream_integration.spec.ts @@ -0,0 +1,362 @@ +/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */ +/* eslint jest/expect-expect: ["error", { "assertFunctionNames": ["expect", "expectSafeTerminalEvent"] }] */ +import type { Redis, Cluster } from 'ioredis'; +import type { FinalEvent, ServerSentEvent } from '~/types'; +import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport'; +import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport'; +import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore'; +import { GenerationJobManagerClass } from '~/stream/GenerationJobManager'; +import { RedisJobStore } from '~/stream/implementations/RedisJobStore'; +import { createIoRedisSubscriber } from '~/cache/redisUtils'; +import { clearRedisTestPrefix } from './helpers/redis'; + +/** Suppress winston Console transport output (survives jest.resetModules) */ +jest.spyOn(console, 'log').mockImplementation(); + +const TRANSIENT_BODY = 'D'.repeat(64 * 1024); +const ANSWER = 'The report concludes that latency improved.'; + +/** + * A terminal event as a controller would hand it over WITHOUT caller-level + * sanitization. Passing the unsanitized shape is the negative control for + * "caller allowlists are enough": every assertion below then proves the + * manager boundary itself excludes the transient data. + */ +function buildUnsanitizedFinal(): FinalEvent { + return { + final: true, + terminalStatus: 'complete', + conversation: { conversationId: 'conv-1', title: 'Report' }, + title: 'Report', + requestMessage: { + messageId: 'um-1', + conversationId: 'conv-1', + text: 'Summarize the attachment', + isCreatedByUser: true, + fileContext: TRANSIENT_BODY, + image_urls: [ + { type: 'image_url', image_url: { url: `data:image/png;base64,${TRANSIENT_BODY}` } }, + ], + files: [ + { + file_id: 'f-1', + filename: 'report.pdf', + filepath: '/uploads/report.pdf', + text: TRANSIENT_BODY, + }, + ], + }, + responseMessage: { + messageId: 'rm-1', + parentMessageId: 'um-1', + conversationId: 'conv-1', + content: [{ type: 'text', text: ANSWER }], + attachments: [{ file_id: 'a-1', filename: 'chart.png', text: TRANSIENT_BODY }], + }, + }; +} + +/** `FinalMessageFields` carries an index signature, so nested collections read + * back as `unknown`; these keep the assertions readable without `any`. */ +const entries = (value: unknown): Array> => + (value ?? []) as Array>; +const filesOf = (message: unknown): Array> => + entries((message as { files?: unknown } | null | undefined)?.files); +const attachmentsOf = (message: unknown): Array> => + entries((message as { attachments?: unknown } | null | undefined)?.attachments); + +/** Every assertion a safe terminal representation must satisfy. */ +function expectSafeTerminalEvent(event: unknown): void { + const final = event as FinalEvent; + expect(final.requestMessage).not.toHaveProperty('fileContext'); + expect(final.requestMessage).not.toHaveProperty('image_urls'); + expect(filesOf(final.requestMessage)[0]).not.toHaveProperty('text'); + expect(attachmentsOf(final.responseMessage)[0]).not.toHaveProperty('text'); + + // authoritative final state and attachment references survive + expect(final.requestMessage?.text).toBe('Summarize the attachment'); + expect(filesOf(final.requestMessage)[0]).toMatchObject({ + file_id: 'f-1', + filename: 'report.pdf', + filepath: '/uploads/report.pdf', + }); + expect(attachmentsOf(final.responseMessage)[0]).toMatchObject({ + file_id: 'a-1', + filename: 'chart.png', + }); + expect((final.responseMessage?.content as Array<{ text: string }>)[0]?.text).toBe(ANSWER); + + // the excluded bodies are nowhere in the serialized representation + expect(JSON.stringify(final)).not.toContain(TRANSIENT_BODY); +} + +describe('terminal projection boundary', () => { + let originalEnv: NodeJS.ProcessEnv; + let ioredisClient: Redis | Cluster | null = null; + const testPrefix = 'Terminal-Projection-Test'; + const redisConfigured = process.env.USE_REDIS === 'true'; + const describeRedis = redisConfigured ? describe : describe.skip; + + beforeAll(async () => { + originalEnv = { ...process.env }; + process.env.USE_REDIS = process.env.USE_REDIS ?? 'true'; + process.env.REDIS_URI = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379'; + process.env.REDIS_KEY_PREFIX = testPrefix; + process.env.REDIS_PING_INTERVAL = '0'; + jest.resetModules(); + const redisModule = await import('~/cache/redisClients'); + ioredisClient = redisModule.ioredisClient; + }); + + afterAll(async () => { + process.env = originalEnv; + }); + + afterEach(async () => { + if (ioredisClient) { + await clearRedisTestPrefix(ioredisClient, testPrefix).catch(() => undefined); + } + }); + + describe('in-memory mode preserves the same external contract', () => { + let manager: GenerationJobManagerClass; + let jobStore: InMemoryJobStore; + + beforeEach(() => { + jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 }); + manager = new GenerationJobManagerClass(); + manager.configure({ jobStore, eventTransport: new InMemoryEventTransport(), isRedis: false }); + manager.initialize(); + }); + + afterEach(async () => { + await manager.destroy(); + }); + + it('excludes transient data from publishTerminalClaim storage, publication and cache', async () => { + const streamId = `inmem-claim-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, { + persistencePending: true, + }); + + const input = buildUnsanitizedFinal(); + const result = await manager.publishTerminalClaim(claim!, input); + + expectSafeTerminalEvent(result.finalEvent); + expect(result.persistenceFailed).toBe(false); + + const stored = await jobStore.getJob(streamId); + expect(stored?.finalEvent).toBeDefined(); + expectSafeTerminalEvent(JSON.parse(stored!.finalEvent!)); + + // the caller's own object is untouched + expect(input.requestMessage).toHaveProperty('fileContext'); + expect(input.requestMessage).toHaveProperty('image_urls'); + }); + + it('excludes transient data from emitDone storage and publication', async () => { + const streamId = `inmem-emitdone-${Date.now()}`; + await manager.createJob(streamId, 'user-1'); + + await manager.emitDone(streamId, buildUnsanitizedFinal()); + + const stored = await jobStore.getJob(streamId); + expectSafeTerminalEvent(JSON.parse(stored!.finalEvent!)); + }); + }); + + describeRedis('Redis mode', () => { + let manager: GenerationJobManagerClass; + let jobStore: RedisJobStore; + let eventTransport: RedisEventTransport; + let subscriber: Redis | Cluster; + + beforeEach(async () => { + jobStore = new RedisJobStore(ioredisClient!, { runningTtl: 60 }); + await jobStore.initialize(); + subscriber = createIoRedisSubscriber(ioredisClient!, 'terminal-projection-sub'); + eventTransport = new RedisEventTransport(ioredisClient!, subscriber); + manager = new GenerationJobManagerClass(); + manager.configure({ jobStore, eventTransport, isRedis: true }); + manager.initialize(); + }); + + afterEach(async () => { + await manager.destroy(); + await subscriber.quit().catch(() => undefined); + }); + + it('excludes transient data from the Redis job hash and the published frame', async () => { + const streamId = `redis-claim-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, { + persistencePending: true, + }); + + const published: ServerSentEvent[] = []; + const originalEmitDone = eventTransport.emitDone.bind(eventTransport); + jest + .spyOn(eventTransport, 'emitDone') + .mockImplementation(async (id: string, event: unknown, generationId?: number) => { + published.push(event as ServerSentEvent); + return originalEmitDone(id, event as ServerSentEvent, generationId); + }); + + const result = await manager.publishTerminalClaim(claim!, buildUnsanitizedFinal()); + + expect(result.persistenceFailed).toBe(false); + expectSafeTerminalEvent(result.finalEvent); + + // durable record actually written to Redis + const stored = await jobStore.getJob(streamId); + expectSafeTerminalEvent(JSON.parse(stored!.finalEvent!)); + + // the frame handed to the transport for Pub/Sub publication + expect(published).toHaveLength(1); + expectSafeTerminalEvent(published[0]); + }); + + it('does not scale the durable terminal record with excluded bodies', async () => { + const sizes: number[] = []; + for (const bodyChars of [1_000, 256 * 1024]) { + const streamId = `redis-scale-${bodyChars}-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob( + streamId, + 'complete', + undefined, + job.createdAt, + { persistencePending: true }, + ); + const body = 'D'.repeat(bodyChars); + await manager.publishTerminalClaim(claim!, { + final: true, + requestMessage: { + messageId: 'um-1', + text: 'q', + fileContext: body, + files: [{ file_id: 'f-1', filename: 'a.pdf', text: body }], + }, + responseMessage: { messageId: 'rm-1', content: [{ type: 'text', text: ANSWER }] }, + } as FinalEvent); + const stored = await jobStore.getJob(streamId); + sizes.push(Buffer.byteLength(stored!.finalEvent!, 'utf8')); + } + + expect(sizes[1]).toBe(sizes[0]); + }); + + /** + * Mixed deployment: the durable record was written by a replica that + * predates projection, and a different replica replays it. The reader has + * no local runtime, so it must project what it parses out of Redis instead + * of re-delivering the stored payload unchanged. + */ + it('projects a legacy unprojected record when another replica replays it', async () => { + const streamId = `redis-legacy-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, { + persistencePending: true, + }); + await manager.publishTerminalClaim(claim!, { final: true } as FinalEvent); + + /** Overwrite with the shape an older producer stored. */ + await jobStore.updateJob( + streamId, + { finalEvent: JSON.stringify(buildUnsanitizedFinal()) }, + job.createdAt, + ); + const rawStored = await jobStore.getJob(streamId); + expect(rawStored!.finalEvent).toContain(TRANSIENT_BODY); + + const replicaStore = new RedisJobStore(ioredisClient!, { runningTtl: 60 }); + await replicaStore.initialize(); + const replicaSubscriber = createIoRedisSubscriber(ioredisClient!, 'terminal-projection-b'); + const replicaTransport = new RedisEventTransport(ioredisClient!, replicaSubscriber); + const replica = new GenerationJobManagerClass(); + replica.configure({ + jobStore: replicaStore, + eventTransport: replicaTransport, + isRedis: true, + }); + replica.initialize(); + + try { + const delivered: ServerSentEvent[] = []; + const subscription = await replica.subscribe( + streamId, + () => undefined, + (event: ServerSentEvent) => { + delivered.push(event); + }, + ); + expect(subscription).not.toBeNull(); + await new Promise((resolve) => setTimeout(resolve, 250)); + + expect(delivered).toHaveLength(1); + expectSafeTerminalEvent(delivered[0]); + subscription!.unsubscribe(); + } finally { + await replica.destroy(); + await replicaSubscriber.quit().catch(() => undefined); + } + }); + + describe('persistence-success bookkeeping', () => { + it('reports success for a normally projected and persisted FINAL', async () => { + const streamId = `redis-ok-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob( + streamId, + 'complete', + undefined, + job.createdAt, + { persistencePending: true }, + ); + + const result = await manager.publishTerminalClaim(claim!, buildUnsanitizedFinal()); + + expect(result.persistenceFailed).toBe(false); + expect((result.finalEvent as FinalEvent).reconcile).toBeUndefined(); + }); + + it('reports failure when the caller publishes conservative reconciliation', async () => { + const streamId = `redis-null-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob( + streamId, + 'complete', + undefined, + job.createdAt, + { persistencePending: true }, + ); + + const result = await manager.publishTerminalClaim(claim!, null); + + expect(result.persistenceFailed).toBe(true); + expect((result.finalEvent as FinalEvent).reconcile).toBe(true); + }); + + it('reports failure when durable finalization fails', async () => { + const streamId = `redis-nodurable-${Date.now()}`; + const job = await manager.createJob(streamId, 'user-1'); + const claim = await manager.claimTerminalJob( + streamId, + 'complete', + undefined, + job.createdAt, + { persistencePending: true }, + ); + jest + .spyOn(jobStore, 'finalizeTerminalPersistence') + .mockRejectedValue(new Error('redis unavailable')); + + const result = await manager.publishTerminalClaim(claim!, buildUnsanitizedFinal()); + + expect(result.persistenceFailed).toBe(true); + expect((result.finalEvent as FinalEvent).reconcile).toBe(true); + }); + }); + }); +}); diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts index bded1f628a..a261bde395 100644 --- a/packages/api/src/stream/index.ts +++ b/packages/api/src/stream/index.ts @@ -64,6 +64,16 @@ export { createStreamServices } from './createStreamServices'; export type { StreamServicesConfig, StreamServices } from './createStreamServices'; export { filterPersistableAbortContent, hasPersistableAbortContent } from './abortContent'; export { getGenerationElapsedMs } from './elapsed'; +export { + projectTerminalEvent, + TRANSIENT_MESSAGE_FIELDS, + TRANSIENT_FILE_FIELDS, +} from './terminalProjection'; +export type { + ProjectedFinalEvent, + ProjectedMessageFields, + TransientMessageField, +} from './terminalProjection'; // Implementations (for advanced use cases) export { InMemoryJobStore } from './implementations/InMemoryJobStore'; diff --git a/packages/api/src/stream/terminalProjection.ts b/packages/api/src/stream/terminalProjection.ts new file mode 100644 index 0000000000..a8da73614c --- /dev/null +++ b/packages/api/src/stream/terminalProjection.ts @@ -0,0 +1,174 @@ +import type { FinalEvent, FinalMessageFields, ServerSentEvent } from '~/types/events'; + +/** + * Transient prompt-building fields that must never reach a terminal cache, + * durable terminal record, publication or replay. + * + * These carry model inputs assembled while building the prompt, not + * authoritative final state: + * - `fileContext`: text extracted from attachments for the prompt. + * - `image_urls`: base64 image inputs accumulated during prompt building. + * + * Adding a field here is the deliberate decision to exclude it everywhere a + * terminal event is stored or transmitted. `ProjectedMessageFields` keeps the + * TypeScript contract aligned so a projected message cannot be read for one of + * them without a type error. + */ +export const TRANSIENT_MESSAGE_FIELDS = ['fileContext', 'image_urls'] as const; + +/** + * Transient fields on a message's nested file/attachment entries. `text` is the + * embedded extracted body; `_id` and `__v` are storage bookkeeping. Every other + * property is display and download metadata the client needs to keep rendering + * the attachment after FINAL, so entries are projected field-by-field rather + * than dropped. + */ +export const TRANSIENT_FILE_FIELDS = ['text', '_id', '__v'] as const; + +/** Message-valued slots of a terminal event that can carry transient inputs. */ +const MESSAGE_SLOTS = ['requestMessage', 'responseMessage'] as const; + +/** Nested collections of file-like entries on a message. */ +const FILE_COLLECTIONS = ['files', 'attachments'] as const; + +export type TransientMessageField = (typeof TRANSIENT_MESSAGE_FIELDS)[number]; + +/** A message whose transient prompt-building fields have been excluded. */ +export type ProjectedMessageFields = Omit; + +/** A terminal event safe to cache, persist, publish and replay. */ +export type ProjectedFinalEvent = Omit< + FinalEvent, + 'requestMessage' | 'responseMessage' | 'runMessages' +> & { + requestMessage?: ProjectedMessageFields | null; + responseMessage?: ProjectedMessageFields | null; + runMessages?: ProjectedMessageFields[]; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Project one file/attachment entry. Returns the same reference when it carries + * no transient field, so a clean event allocates nothing. + */ +function projectFileEntry(entry: unknown): unknown { + if (!isRecord(entry)) { + return entry; + } + let projected: Record | undefined; + for (const field of TRANSIENT_FILE_FIELDS) { + if (field in entry) { + projected ??= { ...entry }; + delete projected[field]; + } + } + return projected ?? entry; +} + +function projectFileCollection(value: unknown): unknown { + if (!Array.isArray(value)) { + return value; + } + let changed = false; + const projected = value.map((entry) => { + const next = projectFileEntry(entry); + if (next !== entry) { + changed = true; + } + return next; + }); + return changed ? projected : value; +} + +/** + * Project a single message slot. Preserves `null` and `undefined` distinctly: + * a caller that deliberately sent `requestMessage: null` (early abort) keeps + * the null, and an absent slot stays absent. + */ +function projectMessage(message: T): T { + if (!isRecord(message)) { + return message; + } + let projected: Record | undefined; + + for (const field of TRANSIENT_MESSAGE_FIELDS) { + if (field in message) { + projected ??= { ...message }; + delete projected[field]; + } + } + + for (const collection of FILE_COLLECTIONS) { + const current = (projected ?? message)[collection]; + const next = projectFileCollection(current); + if (next !== current) { + projected ??= { ...message }; + projected[collection] = next; + } + } + + return (projected ?? message) as T; +} + +function projectRunMessages(value: unknown): { value: unknown; changed: boolean } { + if (!Array.isArray(value)) { + return { value, changed: false }; + } + let changed = false; + const projected = value.map((entry) => { + const next = projectMessage(entry); + if (next !== entry) { + changed = true; + } + return next; + }); + return changed ? { value: projected, changed: true } : { value, changed: false }; +} + +/** A terminal event is the only kind that carries message payloads. */ +function isFinalEvent(event: ServerSentEvent): event is FinalEvent { + return isRecord(event) && (event as { final?: unknown }).final === true; +} + +/** + * Exclude transient prompt-building data from a terminal event before it is + * cached, persisted, published or replayed. + * + * Selects fields rather than deep-cloning: only the containers along a path + * that actually holds transient data are rebuilt, the excluded values are never + * copied, and the input is never mutated (prompt processing, message + * persistence and lifecycle code may still own those objects). + * + * Idempotent and allocation-free for an already-safe event: projecting a clean + * event returns the identical reference. + * + * Non-terminal events (chunks, `created`) carry no message payload and are + * returned unchanged. + */ +export function projectTerminalEvent(event: T): T { + if (!isFinalEvent(event)) { + return event; + } + + let projected: Record | undefined; + + for (const slot of MESSAGE_SLOTS) { + const current = (event as Record)[slot]; + const next = projectMessage(current); + if (next !== current) { + projected ??= { ...(event as Record) }; + projected[slot] = next; + } + } + + const runMessages = projectRunMessages((event as Record).runMessages); + if (runMessages.changed) { + projected ??= { ...(event as Record) }; + projected.runMessages = runMessages.value; + } + + return (projected ?? event) as T; +} diff --git a/packages/api/src/utils/message.ts b/packages/api/src/utils/message.ts index 55d068577a..968aad5113 100644 --- a/packages/api/src/utils/message.ts +++ b/packages/api/src/utils/message.ts @@ -12,8 +12,13 @@ type GetMessagesByParentId = ( /** Fields to strip from files before client transmission */ const FILE_STRIP_FIELDS = ['text', '_id', '__v'] as const; -/** Fields to strip from messages before client transmission */ -const MESSAGE_STRIP_FIELDS = ['fileContext'] as const; +/** Fields to strip from messages before client transmission. + * Both are prompt-building inputs: `fileContext` is text extracted from + * attachments and `image_urls` holds base64 image inputs. Terminal events are + * additionally projected centrally by `projectTerminalEvent`; this stays as + * defense in depth for the non-terminal senders (`error.js`, + * `abortMiddleware.js`). */ +const MESSAGE_STRIP_FIELDS = ['fileContext', 'image_urls'] as const; /** * Strips large/unnecessary fields from a file object before transmitting to client. @@ -63,7 +68,8 @@ export function buildMessageFiles>( /** * Sanitizes a message object before transmitting to client. - * Removes large fields like `fileContext` and strips `text` from embedded files. + * Removes prompt-building fields (`fileContext`, `image_urls`) and strips + * `text` from embedded files. * * @param message - The message object to sanitize * @returns A new message object safe for client transmission