diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 6c1b6a6136..42195a0c3e 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -3600,6 +3600,9 @@ class AgentClient extends BaseClient { (activityLabel ? createAssistantPhaseStampingHandlers(offsetHandlers) : offsetHandlers); const createRunPromise = createRun({ agents, + // Conversation-stable identity for the e2e run hook; a resumed run + // carries no messages, so history cannot identify the conversation. + conversationId: this.conversationId, messages, modelCallbacks: [modelBoundCallback], // This controller implements the full HITL pause/resume lifecycle (handleRunInterrupt @@ -4070,6 +4073,7 @@ class AgentClient extends BaseClient { (activityLabel ? createAssistantPhaseStampingHandlers(offsetHandlers) : offsetHandlers); run = await createRun({ agents, + conversationId: this.conversationId, modelCallbacks: [modelBoundCallback], // State (messages, tool calls) is rehydrated from the checkpoint by // run.resume; createRun only needs the agents to rebuild the graph. diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index 07dda8b569..92efa48463 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -92,6 +92,10 @@ endpoints: - module: e2e/setup/tool-approval-hook.js matcher: ^approval_probe_mcp_e2e-memory$ custom: + # Substituted with a REAL provider endpoint only when the mock config runs + # in model-fixture record mode (E2E_MODEL_FIXTURES=record); a comment + # otherwise, so the credential-free profile never gains a live endpoint. + # __E2E_MODEL_RECORD_PROVIDER__ - name: 'Mock Provider A' apiKey: 'e2e-mock-key-a' baseURL: 'http://127.0.0.1:8889/v1' @@ -184,6 +188,7 @@ modelSpecs: # Surfaces the endpoints menu (modelSelect defaults on when addedEndpoints is # set) limited to entries that don't collide with the spec labels above. addedEndpoints: + # __E2E_MODEL_RECORD_ADDED_ENDPOINT__ - 'Mock Provider C' - 'Mock Provider D' - 'Mock Provider E' diff --git a/e2e/fixtures/model-replay/deepseek-tool-call.jsonl b/e2e/fixtures/model-replay/deepseek-tool-call.jsonl new file mode 100644 index 0000000000..0ede2d819c --- /dev/null +++ b/e2e/fixtures/model-replay/deepseek-tool-call.jsonl @@ -0,0 +1,40 @@ +{"type":"meta","name":"deepseek-tool-call","recordedAt":"2026-08-25T22:23:39.533Z"} +{"type":"invocation","index":0,"userText":"Call the remember_fact tool with fact set to \"the replay lane records tool calls\", then reply with exactly the text the tool returned and nothing else."} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"id":"call_00_MvSzx7ML4G6RE6Twg4cr3997","type":"function","function":{"name":"remember_fact_mcp_e2e-memory","arguments":""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"name":"remember_fact_mcp_e2e-memory","args":"","id":"call_00_MvSzx7ML4G6RE6Twg4cr3997","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"{"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"{","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"fact"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"fact","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":": "}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":": ","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"the"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"the","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" replay"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" replay","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" lane"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" lane","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" records"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" records","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" tool"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" tool","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" calls"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" calls","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"}","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"finish_reason":"tool_calls","system_fingerprint":"a26a7955944dc5c60445bff77fac9c8e","model_name":"deepseek-v4-flash","model_provider":"openai","usage":{"prompt_tokens":517,"completion_tokens":56,"total_tokens":573,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":5}},"tool_call_chunks":[],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":517,"completion_tokens":56,"total_tokens":573,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":5}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":517,"output_tokens":56,"total_tokens":573,"input_token_details":{"cache_read":512}},"id":"run-01a03b05-a093-744d-a6c6-dbf53d88db4a"}} +{"type":"end","invocation":0,"text":""} +{"type":"invocation","index":1,"userText":"Call the remember_fact tool with fact set to \"the replay lane records tool calls\", then reply with exactly the text the tool returned and nothing else."} +{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":"E","message":{"content":"E","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":"2","message":{"content":"2","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":"E","message":{"content":"E","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" M","message":{"content":" M","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":"CP","message":{"content":"CP","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" memory","message":{"content":" memory","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" noted","message":{"content":" noted","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":":","message":{"content":":","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" the","message":{"content":" the","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" replay","message":{"content":" replay","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" lane","message":{"content":" lane","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" records","message":{"content":" records","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" tool","message":{"content":" tool","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":" calls","message":{"content":" calls","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"finish_reason":"stop","system_fingerprint":"a26a7955944dc5c60445bff77fac9c8e","model_name":"deepseek-v4-flash","model_provider":"openai","usage":{"prompt_tokens":599,"completion_tokens":14,"total_tokens":613,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":87}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}} +{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":599,"completion_tokens":14,"total_tokens":613,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":87}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":599,"output_tokens":14,"total_tokens":613,"input_token_details":{"cache_read":512}},"id":"run-01a03b05-a445-728f-821b-b6b325e263e9"}} +{"type":"end","invocation":1,"text":"E2E MCP memory noted: the replay lane records tool calls"} diff --git a/e2e/fixtures/model-replay/deepseek-two-turn.jsonl b/e2e/fixtures/model-replay/deepseek-two-turn.jsonl new file mode 100644 index 0000000000..056d14e4ac --- /dev/null +++ b/e2e/fixtures/model-replay/deepseek-two-turn.jsonl @@ -0,0 +1,52 @@ +{"type":"meta","name":"deepseek-two-turn","recordedAt":"2026-08-25T17:44:11.310Z"} +{"type":"invocation","index":0,"userText":"Name the two prime numbers between 20 and 30, comma separated, and nothing else."} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}} +{"type":"chunk","invocation":0,"text":"23","message":{"content":"23","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}} +{"type":"chunk","invocation":0,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}} +{"type":"chunk","invocation":0,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}} +{"type":"chunk","invocation":0,"text":"29","message":{"content":"29","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{"prompt_tokens":23,"completion_tokens":4,"total_tokens":27,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":23}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}} +{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":23,"completion_tokens":4,"total_tokens":27,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":23}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":23,"output_tokens":4,"total_tokens":27,"input_token_details":{"cache_read":0}}}} +{"type":"end","invocation":0,"text":"23, 29"} +{"type":"invocation","index":1,"userText":"In two short sentences, explain why the sum of those two primes is an even number. Begin with the word \"Because\"."} +{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"Because","message":{"content":"Because","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" any","message":{"content":" any","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" prime","message":{"content":" prime","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" number","message":{"content":" number","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" greater","message":{"content":" greater","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" than","message":{"content":" than","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"2","message":{"content":"2","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" odd","message":{"content":" odd","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" and","message":{"content":" and","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" the","message":{"content":" the","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" sum","message":{"content":" sum","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" of","message":{"content":" of","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" two","message":{"content":" two","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" odd","message":{"content":" odd","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" numbers","message":{"content":" numbers","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" always","message":{"content":" always","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" even","message":{"content":" even","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":".","message":{"content":".","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" Therefore","message":{"content":" Therefore","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"23","message":{"content":"23","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" plus","message":{"content":" plus","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"29","message":{"content":"29","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" equals","message":{"content":" equals","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"52","message":{"content":"52","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" which","message":{"content":" which","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":" even","message":{"content":" even","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":".","message":{"content":".","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{"prompt_tokens":56,"completion_tokens":37,"total_tokens":93,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":56}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}} +{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":56,"completion_tokens":37,"total_tokens":93,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":56}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":56,"output_tokens":37,"total_tokens":93,"input_token_details":{"cache_read":0}}}} +{"type":"end","invocation":1,"text":"Because any prime number greater than 2 is odd, and the sum of two odd numbers is always even. Therefore, 23 plus 29 equals 52, which is even."} diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index 694061c1b2..0787f72fc0 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -26,6 +26,67 @@ const labelServerPath = path.resolve(rootPath, 'e2e/setup/fake-label-server.js') * `writeRuntimeMockConfig` substitutes any override into the generated copy. */ const LABEL_PORT = process.env.E2E_LABEL_PORT || '8889'; const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js'); +/** Model-fixture record mode: the run hook taps the REAL provider stream into a + * replayable fixture instead of overriding the model (e2e/setup/model-replay.js). */ +const modelFixtureRecording = process.env.E2E_MODEL_FIXTURES === 'record'; +const recordModelHookPath = path.resolve(rootPath, 'e2e/setup/record-model.js'); +const recordProviderBaseURL = + process.env.E2E_RECORD_PROVIDER_BASE_URL || 'https://api.deepseek.com/v1'; +const recordProviderModel = process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat'; +if (modelFixtureRecording && !process.env.E2E_RECORD_PROVIDER_API_KEY) { + throw new Error('E2E_MODEL_FIXTURES=record requires E2E_RECORD_PROVIDER_API_KEY'); +} +/** + * Each fixture belongs to exactly one spec, and a spec records only its own. + * Accepting an arbitrary name would leave a second fixture beside the + * committed one carrying the same prompts, and the server-side ambiguity + * check would then refuse to bind either — a successful recording run would + * disable the keyless lane. + */ +const RECORDABLE_FIXTURES = ['deepseek-two-turn', 'deepseek-tool-call']; +if (modelFixtureRecording && !process.env.E2E_MODEL_FIXTURE_NAME) { + throw new Error('E2E_MODEL_FIXTURES=record requires E2E_MODEL_FIXTURE_NAME'); +} +if ( + modelFixtureRecording && + !RECORDABLE_FIXTURES.includes(process.env.E2E_MODEL_FIXTURE_NAME ?? '') +) { + throw new Error( + `E2E_MODEL_FIXTURE_NAME must be one of ${RECORDABLE_FIXTURES.join(', ')}; ` + + `received ${process.env.E2E_MODEL_FIXTURE_NAME}`, + ); +} +/** + * Playwright documents `-c` as an alias for `--config`, so both spellings are + * parsed — recognising only the long form would let the short one slip past. + * + * Derived configs (`playwright.config.redis.ts`, `.mermaid.ts`) spread this + * config and then replace `testMatch`, discarding the record-mode restriction + * below — their specs would reach the paid provider and rewrite the selected + * fixture. The restriction cannot be enforced through a value a consumer can + * overwrite, so record mode refuses any config but this one. + */ +if (modelFixtureRecording) { + /** Only the process that parsed the CLI carries `--config`; Playwright + * workers do not, and must not be judged on an argument they never saw. */ + const configFlagIndex = process.argv.findIndex( + (arg) => + arg === '--config' || arg === '-c' || arg.startsWith('--config=') || arg.startsWith('-c='), + ); + const configFlag = configFlagIndex === -1 ? undefined : process.argv[configFlagIndex]; + let configPath: string | undefined; + if (configFlag?.includes('=')) { + configPath = configFlag.slice(configFlag.indexOf('=') + 1); + } else if (configFlag) { + configPath = process.argv[configFlagIndex + 1]; + } + if (configPath && !/playwright\.config\.mock\.ts$/.test(configPath)) { + throw new Error( + `E2E_MODEL_FIXTURES=record only runs under playwright.config.mock.ts, not ${configPath}; ` + + 'derived configs replace testMatch and would send their specs to the real provider', + ); + } +} const assistantsServerPath = path.resolve(rootPath, 'e2e/setup/fake-assistants-server.js'); const ASSISTANTS_PORT = process.env.E2E_ASSISTANTS_PORT || '8890'; const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); @@ -61,8 +122,15 @@ const baseEnv = { ...getLocalE2EEnv(), CONFIG_PATH: configPath, DEPLOYMENT_SKILLS_DIR: deploymentSkillsPath, - /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */ - LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath, + /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model — + * or, in model-fixture record mode, to tap the real provider stream. */ + LIBRECHAT_TEST_RUN_HOOK: modelFixtureRecording ? recordModelHookPath : fakeModelHookPath, + ...(modelFixtureRecording + ? { + E2E_MODEL_FIXTURE_NAME: process.env.E2E_MODEL_FIXTURE_NAME ?? '', + E2E_RECORD_PROVIDER_API_KEY: process.env.E2E_RECORD_PROVIDER_API_KEY ?? '', + } + : {}), ...(enableDynamicMcp ? { E2E_MCP_LIST_CHANGED: 'true', E2E_MCP_STATE_PATH: MCP_STATE_PATH } : {}), /** The Assistants runtime uses the OpenAI SDK directly, outside the agents run hook. */ ASSISTANTS_API_KEY: 'e2e-mock-assistants-key', @@ -117,7 +185,27 @@ function writeRuntimeMockConfig() { ].join('\n'), } : { allowedDomain: '', stdioEnv: '', networkServers: '' }; + const recordProviderBlock = modelFixtureRecording + ? [ + `- name: 'Replay Record Provider'`, + ` apiKey: '\${E2E_RECORD_PROVIDER_API_KEY}'`, + ` baseURL: '${recordProviderBaseURL}'`, + ' models:', + ' default:', + ` - '${recordProviderModel}'`, + ' fetch: false', + ' titleConvo: false', + ` modelDisplayLabel: 'Replay Record Provider'`, + ].join('\n ') + : '# __E2E_MODEL_RECORD_PROVIDER__'; config = config + .replace('# __E2E_MODEL_RECORD_PROVIDER__', recordProviderBlock) + .replace( + '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__', + modelFixtureRecording + ? `- 'Replay Record Provider'` + : '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__', + ) .replace('# __E2E_DYNAMIC_MCP_ALLOWED_DOMAIN__', dynamicMcpConfig.allowedDomain) .replace('# __E2E_DYNAMIC_MCP_STDIO_ENV__', dynamicMcpConfig.stdioEnv) .replace('# __E2E_DYNAMIC_MCP_NETWORK_SERVERS__', dynamicMcpConfig.networkServers); @@ -177,6 +265,12 @@ export default defineConfig({ globalSetup: require.resolve('./setup/global-setup'), globalTeardown: require.resolve('./setup/global-teardown.mock'), testDir: 'specs/mock/', + /** Record mode swaps the fake model for a real provider, so it must never + * run the whole mock suite: every spec's prompts would reach the paid + * endpoint, and each fresh conversation would truncate and rewrite the one + * selected fixture, leaving whichever scenario ran last. Without this an + * unfiltered entry point (`npm run e2e:mock`) does exactly that. */ + ...(modelFixtureRecording ? { testMatch: /model-replay[a-z-]*\.spec\.ts$/ } : {}), outputDir: 'specs/.test-results', fullyParallel: true, forbidOnly: !!process.env.CI, diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index a6e69ac67d..26e31c63ec 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -12,6 +12,7 @@ const { FakeChatModel } = require('@librechat/agents'); const { ChatGenerationChunk } = require('@langchain/core/outputs'); const { AIMessageChunk } = require('@langchain/core/messages'); +const { tryBindReplay } = require('./model-replay'); const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong'; const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10; @@ -2393,6 +2394,21 @@ module.exports = function fakeModelHook(run, context) { } const text = getLatestUserText(context?.messages); + /** Recorded-session replay outranks marker routing: a conversation whose + * prompt matches a fixture's next recorded invocation streams that recording + * through the real pipeline instead of a scripted mock response. */ + if ( + tryBindReplay({ + graph, + text, + agents: context?.agents, + messages: context?.messages, + conversationId: context?.conversationId, + modelCallbacks: context?.modelCallbacks, + }) + ) { + return; + } const toolNames = collectToolNames(context?.agents); const handoffScript = parseHandoffScript(text); const { diff --git a/e2e/setup/model-replay.js b/e2e/setup/model-replay.js new file mode 100644 index 0000000000..50ca0eab4e --- /dev/null +++ b/e2e/setup/model-replay.js @@ -0,0 +1,732 @@ +/** + * Record-once/replay-forever model fixtures for the mock e2e harness. + * + * Record mode (`E2E_MODEL_FIXTURES=record` + `E2E_MODEL_FIXTURE_NAME=`): + * `record-model.js` replaces the fake-model run hook; instead of overriding the + * graph's model it appends a LangChain callback handler to every agent + * context's `clientOptions.callbacks`, so the REAL provider model carries the + * recorder. Each model invocation's streamed `ChatGenerationChunk`s are + * serialized to `e2e/fixtures/model-replay/.jsonl` exactly as the + * provider emitted them (text deltas, tool_call_chunks, reasoning + * additional_kwargs, usage_metadata). Only the invocation's latest human text + * is recorded for binding — system prompts and tool schemas never enter the + * fixture. + * + * Replay mode (default, keyless): `fake-model.js` consults `tryBindReplay` + * before its marker routing. A conversation binds to a fixture when its latest + * user text equals the fixture's next unconsumed invocation's recorded user + * text. The replaying model is not hand-assigned: `ReplayChatModel` is + * registered as the SDK provider `librechat-e2e-replay` via the agents + * package's `registerProvider`, and the bound instance is constructed through + * the SDK's own `initializeModel` — registry lookup, constructor + * `clientOptions` (carrying the model-bound callbacks the way + * `withModelCallbacks` does), and real `bindTools` over the run's tools — so + * the recorded chunks stream through the same SDK machinery a live provider + * uses: createRun → registered provider model → graph → SSE → persistence. + * Every invocation re-checks its prompt against the recording, an invocation + * past the end of the script throws (over-consumption fails loud in the + * turn), and a per-fixture consumption ledger under + * `e2e/specs/.test-results/model-replay/` lets specs assert at teardown that + * every recorded invocation and chunk was drained (under-consumption fails the + * spec, not silently). + * + * Constraint carried over from the recording model: one live binding per + * fixture per server process — scenarios replaying the same fixture must not + * run concurrently. + */ +const fs = require('fs'); +const path = require('path'); +const { FakeChatModel, registerProvider, initializeModel } = require('@librechat/agents'); +const { ChatGenerationChunk } = require('@langchain/core/outputs'); +const { AIMessageChunk } = require('@langchain/core/messages'); + +const FIXTURES_DIR = path.resolve(__dirname, '../fixtures/model-replay'); +const LEDGER_DIR = path.resolve(__dirname, '../specs/.test-results/model-replay'); +const RECORDER_HANDLER_NAME = 'librechat-e2e-model-recorder'; +const SUMMARIZATION_GUARD_NAME = 'librechat-e2e-summarization-guard'; +const REPLAY_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10; + +function extractText(content) { + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + return ''; + } + const parts = []; + for (const part of content) { + if (typeof part === 'string') { + parts.push(part); + } else if (part && typeof part.text === 'string') { + parts.push(part.text); + } + } + return parts.join(''); +} + +function messageType(message) { + if (typeof message?.getType === 'function') { + return message.getType(); + } + if (typeof message?._getType === 'function') { + return message._getType(); + } + return message?.role; +} + +/** Every human message's text, oldest first. */ +function humanTexts(messages) { + if (!Array.isArray(messages)) { + return []; + } + const texts = []; + for (const message of messages) { + const type = messageType(message); + if (type === 'human' || type === 'user') { + texts.push(extractText(message.content)); + } + } + return texts; +} + +/** The latest human message's text — the binding and prompt-check key. */ +function latestHumanText(messages) { + const texts = humanTexts(messages); + return texts.length > 0 ? texts[texts.length - 1] : ''; +} + +/** + * The distinct user turns a fixture records, in order. A turn that calls a + * tool spans several model invocations under one prompt, so the invocation + * sequence is not the turn sequence and only this collapsed view can be + * compared against a conversation's human messages. + */ +function fixtureTurnTexts(invocations) { + const turns = []; + for (const invocation of invocations) { + if (turns[turns.length - 1] !== invocation.userText) { + turns.push(invocation.userText); + } + } + return turns; +} + +/** + * Whether this conversation is the one that already drove the fixture: its + * human turns open with exactly the fixture's recorded turns, in order. A + * consumed binding has to be retained for such a conversation, or an extra + * user turn would find no next invocation, fall through to ordinary + * fake-model routing, and be answered with a mock reply — leaving the + * over-consumption guard unreached and the drained ledger still passing. + */ +function conversationDroveFixture(messages, fixture) { + const texts = humanTexts(messages); + const turns = fixture.turns; + if (texts.length <= turns.length) { + return false; + } + return turns.every((turn, index) => turn === texts[index]); +} + +function jsonClone(value) { + if (value == null) { + return undefined; + } + try { + return JSON.parse(JSON.stringify(value)); + } catch { + return undefined; + } +} + +/** Minimal AIMessageChunk projection that reconstructs the streamed message. */ +function serializeChunk(chunk, token) { + const message = chunk?.message; + const serialized = { text: chunk?.text ?? token ?? '' }; + if (message) { + serialized.message = { + content: jsonClone(message.content) ?? '', + additional_kwargs: jsonClone(message.additional_kwargs), + response_metadata: jsonClone(message.response_metadata), + tool_call_chunks: jsonClone(message.tool_call_chunks), + usage_metadata: jsonClone(message.usage_metadata), + id: typeof message.id === 'string' ? message.id : undefined, + }; + } + return serialized; +} + +function deserializeChunk(serialized) { + const recorded = serialized.message; + const message = new AIMessageChunk({ + content: recorded?.content ?? serialized.text ?? '', + additional_kwargs: recorded?.additional_kwargs ?? {}, + response_metadata: recorded?.response_metadata ?? {}, + tool_call_chunks: recorded?.tool_call_chunks ?? [], + usage_metadata: recorded?.usage_metadata, + id: recorded?.id, + }); + return new ChatGenerationChunk({ text: serialized.text ?? '', message }); +} + +/* ------------------------------- recording ------------------------------- */ + +const recordingState = { + initialized: false, + fixturePath: undefined, + invocationCounter: 0, + conversationId: undefined, + /** Bumped on every (re)start so handlers left on a superseded graph can be + * told apart from the current attempt's. */ + generation: 0, + runIdToInvocation: new Map(), +}; + +function appendFixtureLine(entry) { + fs.appendFileSync(recordingState.fixturePath, `${JSON.stringify(entry)}\n`); +} + +function initializeRecording(fixtureName) { + fs.mkdirSync(FIXTURES_DIR, { recursive: true }); + recordingState.fixturePath = path.join(FIXTURES_DIR, `${fixtureName}.jsonl`); + fs.writeFileSync(recordingState.fixturePath, ''); + appendFixtureLine({ + type: 'meta', + name: fixtureName, + recordedAt: new Date().toISOString(), + }); + recordingState.initialized = true; + recordingState.invocationCounter = 0; + recordingState.conversationId = undefined; + recordingState.generation += 1; + recordingState.runIdToInvocation.clear(); + console.log(`[e2e model-replay] recording fixture ${recordingState.fixturePath}`); +} + +/** + * The recorder's state is process-global and the web server outlives a + * Playwright retry, so a failed attempt that already recorded invocations + * would otherwise leave the counter advanced: the retry appends 2/3 after + * 0/1 (or keeps a previous attempt's `error` line) and the fixture is + * unusable for replay. + * + * A new attempt is a new conversation. Identity comes from `conversationId` + * rather than from the prompt or the history: a turn that calls a tool + * invokes the model again under the same latest human message, and a resumed + * run after a tool-approval pause rebuilds `createRun` with no messages at + * all because state is rehydrated from the checkpoint. Both would look like + * fresh attempts to any text- or history-based rule, and truncate the fixture + * mid-turn. + */ +function isConversationStart(messages) { + return humanTexts(messages).length <= 1; +} + +function startsNewRecording(conversationId, messages) { + if (recordingState.invocationCounter === 0) { + return false; + } + if (conversationId != null && recordingState.conversationId != null) { + return recordingState.conversationId !== conversationId; + } + return isConversationStart(messages); +} + +/** + * Handlers are stamped with the recording generation they were installed for. + * A failed attempt can still have a provider call in flight when the retry + * resets the recording, and its graph keeps this handler: without the stamp + * that stale call would allocate an invocation index from the new attempt's + * counter, or append an `error` entry whose mapping was cleared, corrupting + * the freshly reset fixture. + */ +function createRecorderHandler() { + const generation = recordingState.generation; + const superseded = () => generation !== recordingState.generation; + return { + name: RECORDER_HANDLER_NAME, + generation, + /** Callbacks must settle before the model call resolves, or the `end` + * line races the durable-completion barrier the recording spec waits on + * (the same contract ModelBoundChatModelCallback declares). */ + awaitHandlers: true, + raiseError: true, + handleChatModelStart(_llm, messageBatches, runId) { + if (superseded()) { + return; + } + const index = recordingState.invocationCounter++; + recordingState.runIdToInvocation.set(runId, index); + appendFixtureLine({ + type: 'invocation', + index, + userText: latestHumanText(messageBatches?.[0]), + }); + }, + handleLLMNewToken(token, _idx, runId, _parentRunId, _tags, fields) { + const invocation = recordingState.runIdToInvocation.get(runId); + if (superseded() || invocation == null) { + return; + } + appendFixtureLine({ + type: 'chunk', + invocation, + ...serializeChunk(fields?.chunk, token), + }); + }, + handleLLMEnd(output, runId) { + const invocation = recordingState.runIdToInvocation.get(runId); + if (superseded() || invocation == null) { + return; + } + recordingState.runIdToInvocation.delete(runId); + const generation = output?.generations?.[0]?.[0]; + appendFixtureLine({ + type: 'end', + invocation, + text: generation?.text ?? extractText(generation?.message?.content), + }); + }, + handleLLMError(error, runId) { + const invocation = recordingState.runIdToInvocation.get(runId); + recordingState.runIdToInvocation.delete(runId); + if (superseded()) { + return; + } + appendFixtureLine({ + type: 'error', + invocation: invocation ?? null, + message: error instanceof Error ? error.message : String(error), + }); + }, + }; +} + +/** + * Attach the recorder to every agent context's model client options. The model + * is created per-invocation from `agentContext.clientOptions`, so appending a + * callback here puts the recorder on the real provider stream without + * replacing the model. + */ +function installRecorder({ graph, messages, conversationId }) { + const fixtureName = process.env.E2E_MODEL_FIXTURE_NAME; + if (!fixtureName) { + console.warn('[e2e model-replay] E2E_MODEL_FIXTURE_NAME unset; not recording'); + return; + } + if (!recordingState.initialized || startsNewRecording(conversationId, messages)) { + initializeRecording(fixtureName); + } + if (conversationId != null) { + recordingState.conversationId = conversationId; + } + const contexts = graph?.agentContexts; + if (!contexts || typeof contexts.values !== 'function') { + console.warn('[e2e model-replay] graph.agentContexts unavailable; not recording'); + return; + } + for (const context of contexts.values()) { + if (!context.clientOptions) { + context.clientOptions = {}; + } + attachRecorder(context.clientOptions); + /** Summarization runs on its own model with its own callback list. + * Recording those invocations without replaying them is worse than + * ignoring them: they would take slots in the fixture sequence that + * replay never consumes, so the next primary call would read the + * summariser's chunks. Replay routes only the agent model + * (`graph.overrideModel`) and subagents, so the honest boundary is to + * refuse a recording the lane could not reproduce. */ + const summarizationParameters = + context.summarizationConfig?.parameters ?? context.summarizationConfig?.config?.parameters; + if (summarizationParameters) { + attachSummarizationGuard(summarizationParameters); + } + } +} + +/** + * Fails a recording the moment the summarization model runs. Its invocations + * would otherwise enter the fixture sequence unreplayable — see + * `installRecorder`. Summarization fixtures need replay routing for that model + * before they can be supported. + */ +function attachSummarizationGuard(options) { + const handler = { + name: SUMMARIZATION_GUARD_NAME, + raiseError: true, + awaitHandlers: true, + handleChatModelStart() { + throw new Error( + '[e2e model-replay] summarization ran during recording, and replay cannot route the ' + + 'summarization model — its invocations would desynchronise the fixture. Record a ' + + 'scenario that stays under the context-pruning threshold.', + ); + }, + }; + const existing = options.callbacks; + if (Array.isArray(existing)) { + if (!existing.some((entry) => entry?.name === SUMMARIZATION_GUARD_NAME)) { + options.callbacks = [...existing, handler]; + } + return; + } + if (existing == null) { + options.callbacks = [handler]; + return; + } + if ( + typeof existing.addHandler === 'function' && + !existing.handlers?.some((entry) => entry?.name === SUMMARIZATION_GUARD_NAME) + ) { + existing.addHandler(handler); + } +} + +/** Append the recorder to a client-options object's callbacks, once. */ +function attachRecorder(options) { + /** Dedupe against the CURRENT generation only: a graph carried across a + * recording restart still holds a superseded handler, which is inert, so + * matching on name alone would leave that options object recording + * nothing. */ + const isCurrent = (handler) => + handler?.name === RECORDER_HANDLER_NAME && handler.generation === recordingState.generation; + const existing = options.callbacks; + if (Array.isArray(existing)) { + if (!existing.some(isCurrent)) { + options.callbacks = [ + ...existing.filter((handler) => handler?.name !== RECORDER_HANDLER_NAME), + createRecorderHandler(), + ]; + } + return; + } + if (existing == null) { + options.callbacks = [createRecorderHandler()]; + return; + } + if (typeof existing.addHandler === 'function') { + if (!existing.handlers?.some(isCurrent)) { + existing.addHandler(createRecorderHandler()); + } + } +} + +/* -------------------------------- replay --------------------------------- */ + +/** name -> { meta, invocations: [{ userText, chunks: [], finalText }] } */ +let fixtureRegistry; +/** name -> { cursor, chunksConsumed, ledger } */ +const replayState = new Map(); + +function parseFixtureFile(filePath) { + const name = path.basename(filePath, '.jsonl'); + const invocations = []; + let meta = { name }; + const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean); + for (const line of lines) { + const entry = JSON.parse(line); + if (entry.type === 'meta') { + meta = entry; + } else if (entry.type === 'invocation') { + invocations[entry.index] = { userText: entry.userText, chunks: [], finalText: '' }; + } else if (entry.type === 'chunk') { + invocations[entry.invocation]?.chunks.push(entry); + } else if (entry.type === 'end') { + const invocation = invocations[entry.invocation]; + if (invocation) { + invocation.finalText = entry.text ?? ''; + } + } else if (entry.type === 'error') { + throw new Error( + `[e2e model-replay] fixture ${name} recorded a provider error (${entry.message}); ` + + 're-record it before replaying', + ); + } + } + const missing = invocations.findIndex((invocation) => invocation == null); + if (missing !== -1) { + throw new Error(`[e2e model-replay] fixture ${name} is missing invocation ${missing}`); + } + /** The file name is the fixture's identity — it is what `E2E_MODEL_FIXTURE_NAME` + * selects, what the spec names, and what the ledger is written under. A + * recorded `meta.name` is descriptive only: trusting it would let a renamed + * or copied fixture collapse onto another's registry key and ledger. */ + return { meta: { ...meta, name }, invocations, turns: fixtureTurnTexts(invocations) }; +} + +function loadFixtureRegistry() { + if (fixtureRegistry) { + return fixtureRegistry; + } + fixtureRegistry = new Map(); + if (!fs.existsSync(FIXTURES_DIR)) { + return fixtureRegistry; + } + for (const file of fs.readdirSync(FIXTURES_DIR)) { + if (file.endsWith('.jsonl')) { + const fixture = parseFixtureFile(path.join(FIXTURES_DIR, file)); + fixtureRegistry.set(fixture.meta.name, fixture); + } + } + return fixtureRegistry; +} + +function writeLedger(name) { + const state = replayState.get(name); + if (!state) { + return; + } + fs.mkdirSync(LEDGER_DIR, { recursive: true }); + fs.writeFileSync( + path.join(LEDGER_DIR, `${name}.json`), + `${JSON.stringify({ fixture: name, ...state.ledger }, null, 2)}\n`, + ); +} + +function freshReplayState(fixture) { + return { + cursor: 0, + ledger: { + invocationsTotal: fixture.invocations.length, + chunksTotal: fixture.invocations.reduce( + (total, invocation) => total + invocation.chunks.length, + 0, + ), + invocationsConsumed: 0, + chunksConsumed: 0, + overruns: [], + promptMismatches: [], + }, + }; +} + +function getReplayState(fixture) { + let state = replayState.get(fixture.meta.name); + if (!state) { + state = freshReplayState(fixture); + replayState.set(fixture.meta.name, state); + } + return state; +} + +/** + * Start the fixture over for a new conversation. The web server outlives a + * Playwright retry, so without this a consumed cursor would leave the retry + * unable to bind its first prompt — it would fall through to marker routing + * and fail deterministically, burning every configured retry. The ledger + * resets with the cursor so the new attempt is judged on its own consumption + * rather than accumulating the previous one's counts. + */ +function restartReplayState(fixture) { + const state = freshReplayState(fixture); + replayState.set(fixture.meta.name, state); + return state; +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const REPLAY_PROVIDER = 'librechat-e2e-replay'; + +/** + * Constructed by the SDK's `initializeModel` through the provider registry, so + * `clientOptions` is the full constructor contract: the fixture binding, the + * shared cursor state, and the run's model-bound callbacks. + */ +class ReplayChatModel extends FakeChatModel { + constructor(clientOptions = {}) { + super({ responses: [''], sleep: 0, emitCustomEvent: false }); + this.fixture = clientOptions.fixture; + this.state = clientOptions.state; + this.boundToolNames = clientOptions.boundToolNames ?? []; + if (clientOptions.callbacks) { + this.callbacks = clientOptions.callbacks; + } + } + + /** Real SDK tool binding: returns a bound copy sharing the replay cursor. */ + bindTools(tools) { + return new ReplayChatModel({ + fixture: this.fixture, + state: this.state, + callbacks: this.callbacks, + boundToolNames: (tools ?? []).map((tool) => tool?.name ?? tool?.function?.name ?? 'unknown'), + }); + } + + async *_streamResponseChunks(messages, _options, runManager) { + const { fixture, state } = this; + const invocation = fixture.invocations[state.cursor]; + if (!invocation) { + state.ledger.overruns.push({ + at: new Date().toISOString(), + userText: latestHumanText(messages), + }); + writeLedger(fixture.meta.name); + throw new Error( + `[e2e model-replay] fixture ${fixture.meta.name} over-consumed: model invoked ` + + `after all ${fixture.invocations.length} recorded invocations were drained`, + ); + } + /** A resumed run carries no human message — state is rehydrated from the + * checkpoint — so there is no prompt to check against. Ownership already + * established which conversation this is; enforcing the recorded prompt + * here would reject every resume. Every real turn still gets checked. */ + const promptText = latestHumanText(messages); + const carriesHumanTurn = humanTexts(messages).length > 0; + if (carriesHumanTurn && promptText !== invocation.userText) { + state.ledger.promptMismatches.push({ + invocation: state.cursor, + expected: invocation.userText, + received: promptText, + }); + writeLedger(fixture.meta.name); + throw new Error( + `[e2e model-replay] fixture ${fixture.meta.name} invocation ${state.cursor} ` + + `prompt mismatch: recorded ${JSON.stringify(invocation.userText)}, ` + + `received ${JSON.stringify(promptText)}`, + ); + } + state.cursor += 1; + for (const chunk of invocation.chunks) { + await sleep(REPLAY_CHUNK_DELAY_MS); + yield deserializeChunk(chunk); + void runManager?.handleLLMNewToken(chunk.text ?? ''); + state.ledger.chunksConsumed += 1; + } + state.ledger.invocationsConsumed += 1; + writeLedger(fixture.meta.name); + } +} + +let replayProviderRegistered = false; + +function ensureReplayProviderRegistered() { + if (replayProviderRegistered) { + return; + } + try { + registerProvider({ provider: REPLAY_PROVIDER, model: ReplayChatModel }); + } catch (error) { + /** The SDK registry is globalThis-scoped while this guard is + * module-scoped: a reloaded copy of this module finds the provider + * already registered. The registered class is stateless (fixture and + * cursor ride `clientOptions`), so any copy's registration serves all. */ + if (!String(error instanceof Error ? error.message : error).includes('already registered')) { + throw error; + } + } + replayProviderRegistered = true; +} + +/** + * Bind a conversation to a recorded fixture when its latest user text matches + * the fixture's next unconsumed invocation. The replay model is built through + * the SDK's registered-provider path (`registerProvider` + + * `initializeModel`), including real `bindTools` over the run's tools. + * Returns true when the graph's model was overridden with the replaying + * model; false lets the fake-model marker routing proceed unchanged. + */ +/** + * Decide how this run relates to a fixture. + * + * `own` — the conversation that claimed the fixture is back. Its cursor is + * authoritative wherever it stands, including past the end, so an extra turn + * reaches the over-consumption guard instead of falling through to the + * scripted fake model, and a resumed run after a tool-approval pause keeps + * replaying even though it arrives with no messages and no prompt text. + * + * `claim` — a different (or first) conversation opening the fixture: rewind + * and take ownership. This is what a Playwright retry looks like. + * + * Anything else is refused, so an unrelated conversation can never continue + * someone else's partly consumed script by happening to repeat a later prompt. + */ +function classifyBinding({ fixture, state, text, messages, conversationId }) { + if (conversationId != null && state.conversationId != null) { + if (state.conversationId === conversationId) { + return 'own'; + } + return fixture.turns[0] === text ? 'claim' : 'refuse'; + } + /** Identity unavailable (an older `@librechat/api` does not supply it): + * fall back to the text and history rules this lane used before. */ + if (state.cursor !== 0 && isConversationStart(messages)) { + return fixture.turns[0] === text ? 'claim' : 'refuse'; + } + if (fixture.invocations[state.cursor]?.userText === text) { + return 'own'; + } + if (state.cursor !== 0 && fixture.turns[0] === text) { + return 'claim'; + } + if (state.cursor >= fixture.invocations.length && conversationDroveFixture(messages, fixture)) { + return 'own'; + } + return 'refuse'; +} + +function tryBindReplay({ graph, agents, text, messages, conversationId, modelCallbacks }) { + const registry = loadFixtureRegistry(); + const matches = []; + for (const fixture of registry.values()) { + let state = getReplayState(fixture); + const binding = classifyBinding({ fixture, state, text, messages, conversationId }); + if (binding === 'refuse') { + continue; + } + if (binding === 'claim') { + state = restartReplayState(fixture); + } + if (conversationId != null) { + state.conversationId = conversationId; + } + matches.push({ fixture, state }); + } + + if (matches.length === 0) { + return false; + } + /** Binding order would otherwise follow filesystem enumeration, so a second + * fixture sharing this prompt could silently redirect a scenario to the + * wrong chunks and ledger. The spec's fixture choice never reaches this + * server-side loop, so ambiguity has to fail rather than pick a winner. */ + if (matches.length > 1) { + throw new Error( + `[e2e model-replay] prompt matches ${matches.length} fixtures ` + + `(${matches.map(({ fixture }) => fixture.meta.name).join(', ')}); ` + + 'fixtures must not share a bindable prompt', + ); + } + + const { fixture, state } = matches[0]; + ensureReplayProviderRegistered(); + const model = initializeModel({ + provider: REPLAY_PROVIDER, + clientOptions: { fixture, state, callbacks: modelCallbacks }, + tools: agents?.[0]?.tools ?? [], + }); + state.ledger.toolsBound = model.boundToolNames ?? []; + graph.overrideModel = model; + /** `graph.overrideModel` is not inherited by child executors, so a fixture + * recording a subagent call — record mode captures child invocations, since + * the recorder attaches to every agent context — would otherwise leave the + * child on its configured provider: an underrun here, and a real provider + * request in a lane that must stay keyless. */ + if (typeof graph.setSubagentModelOverride === 'function') { + graph.setSubagentModelOverride(model); + } + writeLedger(fixture.meta.name); + return true; +} + +module.exports = { + FIXTURES_DIR, + LEDGER_DIR, + installRecorder, + tryBindReplay, + latestHumanText, + serializeChunk, + deserializeChunk, + parseFixtureFile, +}; diff --git a/e2e/setup/record-model.js b/e2e/setup/record-model.js new file mode 100644 index 0000000000..63ff76ab1d --- /dev/null +++ b/e2e/setup/record-model.js @@ -0,0 +1,23 @@ +/** + * Run hook for `E2E_MODEL_FIXTURES=record`: taps the REAL provider model's + * stream instead of overriding it, writing each model invocation's chunks to + * `e2e/fixtures/model-replay/$E2E_MODEL_FIXTURE_NAME.jsonl` for keyless replay + * through `fake-model.js`. Set as `LIBRECHAT_TEST_RUN_HOOK` by the mock + * Playwright config when recording; the server must be booted with a working + * provider credential (`E2E_RECORD_PROVIDER_API_KEY`). + */ +const { installRecorder } = require('./model-replay'); + +/** @type {import('@librechat/api').TestRunHook} */ +module.exports = function recordModelHook(run, context) { + const graph = run?.Graph; + if (!graph) { + console.warn('[e2e model-replay] record hook: run.Graph unavailable'); + return; + } + installRecorder({ + graph, + messages: context?.messages, + conversationId: context?.conversationId, + }); +}; diff --git a/e2e/specs/mock/model-replay-tools.spec.ts b/e2e/specs/mock/model-replay-tools.spec.ts new file mode 100644 index 0000000000..df7d1f1c50 --- /dev/null +++ b/e2e/specs/mock/model-replay-tools.spec.ts @@ -0,0 +1,166 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import type { TMessage } from 'librechat-data-provider'; +import { + assertFixtureConsumed, + fixtureTurns, + readReplayLedger, + removeFixture, +} from './replay.helpers'; +import { + NEW_CHAT_PATH, + fetchJson, + getAccessToken, + selectMockEndpoint, + sendMessageAndWaitForCompletion, +} from './helpers'; + +/** + * The tool-call half of the replay lane: one recorded turn in which the real + * provider calls an MCP tool, the tool runs for real, and the model is invoked + * a second time with its result. + * + * This is the shape a single prompt cannot express — one user turn spanning + * several model invocations — so it is what proves the fixture format carries + * `tool_call_chunks` and that replay advances through a turn's invocations + * rather than binding once per prompt. + * + * Record (needs a real provider key): + * E2E_MODEL_FIXTURES=record E2E_MODEL_FIXTURE_NAME=deepseek-tool-call \ + * E2E_RECORD_PROVIDER_API_KEY= \ + * npx playwright test --config=e2e/playwright.config.mock.ts model-replay-tools + * + * Replay (default, keyless): the same drive steps against the committed + * fixture, with the recorded tool call streamed back through the real graph so + * the tool executes again. + */ +const COMMITTED_FIXTURE = 'deepseek-tool-call'; +const RECORDING = process.env.E2E_MODEL_FIXTURES === 'record'; +const RECORDING_THIS_FIXTURE = + RECORDING && process.env.E2E_MODEL_FIXTURE_NAME === COMMITTED_FIXTURE; +const FIXTURE = COMMITTED_FIXTURE; +const RECORD_ENDPOINT = { + label: 'Replay Record Provider', + model: process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat', +}; + +const MCP_SERVER_TITLE = 'E2E Memory'; +const TOOL_NAME = 'remember_fact'; +/** MCP tools reach the model under a server-qualified name + * (`remember_fact_mcp_e2e-memory`), and that qualification has changed before, + * so assertions match the base name as a prefix rather than pinning the suffix. */ +const namesTool = (names: string[]) => names.some((name) => name.startsWith(TOOL_NAME)); +/** The MCP fixture echoes this back, so the tool result is deterministic. */ +const FACT = 'the replay lane records tool calls'; +const TOOL_PROMPT = + `Call the ${TOOL_NAME} tool with fact set to "${FACT}", then reply with exactly the ` + + 'text the tool returned and nothing else.'; + +/** Enable the ephemeral MCP server whose tools this turn calls. */ +async function selectEphemeralMCP(page: Page) { + await page.getByRole('button', { name: 'MCP Servers', exact: true }).click(); + const serverItem = page.getByRole('menuitemcheckbox', { name: new RegExp(MCP_SERVER_TITLE) }); + await expect(serverItem).toBeVisible(); + await serverItem.click(); + await expect(serverItem).toHaveAttribute('aria-checked', 'true'); + await page.keyboard.press('Escape'); + await expect(page.getByRole('button', { name: new RegExp(MCP_SERVER_TITLE) })).toBeVisible(); +} + +test.describe('recorded tool-call fixture replay', () => { + test('a recorded tool call replays through the real tool node', async ({ page }) => { + test.skip( + RECORDING && !RECORDING_THIS_FIXTURE, + `recording ${process.env.E2E_MODEL_FIXTURE_NAME}`, + ); + test.setTimeout(180_000); + const pageErrors: string[] = []; + page.on('pageerror', (error) => pageErrors.push(error.message)); + + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + if (RECORDING) { + removeFixture(FIXTURE); + await selectMockEndpoint(page, RECORD_ENDPOINT); + } + await selectEphemeralMCP(page); + + await sendMessageAndWaitForCompletion(page, TOOL_PROMPT, { timeout: 120_000 }); + + const conversationId = /\/c\/([^/]+)/.exec(page.url())?.[1]; + expect(conversationId, 'conversation should have a persisted id').toBeTruthy(); + const token = await getAccessToken(page); + const messages = await fetchJson( + page, + `/api/messages/${encodeURIComponent(conversationId as string)}`, + token, + ); + const assistant = messages.filter((message) => message.isCreatedByUser === false); + expect(assistant).toHaveLength(1); + /** The turn's durable proof that the tool ran: a persisted tool_call part + * naming the tool, independent of whatever prose the model wrapped it in. */ + const toolCallParts = (assistant[0].content ?? []).filter((part) => part?.type === 'tool_call'); + expect(toolCallParts.length, 'the turn should persist a tool call').toBeGreaterThan(0); + expect(JSON.stringify(toolCallParts)).toContain(TOOL_NAME); + + if (RECORDING) { + await expect + .poll( + () => { + try { + const settled = fixtureTurns(FIXTURE); + return settled.length >= 2 && settled.every((turn) => turn.userText === TOOL_PROMPT); + } catch { + return false; + } + }, + { + timeout: 15_000, + intervals: [250, 500, 1_000], + message: 'recording should settle with several invocations under one prompt', + }, + ) + .toBe(true); + const recorded = fixtureTurns(FIXTURE); + expect( + recorded.map((turn) => turn.userText), + 'every invocation of this turn shares its one user prompt', + ).toEqual(recorded.map(() => TOOL_PROMPT)); + expect( + recorded[0].toolCallChunkCount, + 'the first invocation should stream the tool call', + ).toBeGreaterThan(0); + expect( + namesTool(recorded[0].toolNames), + `the streamed tool call should name ${TOOL_NAME}, got ${JSON.stringify(recorded[0].toolNames)}`, + ).toBe(true); + expect( + recorded[recorded.length - 1].contentChunkCount, + 'the post-tool invocation should stream the answer', + ).toBeGreaterThan(0); + } else { + const turns = fixtureTurns(FIXTURE); + expect( + turns.length, + 'the fixture should hold more than one invocation for this single turn', + ).toBeGreaterThan(1); + expect( + namesTool(turns[0].toolNames), + `the recorded tool call should name ${TOOL_NAME}, got ${JSON.stringify(turns[0].toolNames)}`, + ).toBe(true); + + /** Replay drives the real tool node, so the tool ran again in this run + * rather than being replayed as recorded output. */ + const toolResult = JSON.stringify(assistant[0].content ?? []); + expect(toolResult, 'the replayed tool call should carry its real result').toContain(FACT); + + const ledger = readReplayLedger(FIXTURE); + expect( + ledger.invocationsConsumed, + 'replay should advance through every invocation of the turn', + ).toBe(ledger.invocationsTotal); + assertFixtureConsumed(FIXTURE); + } + + expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0); + }); +}); diff --git a/e2e/specs/mock/model-replay.spec.ts b/e2e/specs/mock/model-replay.spec.ts new file mode 100644 index 0000000000..03131cdfb9 --- /dev/null +++ b/e2e/specs/mock/model-replay.spec.ts @@ -0,0 +1,191 @@ +import { expect, test } from '@playwright/test'; +import type { TMessage } from 'librechat-data-provider'; +import { + assertFixtureConsumed, + fixtureTurns, + readReplayLedger, + removeFixture, +} from './replay.helpers'; +import { + NEW_CHAT_PATH, + fetchJson, + messagesView, + getAccessToken, + selectMockEndpoint, + sendMessageAndWaitForCompletion, +} from './helpers'; + +/** + * Record-once/replay-forever coverage: one recorded real-provider conversation + * replays keylessly through the real createRun → registered replay provider → + * graph → SSE → persistence chain. + * + * Record (writes the fixture; needs a real provider key): + * E2E_MODEL_FIXTURES=record E2E_MODEL_FIXTURE_NAME=deepseek-two-turn \ + * E2E_RECORD_PROVIDER_API_KEY= \ + * npx playwright test --config=e2e/playwright.config.mock.ts model-replay + * + * Replay (default, keyless): the same drive steps; the conversation binds to + * the committed fixture by prompt text, assistant turns must equal the + * recorded turns exactly, and the consumption ledger must drain completely. + */ +const COMMITTED_FIXTURE = 'deepseek-two-turn'; +const RECORDING = process.env.E2E_MODEL_FIXTURES === 'record'; +/** + * Record mode collects every replay spec, so each one records only the fixture + * it owns and stands down for the others. A spec must never write a fixture + * other than its own: two fixtures carrying the same prompts would make the + * server-side binding ambiguous and refuse both. + */ +const RECORDING_THIS_FIXTURE = + RECORDING && process.env.E2E_MODEL_FIXTURE_NAME === COMMITTED_FIXTURE; +const FIXTURE = COMMITTED_FIXTURE; +const RECORD_ENDPOINT = { + label: 'Replay Record Provider', + model: process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat', +}; + +/** + * The closing prompt deliberately asks for prose: a one-token answer streams + * as a single content delta wrapped in empty initialization and usage frames, + * which cannot demonstrate incremental content streaming however many chunks + * the provider emits around it. + */ +const TURN_PROMPTS = [ + 'Name the two prime numbers between 20 and 30, comma separated, and nothing else.', + 'In two short sentences, explain why the sum of those two primes is an even number. Begin with the word "Because".', +]; + +test.describe('recorded model fixture replay', () => { + test('a recorded conversation replays deterministically through the real pipeline', async ({ + page, + }) => { + test.skip( + RECORDING && !RECORDING_THIS_FIXTURE, + `recording ${process.env.E2E_MODEL_FIXTURE_NAME}`, + ); + test.setTimeout(180_000); + const pageErrors: string[] = []; + page.on('pageerror', (error) => pageErrors.push(error.message)); + + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + if (RECORDING) { + /** Proof of a fresh write: the assertions below cannot be satisfied by a + * pre-existing fixture, so a run whose recorder never installed fails + * instead of greening against a stale artifact. */ + removeFixture(FIXTURE); + await selectMockEndpoint(page, RECORD_ENDPOINT); + } + + for (const prompt of TURN_PROMPTS) { + await sendMessageAndWaitForCompletion(page, prompt, { timeout: 90_000 }); + } + + const conversationId = /\/c\/([^/]+)/.exec(page.url())?.[1]; + expect(conversationId, 'conversation should have a persisted id').toBeTruthy(); + const token = await getAccessToken(page); + const messages = await fetchJson( + page, + `/api/messages/${encodeURIComponent(conversationId as string)}`, + token, + ); + /** Agents-pipeline messages persist their text inside `content` parts; + * top-level `text` stays empty. */ + const persistedText = (message: TMessage): string => { + if (message.text) { + return message.text; + } + return (message.content ?? []) + .map((part) => { + if (part?.type !== 'text') { + return ''; + } + const text = (part as { text?: string | { value?: string } }).text; + return typeof text === 'string' ? text : (text?.value ?? ''); + }) + .join(''); + }; + const assistantTexts = messages + .filter((message) => message.isCreatedByUser === false) + .map(persistedText); + expect(assistantTexts).toHaveLength(TURN_PROMPTS.length); + + if (RECORDING) { + /** The recorder rides LangChain token callbacks, which the provider + * stream dispatches without awaiting — the recording quiesces shortly + * AFTER the durable-completion barrier, so the record-mode harvest + * polls for the settled fixture instead of asserting a single read. + * Replay mode needs no such poll: the replaying generator finishes its + * ledger writes before the turn can persist. */ + await expect + .poll( + () => { + try { + const settled = fixtureTurns(FIXTURE); + return ( + settled.length === TURN_PROMPTS.length && + settled.every( + (turn, index) => + turn.userText === TURN_PROMPTS[index] && + turn.finalText === assistantTexts[index], + ) + ); + } catch { + return false; + } + }, + { + timeout: 15_000, + intervals: [250, 500, 1_000], + message: 'recorded fixture should quiesce with the persisted assistant turns', + }, + ) + .toBe(true); + const recorded = fixtureTurns(FIXTURE); + for (const turn of recorded) { + expect(turn.contentChunkCount, 'each turn should record assistant content').toBeGreaterThan( + 0, + ); + } + expect( + recorded[recorded.length - 1].contentChunkCount, + 'the prose turn should record several content deltas, not one delta padded with empty frames', + ).toBeGreaterThan(1); + } else { + const turns = fixtureTurns(FIXTURE); + expect( + turns.map((turn) => turn.userText), + 'fixture invocations should mirror the driven prompts', + ).toEqual(TURN_PROMPTS); + expect(assistantTexts, 'replayed assistant turns should equal the recording exactly').toEqual( + turns.map((turn) => turn.finalText), + ); + /** Compare against rendered markdown, not the raw recording: a reply + * opening with `52.` is rendered as an ordered-list marker and never + * appears in the DOM text, so a leading enumerator is stripped before + * matching and only a prose prefix is used. */ + const renderedPrefix = turns[turns.length - 1].finalText + .replace(/^\s*\d+[.)]\s*/, '') + .trim() + .slice(0, 30); + expect( + renderedPrefix.length, + 'the prose turn should yield a comparable prefix', + ).toBeGreaterThan(15); + await expect(messagesView(page)).toContainText(renderedPrefix); + + /** Streaming incrementality is asserted from the ledger's drained chunk + * count, not by sampling transient DOM — every recorded chunk passed + * through the live SSE wire before the durable completion barrier. */ + const ledger = readReplayLedger(FIXTURE); + expect(ledger.chunksConsumed).toBe(ledger.chunksTotal); + expect( + turns[turns.length - 1].contentChunkCount, + 'replay should stream several content deltas for the prose turn', + ).toBeGreaterThan(1); + assertFixtureConsumed(FIXTURE); + } + + expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0); + }); +}); diff --git a/e2e/specs/mock/replay.helpers.ts b/e2e/specs/mock/replay.helpers.ts new file mode 100644 index 0000000000..042ded965b --- /dev/null +++ b/e2e/specs/mock/replay.helpers.ts @@ -0,0 +1,145 @@ +import fs from 'fs'; +import path from 'path'; + +/** + * Spec-side readers for the model-fixture replay lane. The server-side + * recorder/replayer (`e2e/setup/model-replay.js`) owns the formats; these + * readers stay dependency-free on that CJS module so the spec plane needs no + * runtime import of server code. + */ +const FIXTURES_DIR = path.resolve(__dirname, '../../fixtures/model-replay'); +const LEDGER_DIR = path.resolve(__dirname, '../.test-results/model-replay'); + +export type FixtureTurn = { + userText: string; + finalText: string; + chunkCount: number; + /** + * Chunks carrying assistant text, as distinct from the empty + * initialization and usage-metadata chunks a provider also emits. Only + * these prove incremental content streaming — a total chunk count above one + * is satisfied by a single content delta wrapped in empty frames. + */ + contentChunkCount: number; + /** Chunks carrying `tool_call_chunks`, i.e. the streamed tool invocation. */ + toolCallChunkCount: number; + /** Tool names streamed by this invocation, in order of first appearance. */ + toolNames: string[]; +}; + +export type ReplayLedger = { + fixture: string; + invocationsTotal: number; + chunksTotal: number; + invocationsConsumed: number; + chunksConsumed: number; + overruns: Array<{ at: string; userText: string }>; + promptMismatches: Array<{ invocation: number; expected: string; received: string }>; +}; + +export function fixturePath(name: string): string { + return path.join(FIXTURES_DIR, `${name}.jsonl`); +} + +/** + * Remove a fixture before a recording run so its assertions cannot be + * satisfied by a pre-existing artifact. Without this, a run whose hook failed + * to install the recorder would still see the live provider answer these + * deterministic prompts while the poll read the stale file — matching answers, + * valid chunk counts, and a green run that wrote nothing. + */ +export function removeFixture(name: string): void { + fs.rmSync(fixturePath(name), { force: true }); +} + +/** + * Parse a fixture's invocations in recorded order. An invocation's final text + * is the concatenation of its recorded chunk texts — the chunks are written + * synchronously during the stream, while the provider's `handleLLMEnd` + * dispatch (the `end` line) can land after the durable-completion barrier a + * spec waits on, so nothing here depends on it. + */ +export function fixtureTurns(name: string): FixtureTurn[] { + const lines = fs + .readFileSync(fixturePath(name), 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + const turns: FixtureTurn[] = []; + for (const entry of lines) { + if (entry.type === 'invocation') { + turns[entry.index as number] = { + userText: entry.userText as string, + finalText: '', + chunkCount: 0, + contentChunkCount: 0, + toolCallChunkCount: 0, + toolNames: [], + }; + } else if (entry.type === 'chunk') { + const turn = turns[entry.invocation as number]; + if (turn) { + const text = (entry.text as string) ?? ''; + turn.chunkCount += 1; + turn.finalText += text; + if (text !== '') { + turn.contentChunkCount += 1; + } + const message = entry.message as + | { tool_call_chunks?: Array<{ name?: string }> } + | undefined; + const toolCallChunks = message?.tool_call_chunks ?? []; + if (toolCallChunks.length > 0) { + turn.toolCallChunkCount += 1; + for (const call of toolCallChunks) { + if (call.name && !turn.toolNames.includes(call.name)) { + turn.toolNames.push(call.name); + } + } + } + } + } else if (entry.type === 'error') { + throw new Error(`Fixture ${name} recorded a provider error: ${String(entry.message)}`); + } + } + return turns; +} + +export function readReplayLedger(name: string): ReplayLedger { + const ledgerPath = path.join(LEDGER_DIR, `${name}.json`); + if (!fs.existsSync(ledgerPath)) { + throw new Error( + `Replay ledger missing for fixture "${name}" (${ledgerPath}); ` + + 'the conversation never bound to the fixture', + ); + } + return JSON.parse(fs.readFileSync(ledgerPath, 'utf8')) as ReplayLedger; +} + +/** + * The teardown consumption check: every recorded invocation and chunk was + * drained, nothing was invoked past the script, and every prompt matched its + * recording. Converts silent underruns and shifted bindings into crisp + * diagnostics. + */ +export function assertFixtureConsumed(name: string): void { + const ledger = readReplayLedger(name); + const failures: string[] = []; + if (ledger.invocationsConsumed !== ledger.invocationsTotal) { + failures.push( + `under-consumed: ${ledger.invocationsConsumed}/${ledger.invocationsTotal} invocations`, + ); + } + if (ledger.chunksConsumed !== ledger.chunksTotal) { + failures.push(`under-streamed: ${ledger.chunksConsumed}/${ledger.chunksTotal} chunks`); + } + if (ledger.overruns.length > 0) { + failures.push(`over-consumed ${ledger.overruns.length}x: ${JSON.stringify(ledger.overruns)}`); + } + if (ledger.promptMismatches.length > 0) { + failures.push(`prompt mismatches: ${JSON.stringify(ledger.promptMismatches)}`); + } + if (failures.length > 0) { + throw new Error(`Fixture "${name}" consumption check failed — ${failures.join('; ')}`); + } +} diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 13b2725986..029807ebba 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -1301,6 +1301,7 @@ function buildSubagentConfigs( export async function createRun({ runId, signal, + conversationId, agents, messages, discoveredToolNames, @@ -1331,6 +1332,9 @@ export async function createRun({ agents: RunAgent[]; signal: AbortSignal; runId?: string; + /** Conversation-stable identity, used by the e2e run hook to tell a resumed + * run apart from a fresh attempt (a resume carries no messages). */ + conversationId?: string; streaming?: boolean; streamUsage?: boolean; requestBody?: t.RequestBody; @@ -1991,6 +1995,11 @@ export async function createRun({ const run = await Run.create(runConfig); applyCustomHandoffPromptKeyCompatibility(run, runConfig.graphConfig); - applyTestRunHook(run, { messages, agents, modelCallbacks }); + applyTestRunHook(run, { + messages, + agents, + modelCallbacks, + conversationId: conversationId ?? requestBody?.conversationId ?? undefined, + }); return run; } diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.ts index ef0a83d968..487369aa86 100644 --- a/packages/api/src/agents/testHook.ts +++ b/packages/api/src/agents/testHook.ts @@ -9,6 +9,13 @@ import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent */ export interface TestRunHookContext { messages?: BaseMessage[]; + /** + * Identifies the conversation this run belongs to. A resumed run rebuilds + * `createRun` with no messages because state is rehydrated from the + * checkpoint, so message history alone cannot tell a resume apart from a + * fresh attempt; this is the stable identity across both. + */ + conversationId?: string; agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>; modelCallbacks?: readonly ModelBoundChatModelCallback[]; }