diff --git a/api/server/middleware/buildEndpointOption.js b/api/server/middleware/buildEndpointOption.js index 7aeffdfb82..a762c153ea 100644 --- a/api/server/middleware/buildEndpointOption.js +++ b/api/server/middleware/buildEndpointOption.js @@ -60,7 +60,14 @@ async function buildEndpointOption(req, res, next) { if (appConfig.modelSpecs?.list?.length && appConfig.modelSpecs?.enforce) { /** @type {{ list: TModelSpec[] }}*/ const { list } = appConfig.modelSpecs; - const { spec } = parsedBody; + const rawSpec = req.body.spec; + const spec = parsedBody.spec ?? (typeof rawSpec === 'string' ? rawSpec : undefined); + const rawChatProjectId = req.body.chatProjectId; + const parsedBodyForModelSpec = + parsedBody.chatProjectId === undefined && + (typeof rawChatProjectId === 'string' || rawChatProjectId === null) + ? { ...parsedBody, chatProjectId: rawChatProjectId } + : parsedBody; if (!spec) { return handleError(res, { text: 'No model spec selected' }); @@ -78,7 +85,7 @@ async function buildEndpointOption(req, res, next) { try { const result = applyModelSpecPreset({ modelSpec: currentModelSpec, - parsedBody, + parsedBody: parsedBodyForModelSpec, endpoint, endpointType, defaultParamsEndpoint, diff --git a/api/server/middleware/buildEndpointOption.spec.js b/api/server/middleware/buildEndpointOption.spec.js index 6c1fa1041d..c4da3a726e 100644 --- a/api/server/middleware/buildEndpointOption.spec.js +++ b/api/server/middleware/buildEndpointOption.spec.js @@ -220,6 +220,46 @@ describe('buildEndpointOption - defaultParamsEndpoint parsing', () => { expect(req.body.endpointOption.chatProjectId).toBe('project-1'); }); + it('should rebuild enforced custom specs from the backend preset when compact parsing drops raw fields', async () => { + mockGetEndpointsConfig.mockResolvedValue({}); + + const modelSpec = { + name: 'approved-custom', + preset: { + endpoint: 'Mock Provider A', + endpointType: EModelEndpoint.custom, + model: 'mock-model-a', + promptPrefix: 'Use the approved custom model spec.', + }, + }; + + const req = createReq( + { + endpoint: 'Mock Provider A', + endpointType: EModelEndpoint.custom, + spec: 'approved-custom', + model: { stale: 'cached-client-value' }, + agent_id: 'agent_from_cached_client_state', + chatProjectId: 'project-1', + }, + { + modelSpecs: { + enforce: true, + list: [modelSpec], + }, + }, + ); + req.baseUrl = '/api/agents/chat'; + + await buildEndpointOption(req, createRes(), jest.fn()); + + expect(parseCompactConvo.mock.results[0].value).toEqual({}); + expect(req.body.endpointOption.spec).toBe('approved-custom'); + expect(req.body.endpointOption.model).toBe('mock-model-a'); + expect(req.body.endpointOption.promptPrefix).toBe('Use the approved custom model spec.'); + expect(req.body.endpointOption.chatProjectId).toBe('project-1'); + }); + it('should restore private model spec preset fields in non-enforced mode', async () => { mockGetEndpointsConfig.mockResolvedValue({}); diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index 1ef0b22a73..d1ef19097d 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -48,8 +48,12 @@ const preservedCredentialEnvKeys = new Set([ */ function writeRuntimeMockConfig() { const template = fs.readFileSync(configTemplatePath, 'utf8'); + const config = + process.env.E2E_MODEL_SPECS_ENFORCE === 'true' + ? template.replace('\n enforce: false\n', '\n enforce: true\n') + : template; fs.mkdirSync(path.dirname(configPath), { recursive: true }); - fs.writeFileSync(configPath, template); + fs.writeFileSync(configPath, config); } function neutralizeCredentialEnv(env: NodeJS.ProcessEnv, keep: Set) { diff --git a/e2e/specs/mock/enforced-model-specs.spec.ts b/e2e/specs/mock/enforced-model-specs.spec.ts new file mode 100644 index 0000000000..d8f9ad1abb --- /dev/null +++ b/e2e/specs/mock/enforced-model-specs.spec.ts @@ -0,0 +1,170 @@ +import { randomUUID } from 'crypto'; +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + getAccessToken, + messagesView, + replyPrompt, + replyText, + requestJson, + selectModelSpec, + sendMessage, +} from './helpers'; + +const NO_PARENT = '00000000-0000-0000-0000-000000000000'; +const ENFORCED_SPEC_NAME = 'e2e-mock-provider-a'; +const ENFORCED_SPEC_LABEL = 'Mock Provider A'; + +type AgentStartResponse = { + conversationId: string; + streamId: string; + status: string; +}; + +async function createProject(page: Page, name: string): Promise { + await page.goto('/projects', { timeout: 10000 }); + await page.getByRole('button', { name: 'New project' }).first().click(); + + const dialog = page.getByRole('dialog'); + await dialog.getByRole('textbox', { name: 'Project name' }).fill(name); + await dialog.getByRole('button', { name: 'Create project' }).click(); + + await expect(page.getByRole('heading', { name })).toBeVisible(); + const projectId = new URL(page.url()).pathname.split('/projects/')[1]; + expect(projectId).toBeTruthy(); + return projectId; +} + +const uniqueName = (prefix: string) => `${prefix} ${Date.now()}-${Math.floor(Math.random() * 1e4)}`; + +async function waitForStreamReply(page: Page, token: string, streamId: string, expected: string) { + await page.evaluate( + async ({ accessToken, expectedText, currentStreamId }) => { + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), 60000); + let buffered = ''; + + try { + const response = await fetch( + `/api/agents/chat/stream/${encodeURIComponent(currentStreamId)}?resume=true`, + { + method: 'GET', + credentials: 'include', + headers: { Authorization: `Bearer ${accessToken}` }, + signal: controller.signal, + }, + ); + + if (!response.ok || !response.body) { + throw new Error(`Expected stream to return 2xx, got ${response.status}`); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + buffered += decoder.decode(value, { stream: true }); + if (buffered.includes(expectedText)) { + await reader.cancel(); + return; + } + + if (buffered.includes('event: error')) { + await reader.cancel(); + throw new Error(`Stream emitted an error before ${expectedText}:\n${buffered}`); + } + } + } finally { + window.clearTimeout(timeout); + } + + throw new Error(`Timed out waiting for ${expectedText}. Latest stream:\n${buffered}`); + }, + { accessToken: token, currentStreamId: streamId, expectedText: expected }, + ); +} + +test.describe('enforced model specs', () => { + test.skip( + process.env.E2E_MODEL_SPECS_ENFORCE !== 'true', + 'requires E2E_MODEL_SPECS_ENFORCE=true', + ); + + test('rebuilds a valid enforced spec from the backend preset when the request is stale', async ({ + page, + }) => { + test.setTimeout(120000); + const label = uniqueName('reported-spec').replace(/\s+/g, '-'); + + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + const token = await getAccessToken(page); + const userMessageId = randomUUID(); + const start = await requestJson(page, { + path: `/api/agents/chat/${encodeURIComponent(MOCK_ENDPOINTS[0].label)}`, + token, + method: 'POST', + body: { + text: replyPrompt(label), + sender: 'User', + clientTimestamp: new Date().toLocaleString('sv').replace(' ', 'T'), + isCreatedByUser: true, + parentMessageId: NO_PARENT, + conversationId: 'new', + messageId: userMessageId, + responseMessageId: `${userMessageId}_`, + endpoint: MOCK_ENDPOINTS[0].label, + endpointType: 'custom', + model: { stale: 'cached-client-value' }, + agent_id: 'agent_from_cached_client_state', + spec: ENFORCED_SPEC_NAME, + isTemporary: false, + isRegenerate: false, + error: false, + }, + }); + + expect(start.status).toBe('started'); + expect(start.conversationId).toBeTruthy(); + + await waitForStreamReply(page, token, start.streamId, replyText(label)); + }); + + test('keeps a project-scoped chat attached when sending with an enforced spec', async ({ + page, + }) => { + test.setTimeout(120000); + const name = uniqueName('E2E Enforced Project'); + const projectId = await createProject(page, name); + const label = uniqueName('project-spec').replace(/\s+/g, '-'); + + await page.goto(`/c/new?projectId=${projectId}`, { timeout: 10000 }); + await expect(page.getByRole('button', { name: 'Remove from project' })).toBeVisible(); + await expect(page.getByRole('textbox', { name: 'Message input' })).toHaveAttribute( + 'placeholder', + new RegExp(name), + ); + + await selectModelSpec(page, ENFORCED_SPEC_LABEL); + await expect(page.getByRole('button', { name: 'Remove from project' })).toBeVisible(); + + const response = await sendMessage(page, replyPrompt(label)); + expect(response.ok()).toBeTruthy(); + await expect(messagesView(page).getByText(replyText(label))).toBeVisible({ timeout: 30000 }); + await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 }); + + const projectRow = page.getByRole('button', { name }).first(); + if ((await projectRow.getAttribute('aria-expanded')) !== 'true') { + await projectRow.click(); + } + await expect( + page.getByTestId(`project-chats-${projectId}`).getByTestId('convo-item').first(), + ).toBeVisible(); + }); +}); diff --git a/package.json b/package.json index 4b85298b79..7f83b60f7a 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "e2e:a11y": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.a11y.ts --headed", "e2e:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.ts", "e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", + "e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts", "e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts", "e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts", "e2e:record": "npm run e2e:prepare && cross-env E2E_BASE_URL=http://localhost:3333 node e2e/setup/record.js",