diff --git a/client/src/components/Chat/Messages/Content/MarkdownBlocks.bench.tsx b/client/src/components/Chat/Messages/Content/MarkdownBlocks.bench.tsx index e982f0b558..5f7ad381cc 100644 --- a/client/src/components/Chat/Messages/Content/MarkdownBlocks.bench.tsx +++ b/client/src/components/Chat/Messages/Content/MarkdownBlocks.bench.tsx @@ -3,6 +3,7 @@ import { RecoilRoot } from 'recoil'; import ReactMarkdown from 'react-markdown'; import { render } from '@testing-library/react'; import { getRemarkPlugins, getRehypePlugins, getMarkdownComponents } from './markdownConfig'; +import { splitMarkdownIntoBlocks, splitMarkdownIntoBlocksUncached } from './splitMarkdown'; import { ArtifactProvider, CodeBlockProvider, MessageContext } from '~/Providers'; import CodeBlock from '~/components/Messages/Content/CodeBlock'; import MarkdownBlocks from './MarkdownBlocks'; @@ -139,6 +140,48 @@ const measure = ( return result; }; +describe('splitMarkdownIntoBlocks streaming benchmark (full re-parse vs incremental)', () => { + it('reports split cost across a simulated stream', () => { + const content = buildMessage(40); + const steps = 1500; + const prefixes = makePrefixes(content, steps); + const iterations = 3; + + const time = (split: (content: string) => void): number => { + split('warm up and reset the cache'); + const start = performance.now(); + for (const prefix of prefixes) { + split(prefix); + } + return performance.now() - start; + }; + + time(splitMarkdownIntoBlocksUncached); + time(splitMarkdownIntoBlocks); + let fullMs = Infinity; + let incrementalMs = Infinity; + for (let i = 0; i < iterations; i += 1) { + fullMs = Math.min(fullMs, time(splitMarkdownIntoBlocksUncached)); + incrementalMs = Math.min(incrementalMs, time(splitMarkdownIntoBlocks)); + } + + console.log( + [ + '', + '============ splitMarkdownIntoBlocks streaming benchmark ============', + `message size: ${content.length} chars, stream steps: ${steps}, iterations: ${iterations}`, + ` full re-parse : ${fullMs.toFixed(1)} ms`, + ` incremental : ${incrementalMs.toFixed(1)} ms`, + ` speedup : ${(fullMs / incrementalMs).toFixed(2)}x`, + '=====================================================================', + '', + ].join('\n'), + ); + + expect(incrementalMs).toBeLessThan(fullMs); + }); +}); + describe('Markdown streaming benchmark (OLD whole-message vs NEW per-block)', () => { it('reports render cost across a simulated stream', () => { const content = buildMessage(12); diff --git a/client/src/components/Chat/Messages/Content/MarkdownBlocks.tsx b/client/src/components/Chat/Messages/Content/MarkdownBlocks.tsx index 256bc9cc75..3d93da5b50 100644 --- a/client/src/components/Chat/Messages/Content/MarkdownBlocks.tsx +++ b/client/src/components/Chat/Messages/Content/MarkdownBlocks.tsx @@ -2,8 +2,9 @@ import React, { memo, useMemo, useState, useLayoutEffect } from 'react'; import ReactMarkdown from 'react-markdown'; import type { PluggableList } from 'unified'; import type { ElementType } from 'react'; +import type { MarkdownSplitter } from './splitMarkdown'; import { ArtifactProvider, CodeBlockProvider } from '~/Providers'; -import { splitMarkdownIntoBlocks } from './splitMarkdown'; +import { createMarkdownSplitter } from './splitMarkdown'; import { createFadePlugin } from './animate'; type SharedProps = { @@ -109,11 +110,11 @@ type BlockEntry = { * a block before existing code or artifacts remounts the blocks it shifted, while * append-only streaming keeps completed blocks mounted. */ -const toBlockEntries = (content: string): BlockEntry[] => { +const toBlockEntries = (content: string, splitMarkdown: MarkdownSplitter): BlockEntry[] => { let codeBaseIndex = 0; let artifactBaseIndex = 0; let mermaidBaseIndex = 0; - return splitMarkdownIntoBlocks(content).map((block, index) => { + return splitMarkdown(content).map((block, index) => { const entry = { key: `${index}-${codeBaseIndex}-${artifactBaseIndex}-${mermaidBaseIndex}`, raw: block.raw, @@ -166,16 +167,17 @@ const MarkdownBlocks = memo(function MarkdownBlocks({ animate, hydrated, }: MarkdownBlocksProps) { + const splitMarkdown = useMemo(() => createMarkdownSplitter(), []); const [mountedContent] = useState(content); const [hasChanged, setHasChanged] = useState(streaming); const changing = streaming || content !== mountedContent; if (changing && !hasChanged) { setHasChanged(true); } - const perBlock = hasChanged || changing; const blocks = useMemo( - () => (perBlock ? toBlockEntries(content) : toWholeMessage(content)), - [content, perBlock], + () => + hasChanged || changing ? toBlockEntries(content, splitMarkdown) : toWholeMessage(content), + [content, hasChanged, changing, splitMarkdown], ); return ( diff --git a/client/src/components/Chat/Messages/Content/__tests__/MarkdownBlocks.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/MarkdownBlocks.test.tsx index 9be9d069b7..c8fb5eac72 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/MarkdownBlocks.test.tsx +++ b/client/src/components/Chat/Messages/Content/__tests__/MarkdownBlocks.test.tsx @@ -73,7 +73,12 @@ jest.mock('~/components/Messages/Content/CodeBlock', () => ({ /** The real splitter, observed: whether a render paid for the block-boundary parse. */ jest.mock('../splitMarkdown', () => { const actual = jest.requireActual('../splitMarkdown'); - return { ...actual, splitMarkdownIntoBlocks: jest.fn(actual.splitMarkdownIntoBlocks) }; + const splitSpy = jest.fn(actual.splitMarkdownIntoBlocks); + return { + ...actual, + splitMarkdownIntoBlocks: splitSpy, + createMarkdownSplitter: jest.fn(() => jest.fn((content: string) => splitSpy(content))), + }; }); const splitSpy = jest.mocked(splitMarkdownIntoBlocks); @@ -284,6 +289,32 @@ describe('MarkdownBlocks code-block index parity', () => { ); }); +describe('MarkdownBlocks provisional boundaries', () => { + it.each([ + ['Intro\n***', 'Intro\n***important***.'], + ['Intro\n#', 'Intro\n#hashtag'], + ['Intro\n```', 'Intro\n```inline```'], + ['- item\n***', '- item\n***important***.'], + ['> quote\n***', '> quote\n***important***.'], + ])('matches a fresh render after streaming past %j', (prefix, content) => { + const view = (text: string, submitting: boolean) => ( + + + + ); + const { container, rerender } = render(view(prefix, true)); + rerender(view(content, true)); + rerender(view(content, false)); + + const { container: fresh } = render( + + + , + ); + expect(normalizeHtml(container.innerHTML)).toBe(normalizeHtml(fresh.innerHTML)); + }); +}); + describe('MarkdownBlocks finished and streamed messages', () => { const PROSE = '# H\n\nPara with `code`.\n\n| x | y |\n| - | - |\n| 1 | 2 |\n\n- a\n- b'; diff --git a/client/src/components/Chat/Messages/Content/__tests__/splitMarkdown.test.ts b/client/src/components/Chat/Messages/Content/__tests__/splitMarkdown.test.ts index 5aed247c15..7317487023 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/splitMarkdown.test.ts +++ b/client/src/components/Chat/Messages/Content/__tests__/splitMarkdown.test.ts @@ -1,8 +1,142 @@ -import { splitMarkdownIntoBlocks } from '../splitMarkdown'; +import { + createMarkdownSplitter, + splitMarkdownIntoBlocks, + splitMarkdownIntoBlocksUncached, +} from '../splitMarkdown'; const raws = (content: string) => splitMarkdownIntoBlocks(content).map((block) => block.raw); describe('splitMarkdownIntoBlocks', () => { + describe('incremental append-only streaming', () => { + /** Splits every prefix in order (incremental) and from scratch (reference). */ + const streamed = (content: string) => { + splitMarkdownIntoBlocks('unrelated content to reset the cache'); + const prefixes = Array.from({ length: content.length }, (_, i) => content.slice(0, i + 1)); + return { + incremental: prefixes.map((prefix) => splitMarkdownIntoBlocks(prefix)), + reference: prefixes.map((prefix) => splitMarkdownIntoBlocksUncached(prefix)), + }; + }; + + it('matches a full parse at every prefix of a mixed document', () => { + const { incremental, reference } = streamed( + [ + '# Title', + '', + 'Para one with `code` and **bold**.', + '', + '```js', + 'const a = 1;', + '', + 'const b = 2;', + '```', + '', + '| a | b |', + '| - | - |', + '| 1 | 2 |', + '', + '- one', + '- two', + '', + ' nested para', + '', + '$$', + 'E = mc^2', + '$$', + '', + ':::artifact{title="t"}', + 'inner', + '', + '```mermaid', + 'graph TD; A-->B;', + '```', + ':::', + '', + 'Setext', + '===', + '', + '> quote', + 'lazy continuation', + '', + ' indented', + '', + ' code continues', + '', + 'Final.', + ].join('\n'), + ); + expect(incremental).toEqual(reference); + }); + + it.each([ + 'Intro\n***important***.', + 'Intro\n#hashtag', + 'Intro\n```inline```', + '- item\n***important***.', + '> quote\n***important***.', + 'Intro\n$$inline$$', + 'Intro\n:::artifact{title="t"}\nbody\n:::', + ])('reconsiders provisional boundaries at every prefix of %j', (content) => { + const document = `# Stable heading\n\nEarlier paragraph.\n\n${content}\n\nFinal.`; + const { incremental, reference } = streamed(document); + expect(incremental).toEqual(reference); + }); + + it('reuses blocks before the two-block tail', () => { + const split = createMarkdownSplitter(); + const content = '# Stable heading\n\nEarlier paragraph.\n\nIntro\n***'; + const before = split(content); + const after = split(`${content}important***.`); + expect(after).toEqual(splitMarkdownIntoBlocksUncached(`${content}important***.`)); + expect(after[0]).toBe(before[0]); + expect(after[1]).toBe(before[1]); + }); + + it('matches a full parse when a streamed definition forces whole-message rendering', () => { + const { incremental, reference } = streamed( + 'See [docs][d] for details.\n\nMore text.\n\n[d]: https://example.com/docs\n\nEnd.', + ); + expect(incremental).toEqual(reference); + }); + + it('matches a full parse when a streamed footnote definition nests in a list item', () => { + const { incremental, reference } = streamed('Claim.[^1]\n\n- item\n\n [^1]: note\n\nEnd.'); + expect(incremental).toEqual(reference); + }); + + it('matches a full parse when a streamed HTML block forces whole-message rendering', () => { + const { incremental, reference } = streamed( + 'Intro.\n\n
a
\n\n
b
\n\nOutro.', + ); + expect(incremental).toEqual(reference); + }); + + it('matches a full parse after non-append edits', () => { + splitMarkdownIntoBlocks('First.\n\nSecond.'); + const edited = 'First.\n\nChanged.\n\nThird.'; + expect(splitMarkdownIntoBlocks(edited)).toEqual(splitMarkdownIntoBlocksUncached(edited)); + const shorter = 'First.'; + expect(splitMarkdownIntoBlocks(shorter)).toEqual(splitMarkdownIntoBlocksUncached(shorter)); + }); + + it('keeps independent stream caches isolated', () => { + const first = createMarkdownSplitter(); + const second = createMarkdownSplitter(); + const firstPrefix = 'First stream.\n\nActive block'; + const secondPrefix = 'Second stream.\n\nActive block'; + + first(firstPrefix); + second(secondPrefix); + + expect(first(`${firstPrefix} continued.`)).toEqual( + splitMarkdownIntoBlocksUncached(`${firstPrefix} continued.`), + ); + expect(second(`${secondPrefix} continued.`)).toEqual( + splitMarkdownIntoBlocksUncached(`${secondPrefix} continued.`), + ); + }); + }); + it('returns [] for empty content', () => { expect(splitMarkdownIntoBlocks('')).toEqual([]); }); diff --git a/client/src/components/Chat/Messages/Content/splitMarkdown.ts b/client/src/components/Chat/Messages/Content/splitMarkdown.ts index 2fc3d6d0f6..af2b59c234 100644 --- a/client/src/components/Chat/Messages/Content/splitMarkdown.ts +++ b/client/src/components/Chat/Messages/Content/splitMarkdown.ts @@ -101,6 +101,19 @@ const parseToMdast = (content: string): MdastNode => mdastExtensions: [gfmFromMarkdown(), directiveFromMarkdown(), mathFromMarkdown()], }) as MdastNode; +type SplitResult = { + blocks: MarkdownBlock[]; + /** Offset where the final two blocks start; the last boundary may still disappear. */ + reparseStart: number; + /** Number of blocks before reparseStart, even after the tail merges into one block. */ + stableBlockCount: number; +}; + +/** The parsed top-level nodes, returned when the text cannot be split. */ +type WholeMessage = { children: MdastNode[] }; + +const isSplit = (result: SplitResult | WholeMessage): result is SplitResult => 'blocks' in result; + /** * Where the line holding a top-level block begins, when only indentation precedes * the block on it. mdast starts a node after its indentation, but that indentation @@ -114,63 +127,116 @@ const lineStartOf = (content: string, offset: number): number => { }; /** - * Split a markdown string into its top-level blocks, returning the exact source - * slice for each block plus the index counts it consumes. Completed blocks - * produce byte-identical slices (and stable counts) across streamed updates, - * which is what makes per-block memoization effective: only the final, still- - * growing block changes from one token to the next. - * - * Inter-block whitespace (blank lines) is not part of any node's span and is - * dropped; block-level elements carry their own margins, so rendering each - * slice independently is visually equivalent to rendering the whole string. + * Parse `text` and split it into per-node blocks. Returns the parsed nodes + * instead when the text cannot be rendered block-by-block and must instead + * render as one whole: + * - reference/footnote definitions are document-scoped (and may be nested in + * a blockquote or list item), so a reference would otherwise render as + * literal text once severed from its definition; + * - top-level raw HTML blocks are escaped to text (rehypeRaw is not enabled), + * so the separator between adjacent HTML blocks would otherwise be dropped. */ -export function splitMarkdownIntoBlocks(content: string): MarkdownBlock[] { - if (!content) { - return []; - } - - const tree = parseToMdast(content); - const children = tree.children ?? []; - +const splitBlocks = (text: string): SplitResult | WholeMessage => { + const children = parseToMdast(text).children ?? []; if (children.length === 0) { - return [{ raw: content, codeBlockCount: 0, artifactCount: 0, mermaidCount: 0 }]; - } - - // Per-block rendering loses document-global context, so render the whole - // message as one block when it uses a construct that needs it: - // - reference/footnote definitions are document-scoped (and may be nested in - // a blockquote or list item), so a reference would otherwise render as - // literal text once severed from its definition; and - // - top-level raw HTML blocks are escaped to text (rehypeRaw is not enabled), - // so the separator between adjacent HTML blocks would otherwise be dropped. - const requiresWholeMessage = children.some( - (node) => node.type === 'html' || containsDefinition(node), - ); - if (requiresWholeMessage) { - return [{ raw: content, ...blockCounts(children) }]; + return { children }; } const blocks: MarkdownBlock[] = []; + const stableBlockCount = Math.max(0, children.length - 2); + let reparseStart = 0; for (const node of children) { + if (node.type === 'html' || containsDefinition(node)) { + return { children }; + } const start = node.position?.start?.offset; const end = node.position?.end?.offset; if (start == null || end == null) { - return [{ raw: content, ...blockCounts(children) }]; + return { children }; } const counts = { code: 0, artifact: 0, mermaid: 0 }; countWithin(node, counts); + const rawStart = lineStartOf(text, start); + if (blocks.length === stableBlockCount) { + reparseStart = rawStart; + } blocks.push({ - raw: content.slice(lineStartOf(content, start), end), + raw: text.slice(rawStart, end), codeBlockCount: counts.code, artifactCount: counts.artifact, mermaidCount: counts.mermaid, }); } - return blocks; + return { blocks, reparseStart, stableBlockCount }; +}; + +/** + * Split a markdown string into its top-level blocks by parsing it in full, + * returning the exact source slice for each block plus the index counts it + * consumes. Prefer `splitMarkdownIntoBlocks`, which reuses the previous split + * when the content only grew. + * + * Inter-block whitespace (blank lines) is not part of any node's span and is + * dropped; block-level elements carry their own margins, so rendering each + * slice independently is visually equivalent to rendering the whole string. + */ +export function splitMarkdownIntoBlocksUncached(content: string): MarkdownBlock[] { + if (!content) { + return []; + } + const split = splitBlocks(content); + return isSplit(split) ? split.blocks : [{ raw: content, ...blockCounts(split.children) }]; } +type SplitCache = SplitResult & { content: string }; + +export type MarkdownSplitter = (content: string) => MarkdownBlock[]; + +/** + * Create an isolated splitter cache for one markdown surface. MarkdownBlocks + * keeps one instance per mounted message so concurrent streams cannot evict + * one another's active tail. The tail includes the preceding block because an + * unfinished marker (such as `***` or `#`) can stop interrupting it when more + * text arrives. Reusing that block before the boundary settles loses paragraph + * and lazy list/blockquote continuations. + */ +export function createMarkdownSplitter(): MarkdownSplitter { + let lastSplit: SplitCache | null = null; + + return (content: string): MarkdownBlock[] => { + if (!content) { + lastSplit = null; + return []; + } + + const prev = lastSplit; + if (prev != null && content.length > prev.content.length && content.startsWith(prev.content)) { + const tail = splitBlocks(content.slice(prev.reparseStart)); + if (isSplit(tail)) { + lastSplit = { + content, + blocks: [...prev.blocks.slice(0, prev.stableBlockCount), ...tail.blocks], + reparseStart: prev.reparseStart + tail.reparseStart, + stableBlockCount: prev.stableBlockCount + tail.stableBlockCount, + }; + return lastSplit.blocks; + } + } + + const split = splitBlocks(content); + if (!isSplit(split)) { + lastSplit = null; + return [{ raw: content, ...blockCounts(split.children) }]; + } + lastSplit = { content, ...split }; + return split.blocks; + }; +} + +export const splitMarkdownIntoBlocks = createMarkdownSplitter(); + const blockCounts = ( children: MdastNode[], ): { codeBlockCount: number; artifactCount: number; mermaidCount: number } => { diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index 69f02b71d0..7dd40b9577 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -53,6 +53,7 @@ const ASK_USER_QUESTION_MARKER = 'E2E_ASK_USER_QUESTION:'; const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:'; const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:'; const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY'; +const STREAMING_MARKDOWN_REPLY_MARKER = 'E2E_STREAMING_MARKDOWN_REPLY'; const STATEFUL_CODE_MARKER = 'E2E_STATEFUL_CODE:'; /** Two prose paragraphs, so a spec can select the message's *closing* block. */ const PARAGRAPHS_REPLY_MARKER = 'E2E_PARAGRAPHS_REPLY'; @@ -531,6 +532,28 @@ function replyResponses(text) { ], }; } + if (text.includes(STREAMING_MARKDOWN_REPLY_MARKER)) { + return { + responses: [ + [ + '## E2E streaming markdown heading', + '', + 'E2E streaming opening paragraph with 日本語 content.', + '', + '```javascript', + 'const e2eIncrementalMarkdown = "complete";', + '```', + '', + '| E2E column | E2E value |', + '| --- | --- |', + '| completed block | visible |', + '', + 'E2E streaming markdown final paragraph.', + ].join('\n'), + ], + sleep: SLOW_CHUNK_DELAY_MS, + }; + } if (text.includes(PARAGRAPHS_REPLY_MARKER)) { /** The quoted cell sits in the first column, so scrolling the table to its diff --git a/e2e/specs/mock/scenarios/streaming-markdown-blocks.spec.ts b/e2e/specs/mock/scenarios/streaming-markdown-blocks.spec.ts new file mode 100644 index 0000000000..6c21718b94 --- /dev/null +++ b/e2e/specs/mock/scenarios/streaming-markdown-blocks.spec.ts @@ -0,0 +1,76 @@ +import { expect, test } from '@playwright/test'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + messagesView, + selectMockEndpoint, + sendMessage, +} from '../helpers'; + +test.describe('streaming markdown blocks', () => { + test('preserves completed blocks while streaming and renders the final markdown @scenario:streamed-markdown-keeps-completed-blocks-and-final-content', async ({ + page, + }) => { + test.setTimeout(90_000); + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + await selectMockEndpoint(page, MOCK_ENDPOINTS[0]); + + const response = await sendMessage(page, 'E2E_STREAMING_MARKDOWN_REPLY'); + expect(response.ok()).toBeTruthy(); + + const assistantMessage = messagesView(page).locator('.message-render').last(); + await expect(page.getByRole('button', { name: 'Stop generating' })).toBeVisible({ + timeout: 30_000, + }); + await expect( + assistantMessage.getByRole('heading', { name: 'E2E streaming markdown heading' }), + ).toBeVisible({ timeout: 30_000 }); + + await expect(assistantMessage.getByText('E2E streaming markdown final paragraph.')).toBeVisible( + { timeout: 60_000 }, + ); + await expect( + assistantMessage.locator('code').filter({ hasText: 'e2eIncrementalMarkdown' }), + ).toBeVisible(); + await expect(assistantMessage.getByRole('table')).toBeVisible(); + await expect(assistantMessage).toContainText('日本語'); + }); + + test('isolates markdown blocks across concurrent conversations @scenario:concurrent-streams-keep-message-blocks-isolated', async ({ + page, + context, + }) => { + test.setTimeout(120_000); + const secondPage = await context.newPage(); + try { + await Promise.all([ + page.goto(NEW_CHAT_PATH, { timeout: 10_000 }), + secondPage.goto(NEW_CHAT_PATH, { timeout: 10_000 }), + ]); + await Promise.all([ + selectMockEndpoint(page, MOCK_ENDPOINTS[0]), + selectMockEndpoint(secondPage, MOCK_ENDPOINTS[0]), + ]); + + await Promise.all([ + sendMessage(page, 'E2E_STREAMING_MARKDOWN_REPLY'), + sendMessage(secondPage, 'E2E_STREAMING_MARKDOWN_REPLY'), + ]); + + const firstAssistantMessage = messagesView(page).locator('.message-render').last(); + const secondAssistantMessage = messagesView(secondPage).locator('.message-render').last(); + await Promise.all([ + expect( + firstAssistantMessage.getByText('E2E streaming markdown final paragraph.'), + ).toBeVisible({ timeout: 90_000 }), + expect( + secondAssistantMessage.getByText('E2E streaming markdown final paragraph.'), + ).toBeVisible({ timeout: 90_000 }), + ]); + await expect(firstAssistantMessage).toContainText('日本語'); + await expect(secondAssistantMessage).toContainText('日本語'); + } finally { + await secondPage.close(); + } + }); +});