⚡ perf: Split Streaming Markdown Blocks Incrementally (#16071)

* ⚡ perf: Split Streaming Markdown Blocks Incrementally

* test: cover streamed markdown block rendering

* fix: isolate streamed markdown splitter caches

* fix: satisfy markdown splitter static checks

* test: mock isolated markdown splitter

* test: isolate markdown splitter spies

* test: cover concurrent streamed markdown messages

* fix: preserve provisional streaming markdown boundaries

---------

Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
Marco Beretta 2026-09-21 01:26:38 +02:00 • committed by GitHub
parent 1f66e9a8e5
commit 9cd1de30f6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 418 additions and 43 deletions

View file

@ -3,6 +3,7 @@ import { RecoilRoot } from 'recoil';
import ReactMarkdown from 'react-markdown';
import { render } from '@testing-library/react';
import { getRemarkPlugins, getRehypePlugins, getMarkdownComponents } from './markdownConfig';
import { splitMarkdownIntoBlocks, splitMarkdownIntoBlocksUncached } from './splitMarkdown';
import { ArtifactProvider, CodeBlockProvider, MessageContext } from '~/Providers';
import CodeBlock from '~/components/Messages/Content/CodeBlock';
import MarkdownBlocks from './MarkdownBlocks';
@ -139,6 +140,48 @@ const measure = (
return result;
};
describe('splitMarkdownIntoBlocks streaming benchmark (full re-parse vs incremental)', () => {
it('reports split cost across a simulated stream', () => {
const content = buildMessage(40);
const steps = 1500;
const prefixes = makePrefixes(content, steps);
const iterations = 3;
const time = (split: (content: string) => void): number => {
split('warm up and reset the cache');
const start = performance.now();
for (const prefix of prefixes) {
split(prefix);
}
return performance.now() - start;
};
time(splitMarkdownIntoBlocksUncached);
time(splitMarkdownIntoBlocks);
let fullMs = Infinity;
let incrementalMs = Infinity;
for (let i = 0; i < iterations; i += 1) {
fullMs = Math.min(fullMs, time(splitMarkdownIntoBlocksUncached));
incrementalMs = Math.min(incrementalMs, time(splitMarkdownIntoBlocks));
}
console.log(
[
'',
'============ splitMarkdownIntoBlocks streaming benchmark ============',
`message size: ${content.length} chars, stream steps: ${steps}, iterations: ${iterations}`,
` full re-parse : ${fullMs.toFixed(1)} ms`,
` incremental : ${incrementalMs.toFixed(1)} ms`,
` speedup : ${(fullMs / incrementalMs).toFixed(2)}x`,
'=====================================================================',
'',
].join('\n'),
);
expect(incrementalMs).toBeLessThan(fullMs);
});
});
describe('Markdown streaming benchmark (OLD whole-message vs NEW per-block)', () => {
it('reports render cost across a simulated stream', () => {
const content = buildMessage(12);

View file

@ -2,8 +2,9 @@ import React, { memo, useMemo, useState, useLayoutEffect } from 'react';
import ReactMarkdown from 'react-markdown';
import type { PluggableList } from 'unified';
import type { ElementType } from 'react';
import type { MarkdownSplitter } from './splitMarkdown';
import { ArtifactProvider, CodeBlockProvider } from '~/Providers';
import { splitMarkdownIntoBlocks } from './splitMarkdown';
import { createMarkdownSplitter } from './splitMarkdown';
import { createFadePlugin } from './animate';
type SharedProps = {
@ -109,11 +110,11 @@ type BlockEntry = {
* a block before existing code or artifacts remounts the blocks it shifted, while
* append-only streaming keeps completed blocks mounted.
*/
const toBlockEntries = (content: string): BlockEntry[] => {
const toBlockEntries = (content: string, splitMarkdown: MarkdownSplitter): BlockEntry[] => {
let codeBaseIndex = 0;
let artifactBaseIndex = 0;
let mermaidBaseIndex = 0;
return splitMarkdownIntoBlocks(content).map((block, index) => {
return splitMarkdown(content).map((block, index) => {
const entry = {
key: `${index}-${codeBaseIndex}-${artifactBaseIndex}-${mermaidBaseIndex}`,
raw: block.raw,
@ -166,16 +167,17 @@ const MarkdownBlocks = memo(function MarkdownBlocks({
animate,
hydrated,
}: MarkdownBlocksProps) {
const splitMarkdown = useMemo(() => createMarkdownSplitter(), []);
const [mountedContent] = useState(content);
const [hasChanged, setHasChanged] = useState(streaming);
const changing = streaming || content !== mountedContent;
if (changing && !hasChanged) {
setHasChanged(true);
}
const perBlock = hasChanged || changing;
const blocks = useMemo(
() => (perBlock ? toBlockEntries(content) : toWholeMessage(content)),
[content, perBlock],
() =>
hasChanged || changing ? toBlockEntries(content, splitMarkdown) : toWholeMessage(content),
[content, hasChanged, changing, splitMarkdown],
);
return (

View file

@ -73,7 +73,12 @@ jest.mock('~/components/Messages/Content/CodeBlock', () => ({
/** The real splitter, observed: whether a render paid for the block-boundary parse. */
jest.mock('../splitMarkdown', () => {
const actual = jest.requireActual<typeof import('../splitMarkdown')>('../splitMarkdown');
return { ...actual, splitMarkdownIntoBlocks: jest.fn(actual.splitMarkdownIntoBlocks) };
const splitSpy = jest.fn(actual.splitMarkdownIntoBlocks);
return {
...actual,
splitMarkdownIntoBlocks: splitSpy,
createMarkdownSplitter: jest.fn(() => jest.fn((content: string) => splitSpy(content))),
};
});
const splitSpy = jest.mocked(splitMarkdownIntoBlocks);
@ -284,6 +289,32 @@ describe('MarkdownBlocks code-block index parity', () => {
);
});
describe('MarkdownBlocks provisional boundaries', () => {
it.each([
['Intro\n***', 'Intro\n***important***.'],
['Intro\n#', 'Intro\n#hashtag'],
['Intro\n```', 'Intro\n```inline```'],
['- item\n***', '- item\n***important***.'],
['> quote\n***', '> quote\n***important***.'],
])('matches a fresh render after streaming past %j', (prefix, content) => {
const view = (text: string, submitting: boolean) => (
<TestProviders>
<LiveMarkdown content={text} submitting={submitting} />
</TestProviders>
);
const { container, rerender } = render(view(prefix, true));
rerender(view(content, true));
rerender(view(content, false));
const { container: fresh } = render(
<TestProviders>
<OldMarkdown content={content} />
</TestProviders>,
);
expect(normalizeHtml(container.innerHTML)).toBe(normalizeHtml(fresh.innerHTML));
});
});
describe('MarkdownBlocks finished and streamed messages', () => {
const PROSE = '# H\n\nPara with `code`.\n\n| x | y |\n| - | - |\n| 1 | 2 |\n\n- a\n- b';

View file

@ -1,8 +1,142 @@
import { splitMarkdownIntoBlocks } from '../splitMarkdown';
import {
createMarkdownSplitter,
splitMarkdownIntoBlocks,
splitMarkdownIntoBlocksUncached,
} from '../splitMarkdown';
const raws = (content: string) => splitMarkdownIntoBlocks(content).map((block) => block.raw);
describe('splitMarkdownIntoBlocks', () => {
describe('incremental append-only streaming', () => {
/** Splits every prefix in order (incremental) and from scratch (reference). */
const streamed = (content: string) => {
splitMarkdownIntoBlocks('unrelated content to reset the cache');
const prefixes = Array.from({ length: content.length }, (_, i) => content.slice(0, i + 1));
return {
incremental: prefixes.map((prefix) => splitMarkdownIntoBlocks(prefix)),
reference: prefixes.map((prefix) => splitMarkdownIntoBlocksUncached(prefix)),
};
};
it('matches a full parse at every prefix of a mixed document', () => {
const { incremental, reference } = streamed(
[
'# Title',
'',
'Para one with `code` and **bold**.',
'',
'```js',
'const a = 1;',
'',
'const b = 2;',
'```',
'',
'| a | b |',
'| - | - |',
'| 1 | 2 |',
'',
'- one',
'- two',
'',
' nested para',
'',
'$$',
'E = mc^2',
'$$',
'',
':::artifact{title="t"}',
'inner',
'',
'```mermaid',
'graph TD; A-->B;',
'```',
':::',
'',
'Setext',
'===',
'',
'> quote',
'lazy continuation',
'',
' indented',
'',
' code continues',
'',
'Final.',
].join('\n'),
);
expect(incremental).toEqual(reference);
});
it.each([
'Intro\n***important***.',
'Intro\n#hashtag',
'Intro\n```inline```',
'- item\n***important***.',
'> quote\n***important***.',
'Intro\n$$inline$$',
'Intro\n:::artifact{title="t"}\nbody\n:::',
])('reconsiders provisional boundaries at every prefix of %j', (content) => {
const document = `# Stable heading\n\nEarlier paragraph.\n\n${content}\n\nFinal.`;
const { incremental, reference } = streamed(document);
expect(incremental).toEqual(reference);
});
it('reuses blocks before the two-block tail', () => {
const split = createMarkdownSplitter();
const content = '# Stable heading\n\nEarlier paragraph.\n\nIntro\n***';
const before = split(content);
const after = split(`${content}important***.`);
expect(after).toEqual(splitMarkdownIntoBlocksUncached(`${content}important***.`));
expect(after[0]).toBe(before[0]);
expect(after[1]).toBe(before[1]);
});
it('matches a full parse when a streamed definition forces whole-message rendering', () => {
const { incremental, reference } = streamed(
'See [docs][d] for details.\n\nMore text.\n\n[d]: https://example.com/docs\n\nEnd.',
);
expect(incremental).toEqual(reference);
});
it('matches a full parse when a streamed footnote definition nests in a list item', () => {
const { incremental, reference } = streamed('Claim.[^1]\n\n- item\n\n [^1]: note\n\nEnd.');
expect(incremental).toEqual(reference);
});
it('matches a full parse when a streamed HTML block forces whole-message rendering', () => {
const { incremental, reference } = streamed(
'Intro.\n\n<div>a</div>\n\n<div>b</div>\n\nOutro.',
);
expect(incremental).toEqual(reference);
});
it('matches a full parse after non-append edits', () => {
splitMarkdownIntoBlocks('First.\n\nSecond.');
const edited = 'First.\n\nChanged.\n\nThird.';
expect(splitMarkdownIntoBlocks(edited)).toEqual(splitMarkdownIntoBlocksUncached(edited));
const shorter = 'First.';
expect(splitMarkdownIntoBlocks(shorter)).toEqual(splitMarkdownIntoBlocksUncached(shorter));
});
it('keeps independent stream caches isolated', () => {
const first = createMarkdownSplitter();
const second = createMarkdownSplitter();
const firstPrefix = 'First stream.\n\nActive block';
const secondPrefix = 'Second stream.\n\nActive block';
first(firstPrefix);
second(secondPrefix);
expect(first(`${firstPrefix} continued.`)).toEqual(
splitMarkdownIntoBlocksUncached(`${firstPrefix} continued.`),
);
expect(second(`${secondPrefix} continued.`)).toEqual(
splitMarkdownIntoBlocksUncached(`${secondPrefix} continued.`),
);
});
});
it('returns [] for empty content', () => {
expect(splitMarkdownIntoBlocks('')).toEqual([]);
});

View file

@ -101,6 +101,19 @@ const parseToMdast = (content: string): MdastNode =>
mdastExtensions: [gfmFromMarkdown(), directiveFromMarkdown(), mathFromMarkdown()],
}) as MdastNode;
type SplitResult = {
blocks: MarkdownBlock[];
/** Offset where the final two blocks start; the last boundary may still disappear. */
reparseStart: number;
/** Number of blocks before reparseStart, even after the tail merges into one block. */
stableBlockCount: number;
};
/** The parsed top-level nodes, returned when the text cannot be split. */
type WholeMessage = { children: MdastNode[] };
const isSplit = (result: SplitResult | WholeMessage): result is SplitResult => 'blocks' in result;
/**
* Where the line holding a top-level block begins, when only indentation precedes
* the block on it. mdast starts a node after its indentation, but that indentation
@ -114,63 +127,116 @@ const lineStartOf = (content: string, offset: number): number => {
};
/**
* Split a markdown string into its top-level blocks, returning the exact source
* slice for each block plus the index counts it consumes. Completed blocks
* produce byte-identical slices (and stable counts) across streamed updates,
* which is what makes per-block memoization effective: only the final, still-
* growing block changes from one token to the next.
*
* Inter-block whitespace (blank lines) is not part of any node's span and is
* dropped; block-level elements carry their own margins, so rendering each
* slice independently is visually equivalent to rendering the whole string.
* Parse `text` and split it into per-node blocks. Returns the parsed nodes
* instead when the text cannot be rendered block-by-block and must instead
* render as one whole:
* - reference/footnote definitions are document-scoped (and may be nested in
* a blockquote or list item), so a reference would otherwise render as
* literal text once severed from its definition;
* - top-level raw HTML blocks are escaped to text (rehypeRaw is not enabled),
* so the separator between adjacent HTML blocks would otherwise be dropped.
*/
export function splitMarkdownIntoBlocks(content: string): MarkdownBlock[] {
if (!content) {
return [];
}
const tree = parseToMdast(content);
const children = tree.children ?? [];
const splitBlocks = (text: string): SplitResult | WholeMessage => {
const children = parseToMdast(text).children ?? [];
if (children.length === 0) {
return [{ raw: content, codeBlockCount: 0, artifactCount: 0, mermaidCount: 0 }];
}
// Per-block rendering loses document-global context, so render the whole
// message as one block when it uses a construct that needs it:
// - reference/footnote definitions are document-scoped (and may be nested in
// a blockquote or list item), so a reference would otherwise render as
// literal text once severed from its definition; and
// - top-level raw HTML blocks are escaped to text (rehypeRaw is not enabled),
// so the separator between adjacent HTML blocks would otherwise be dropped.
const requiresWholeMessage = children.some(
(node) => node.type === 'html' || containsDefinition(node),
);
if (requiresWholeMessage) {
return [{ raw: content, ...blockCounts(children) }];
return { children };
}
const blocks: MarkdownBlock[] = [];
const stableBlockCount = Math.max(0, children.length - 2);
let reparseStart = 0;
for (const node of children) {
if (node.type === 'html' || containsDefinition(node)) {
return { children };
}
const start = node.position?.start?.offset;
const end = node.position?.end?.offset;
if (start == null || end == null) {
return [{ raw: content, ...blockCounts(children) }];
return { children };
}
const counts = { code: 0, artifact: 0, mermaid: 0 };
countWithin(node, counts);
const rawStart = lineStartOf(text, start);
if (blocks.length === stableBlockCount) {
reparseStart = rawStart;
}
blocks.push({
raw: content.slice(lineStartOf(content, start), end),
raw: text.slice(rawStart, end),
codeBlockCount: counts.code,
artifactCount: counts.artifact,
mermaidCount: counts.mermaid,
});
}
return blocks;
return { blocks, reparseStart, stableBlockCount };
};
/**
* Split a markdown string into its top-level blocks by parsing it in full,
* returning the exact source slice for each block plus the index counts it
* consumes. Prefer `splitMarkdownIntoBlocks`, which reuses the previous split
* when the content only grew.
*
* Inter-block whitespace (blank lines) is not part of any node's span and is
* dropped; block-level elements carry their own margins, so rendering each
* slice independently is visually equivalent to rendering the whole string.
*/
export function splitMarkdownIntoBlocksUncached(content: string): MarkdownBlock[] {
if (!content) {
return [];
}
const split = splitBlocks(content);
return isSplit(split) ? split.blocks : [{ raw: content, ...blockCounts(split.children) }];
}
type SplitCache = SplitResult & { content: string };
export type MarkdownSplitter = (content: string) => MarkdownBlock[];
/**
* Create an isolated splitter cache for one markdown surface. MarkdownBlocks
* keeps one instance per mounted message so concurrent streams cannot evict
* one another's active tail. The tail includes the preceding block because an
* unfinished marker (such as `***` or `#`) can stop interrupting it when more
* text arrives. Reusing that block before the boundary settles loses paragraph
* and lazy list/blockquote continuations.
*/
export function createMarkdownSplitter(): MarkdownSplitter {
let lastSplit: SplitCache | null = null;
return (content: string): MarkdownBlock[] => {
if (!content) {
lastSplit = null;
return [];
}
const prev = lastSplit;
if (prev != null && content.length > prev.content.length && content.startsWith(prev.content)) {
const tail = splitBlocks(content.slice(prev.reparseStart));
if (isSplit(tail)) {
lastSplit = {
content,
blocks: [...prev.blocks.slice(0, prev.stableBlockCount), ...tail.blocks],
reparseStart: prev.reparseStart + tail.reparseStart,
stableBlockCount: prev.stableBlockCount + tail.stableBlockCount,
};
return lastSplit.blocks;
}
}
const split = splitBlocks(content);
if (!isSplit(split)) {
lastSplit = null;
return [{ raw: content, ...blockCounts(split.children) }];
}
lastSplit = { content, ...split };
return split.blocks;
};
}
export const splitMarkdownIntoBlocks = createMarkdownSplitter();
const blockCounts = (
children: MdastNode[],
): { codeBlockCount: number; artifactCount: number; mermaidCount: number } => {

View file

@ -53,6 +53,7 @@ const ASK_USER_QUESTION_MARKER = 'E2E_ASK_USER_QUESTION:';
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
const STREAMING_MARKDOWN_REPLY_MARKER = 'E2E_STREAMING_MARKDOWN_REPLY';
const STATEFUL_CODE_MARKER = 'E2E_STATEFUL_CODE:';
/** Two prose paragraphs, so a spec can select the message's *closing* block. */
const PARAGRAPHS_REPLY_MARKER = 'E2E_PARAGRAPHS_REPLY';
@ -531,6 +532,28 @@ function replyResponses(text) {
],
};
}
if (text.includes(STREAMING_MARKDOWN_REPLY_MARKER)) {
return {
responses: [
[
'## E2E streaming markdown heading',
'',
'E2E streaming opening paragraph with 日本語 content.',
'',
'```javascript',
'const e2eIncrementalMarkdown = "complete";',
'```',
'',
'| E2E column | E2E value |',
'| --- | --- |',
'| completed block | visible |',
'',
'E2E streaming markdown final paragraph.',
].join('\n'),
],
sleep: SLOW_CHUNK_DELAY_MS,
};
}
if (text.includes(PARAGRAPHS_REPLY_MARKER)) {
/** The quoted cell sits in the first column, so scrolling the table to its

View file

@ -0,0 +1,76 @@
import { expect, test } from '@playwright/test';
import {
MOCK_ENDPOINTS,
NEW_CHAT_PATH,
messagesView,
selectMockEndpoint,
sendMessage,
} from '../helpers';
test.describe('streaming markdown blocks', () => {
test('preserves completed blocks while streaming and renders the final markdown @scenario:streamed-markdown-keeps-completed-blocks-and-final-content', async ({
page,
}) => {
test.setTimeout(90_000);
await page.goto(NEW_CHAT_PATH, { timeout: 10_000 });
await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
const response = await sendMessage(page, 'E2E_STREAMING_MARKDOWN_REPLY');
expect(response.ok()).toBeTruthy();
const assistantMessage = messagesView(page).locator('.message-render').last();
await expect(page.getByRole('button', { name: 'Stop generating' })).toBeVisible({
timeout: 30_000,
});
await expect(
assistantMessage.getByRole('heading', { name: 'E2E streaming markdown heading' }),
).toBeVisible({ timeout: 30_000 });
await expect(assistantMessage.getByText('E2E streaming markdown final paragraph.')).toBeVisible(
{ timeout: 60_000 },
);
await expect(
assistantMessage.locator('code').filter({ hasText: 'e2eIncrementalMarkdown' }),
).toBeVisible();
await expect(assistantMessage.getByRole('table')).toBeVisible();
await expect(assistantMessage).toContainText('日本語');
});
test('isolates markdown blocks across concurrent conversations @scenario:concurrent-streams-keep-message-blocks-isolated', async ({
page,
context,
}) => {
test.setTimeout(120_000);
const secondPage = await context.newPage();
try {
await Promise.all([
page.goto(NEW_CHAT_PATH, { timeout: 10_000 }),
secondPage.goto(NEW_CHAT_PATH, { timeout: 10_000 }),
]);
await Promise.all([
selectMockEndpoint(page, MOCK_ENDPOINTS[0]),
selectMockEndpoint(secondPage, MOCK_ENDPOINTS[0]),
]);
await Promise.all([
sendMessage(page, 'E2E_STREAMING_MARKDOWN_REPLY'),
sendMessage(secondPage, 'E2E_STREAMING_MARKDOWN_REPLY'),
]);
const firstAssistantMessage = messagesView(page).locator('.message-render').last();
const secondAssistantMessage = messagesView(secondPage).locator('.message-render').last();
await Promise.all([
expect(
firstAssistantMessage.getByText('E2E streaming markdown final paragraph.'),
).toBeVisible({ timeout: 90_000 }),
expect(
secondAssistantMessage.getByText('E2E streaming markdown final paragraph.'),
).toBeVisible({ timeout: 90_000 }),
]);
await expect(firstAssistantMessage).toContainText('日本語');
await expect(secondAssistantMessage).toContainText('日本語');
} finally {
await secondPage.close();
}
});
});