Add a batched draw path for runs of non-ASCII text

Draw runs of non-ASCII chars that each start a new cell of width one or
two in a tight loop, with the run invariant checks done once and state
kept in locals, falling back to the scalar path for anything that needs
special handling: combining chars, zero width chars, the image
placeholder, existing multicell cells, wrapping, insert mode and charset
translation.

Cells are written as 64 bit values built in registers, since assembling
them as structs makes compilers go through the stack with partial stores
that stall on store forwarding. The grapheme segmentation step is cached
as it depends only on the state and the segmentation property of the
char, which repeat in a run of text, breaking the chain of dependent
table lookups between consecutive chars.

This speeds up the unicode benchmark by ~30%, drawing CJK text by ~70%.

A differential test compares the batched and scalar paths on random
operations.
This commit is contained in:
Kovid Goyal 2026-11-06 15:27:47 +05:30
parent 598cc8e27d
commit 22cb56ea17
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
2 changed files with 217 additions and 0 deletions

View file

@ -1545,6 +1545,104 @@ draw_control_char(Screen *self, text_loop_state *s, uint32_t ch) {
}
}
static bool batched_non_ascii_draw_disabled_for_tests = false;
static_assert(sizeof(CPUCell) == 3 * sizeof(uint32_t), "draw_non_ascii_run() needs updating");
static inline uint64_t
pack_cell_words(uint32_t first, uint32_t second) {
// Two consecutive 32 bit words of cell memory as one 64 bit value
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return first | ((uint64_t)second << 32);
#else
return ((uint64_t)first << 32) | second;
#endif
}
// Not inlined, so that it gets its own register allocation rather than competing
// with the rest of draw_text_loop() for registers. It is called once per run.
__attribute__((noinline)) static size_t
draw_non_ascii_run(Screen *self, text_loop_state *s, const uint32_t *chars, size_t num_chars) {
// Batched fast path: draw a run of non-ASCII chars each of which starts a new cell
// of width one or two in the current line, keeping all state in locals so it is
// not reloaded for every char. Stops at the first char needing the scalar path:
// ASCII, invalid, zero width, a char that combines with the previous cell, the
// image placeholder, a char that does not fit in the line or an existing multicell
// cell. The caller must ensure there is no charset translation or insert mode and
// that the cursor is within the line. Returns the number of chars drawn.
const index_type columns = self->columns;
index_type x = self->cursor->x, last_x = x;
CPUCell *cp = s->cp;
GPUCell *gp = s->gp;
const GPUCell g = s->g;
// Cells are written from these templates as 64 bit values built in registers.
// The first word of a cell is its char. Assembling cells as CPUCell values or
// word arrays makes compilers build them on the stack with partial stores and
// reload them with wider loads, which stall on store forwarding.
uint32_t narrow[3], wide_first[3], wide_second[3];
{
const CPUCell wide = {.is_multicell = true, .width = 2, .scale = 1, .natural_width = true, .hyperlink_id = s->cc.hyperlink_id};
CPUCell second = wide;
second.x = 1;
memcpy(narrow, &s->cc, sizeof(narrow));
memcpy(wide_first, &wide, sizeof(wide_first));
memcpy(wide_second, &second, sizeof(wide_second));
}
GraphemeSegmentationResult seg = s->seg;
// The segmentation step depends only on the current state and the grapheme
// segmentation property of the char, and in a run of text the same pair
// repeats, so cache the last step. This breaks the chain of dependent table
// lookups that otherwise links every char to the previous one.
uint32_t cached_seg_key = UINT32_MAX;
GraphemeSegmentationResult cached_seg = {0};
size_t i = 0;
for (; i < num_chars; i++) {
const char_type ch = chars[i];
if (ch < DEL || ch == IMAGE_PLACEHOLDER_CHAR) break;
const CharProps props = char_props_for(ch);
if (props.is_invalid) break;
// Negative widths are East Asian ambiguous width chars, which are drawn as narrow
int width = wcwidth_std(props);
if (width < 0) width = 1;
if (width == 0 || x + (index_type)width > columns || cp[x].is_multicell) break;
if (width == 2 && cp[x + 1].is_multicell) break;
const uint32_t seg_key = ((uint32_t)seg.val << 8) | props.grapheme_segmentation_property;
if (seg_key != cached_seg_key) {
cached_seg = grapheme_segmentation_step(seg, props);
cached_seg_key = seg_key;
}
if (cached_seg.add_to_current_cell) break;
seg = cached_seg;
last_x = x;
const uint32_t ch_word = ch & 0x7fffffff;
uint8_t *dest = (uint8_t *)(cp + x);
if (width == 1) {
const uint64_t first = pack_cell_words(ch_word, narrow[1]);
memcpy(dest, &first, sizeof(first));
memcpy(dest + sizeof(first), &narrow[2], sizeof(narrow[2]));
gp[x++] = g;
} else {
const uint64_t first = pack_cell_words(ch_word, wide_first[1]), second = pack_cell_words(wide_first[2], ch_word),
third = pack_cell_words(wide_second[1], wide_second[2]);
memcpy(dest, &first, sizeof(first));
memcpy(dest + sizeof(first), &second, sizeof(second));
memcpy(dest + 2 * sizeof(first), &third, sizeof(third));
gp[x] = g;
gp[x + 1] = g;
x += 2;
}
}
if (i) {
self->cursor->x = x;
self->last_graphic_char = chars[i - 1];
s->seg = seg;
s->prev.y = self->cursor->y;
s->prev.x = last_x;
s->prev.cc = cp + last_x;
}
return i;
}
static void
draw_text_loop(Screen *self, const uint32_t *chars, size_t num_chars, text_loop_state *s) {
init_text_loop_line(self, s);
@ -1620,6 +1718,14 @@ draw_text_loop(Screen *self, const uint32_t *chars, size_t num_chars, text_loop_
char_width = 1;
s->seg = (GraphemeSegmentationResult){.grapheme_break = GBP_None};
} else {
if (ch > DEL && LIKELY(!self->charset.current && !self->modes.mIRM && self->cursor->x < self->columns) &&
!batched_non_ascii_draw_disabled_for_tests) {
const size_t n = draw_non_ascii_run(self, s, chars + i, num_chars - i);
if (n) {
i += n - 1;
continue;
}
}
CharProps cp = char_props_for(ch);
if (cp.is_invalid) {
if (ch < ' ') draw_control_char(self, s, ch);
@ -7832,11 +7938,18 @@ test_set_history_reuse(PyObject *self UNUSED, PyObject *val) {
Py_RETURN_NONE;
}
static PyObject *
test_set_batched_non_ascii_draw(PyObject *self UNUSED, PyObject *val) {
batched_non_ascii_draw_disabled_for_tests = !PyObject_IsTrue(val);
Py_RETURN_NONE;
}
static PyMethodDef module_methods[] = {
{"is_emoji_presentation_base", (PyCFunction)screen_is_emoji_presentation_base, METH_O, ""},
{"truncate_point_for_length", (PyCFunction)screen_truncate_point_for_length, METH_VARARGS, ""},
{"test_ch_and_idx", test_ch_and_idx, METH_O, ""},
{"test_set_history_reuse", test_set_history_reuse, METH_O, ""},
{"test_set_batched_non_ascii_draw", test_set_batched_non_ascii_draw, METH_O, ""},
{NULL} /* Sentinel */
};

View file

@ -930,6 +930,110 @@ class TestScreen(BaseTest):
finally:
test_set_history_reuse(True)
def test_batched_non_ascii_draw(self):
import random
from kitty.fast_data_types import test_set_batched_non_ascii_draw
texts = (
'日本語の文字列',
'éàüñ',
'αβγ',
'é',
'😀',
'👍🏽',
'👨‍👩‍👧',
'❤️',
'😀︎',
'\U0001f1fa\U0001f1f8',
'กำ',
'각',
'\U0010eeee̅',
'abc',
' ',
'̀',
'‍',
'\x85',
'\x9b',
'￿',
'\U000e0001',
)
def ops_for(seed, cols):
rng = random.Random(seed)
ops = []
for i in range(rng.randint(20, 100)):
r = rng.random()
if r < 0.5:
ops.append(('draw', ''.join(rng.choice(texts) for _ in range(rng.randint(1, 2 * cols)))))
elif r < 0.6:
ops.append(
(
'bytes',
rng.choice(
(
b'\x1b[1;31m',
b'\x1b[4:3;48;2;1;2;3m',
b'\x1b[m',
b'\x1b]8;;http://x\x1b\\',
b'\x1b]8;;\x1b\\',
)
),
)
)
elif r < 0.65:
ops.append(('bytes', rng.choice((b'\x1b[4h', b'\x1b[4l', b'\x1b(0', b'\x1b(B', b'\x1b[3b'))))
elif r < 0.75:
ops.append(('bytes', f'\x1b[{rng.randint(1, 3)}{rng.choice("CDAB")}'.encode()))
elif r < 0.8:
ops.append(('mc', rng.choice('A日'), rng.randint(2, 3)))
elif r < 0.85:
ops.append(('bytes', f'\x1b[{rng.randint(1, 4)};{rng.randint(1, cols)}H'.encode()))
else:
ops.append(('nl',))
return ops
def build(ops, cols, lines):
s = self.create_screen(cols=cols, lines=lines, scrollback=20)
for op in ops:
if op[0] == 'draw':
s.draw(op[1])
elif op[0] == 'bytes':
parse_bytes(s, op[1])
elif op[0] == 'mc':
draw_multicell(s, op[1], scale=op[2])
else:
s.carriage_return(), s.linefeed()
# makes last_graphic_char observable
parse_bytes(s, b'\x1b[4l\x1b(B\x1b[2b')
return s
def compare(a, b, msg):
self.ae((a.cursor.x, a.cursor.y), (b.cursor.x, b.cursor.y), msg)
self.ae(a.historybuf.count, b.historybuf.count, msg)
for i in range(a.historybuf.count):
self.assertTrue(a.historybuf.line(i) == b.historybuf.line(i), f'{msg} history line {i}')
self.ae(a.historybuf.line(i).as_ansi(), b.historybuf.line(i).as_ansi(), f'{msg} history line {i}')
for i in range(a.lines):
self.assertTrue(a.linebuf.line(i) == b.linebuf.line(i), f'{msg} line {i}')
self.ae(a.linebuf.line(i).as_ansi(), b.linebuf.line(i).as_ansi(), f'{msg} line {i}')
self.ae(a.cpu_cells(i), b.cpu_cells(i), f'{msg} line {i}')
self.ae(a.linebuf.is_continued(i), b.linebuf.is_continued(i), f'{msg} line {i}')
self.ae(a.linebuf.dirty_lines(), b.linebuf.dirty_lines(), msg)
try:
for seed in range(400):
rng = random.Random(seed)
cols, lines = rng.randint(1, 14), rng.randint(1, 6)
ops = ops_for(seed, cols)
test_set_batched_non_ascii_draw(True)
a = build(ops, cols, lines)
test_set_batched_non_ascii_draw(False)
b = build(ops, cols, lines)
compare(a, b, f'seed={seed} cols={cols} lines={lines}')
finally:
test_set_batched_non_ascii_draw(True)
def test_tab_stops(self):
# Taken from vttest/main.c
s = self.create_screen(cols=80, lines=2)