Use non-temporal stores to add lines to the scrollback

The history buffer is much larger than the CPU cache and lines added to
it are not read again soon. Normal stores to it stall acquiring
ownership of the destination cache lines, filling the store buffer and
stalling the drawing of the next line. Write the whole cache lines of
the cells with non-temporal stores instead, using normal stores for the
partial cache lines at either end, as partial non-temporal writes are
slow. Fence once per parse pass rather than after every line.

benchmark.py scrollback: +32%, in-process parsing to the main screen
+26% to +62% depending on the number of columns. Builds without SSE2
keep using prefetching.
This commit is contained in:
Kovid Goyal 2026-11-08 10:51:52 +05:30
parent 97709a90ad
commit 6daef03deb
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
3 changed files with 54 additions and 1 deletions

View file

@ -359,6 +359,40 @@ historybuf_push(HistoryBuf *self, ANSIBuf *as_ansi_buf, bool *needs_clear) {
return idx;
}
#ifdef __SSE2__
#include <emmintrin.h>
#endif
static void
copy_cells_to_history(CPUCell *dest, const CPUCell *src, index_type num) {
// The history buffer is typically much larger than the CPU cache and lines
// added to it are not read again soon, so the cells are written with non
// temporal stores where available. Normal stores to it stall acquiring
// ownership of the destination cache lines, filling the store buffer, and
// stalling the parsing and drawing of the next line, see benchmark.py.
// Non temporal stores are only fast when they write whole cache lines, so
// the partial cache lines at either end are written with normal stores.
uint8_t *d = (uint8_t *)dest;
const uint8_t *s = (const uint8_t *)src;
size_t n = (size_t)num * sizeof(CPUCell);
#ifdef __SSE2__
enum { CACHE_LINE = 64 };
const size_t head = (CACHE_LINE - ((uintptr_t)d & (CACHE_LINE - 1))) & (CACHE_LINE - 1);
if (n >= head + CACHE_LINE) {
memcpy(d, s, head);
d += head;
s += head;
n -= head;
// The stores are made visible to other threads by historybuf_finish_writes()
for (; n >= CACHE_LINE; n -= CACHE_LINE, d += CACHE_LINE, s += CACHE_LINE) {
for (unsigned i = 0; i < CACHE_LINE; i += 16) _mm_stream_si128((__m128i *)(d + i), _mm_loadu_si128((const __m128i *)(s + i)));
}
}
#endif
memcpy(d, s, n);
}
#ifndef __SSE2__
static void
prefetch_line_for_write(HistoryBuf *self, index_type y) {
// The history buffer is typically much larger than the CPU cache, so the
@ -371,18 +405,21 @@ prefetch_line_for_write(HistoryBuf *self, index_type y) {
const size_t cache_line_sz = 64;
for (size_t i = 0; i < self->xnum * sizeof(CPUCell); i += cache_line_sz) __builtin_prefetch(cpu + i, 1, 0);
}
#endif
void
historybuf_add_line(HistoryBuf *self, const Line *line, ANSIBuf *as_ansi_buf) {
bool needs_clear;
index_type idx = historybuf_push(self, as_ansi_buf, &needs_clear);
init_line(self, idx, self->line);
copy_line(line, self->line);
copy_cells_to_history(self->line->cpu_cells, line->cpu_cells, MIN(line->xnum, self->line->xnum));
LineAttrs *attrs = attrptr(self, idx);
*attrs = line->attrs;
// the render data of the line is not stored, so it must be rendered if displayed
attrs->has_dirty_text = true;
#ifndef __SSE2__
prefetch_line_for_write(self, (self->start_of_data + self->count) % self->ynum);
#endif
}
bool

View file

@ -47,6 +47,21 @@ typedef struct {
HistoryBuf *alloc_historybuf(unsigned int, unsigned int, unsigned int, TextCache *tc);
#ifdef __SSE2__
#include <emmintrin.h>
#endif
static inline void
historybuf_finish_writes(void) {
// Lines are added to the history buffer with non-temporal stores, which
// are weakly ordered. Order them before any later stores, so that they
// are visible to other threads once later stores, such as the unlocking
// of a mutex, are. Called once per parse pass, as fencing after every
// line is costly. The thread that added the lines always sees them.
#ifdef __SSE2__
_mm_sfence();
#endif
}
HistoryBuf *historybuf_alloc_for_rewrap(unsigned int columns, HistoryBuf *self);
void historybuf_finish_rewrap(HistoryBuf *dest, HistoryBuf *src);
void historybuf_fast_rewrap(HistoryBuf *dest, HistoryBuf *src, index_type count, bool steal);

View file

@ -1715,6 +1715,7 @@ run_worker(void *p, ParseData *pd, bool flush) {
if (compact_buffer(self)) write_space_created = true;
}
pd->has_pending_input = self->read.pos < self->read.sz;
historybuf_finish_writes();
}
end_with_lock;
if (write_space_created && pd->write_space_created) pd->write_space_created(pd->write_space_created_data);