mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-19 14:45:29 +00:00
Use non-temporal stores to add lines to the scrollback
The history buffer is much larger than the CPU cache and lines added to it are not read again soon. Normal stores to it stall acquiring ownership of the destination cache lines, filling the store buffer and stalling the drawing of the next line. Write the whole cache lines of the cells with non-temporal stores instead, using normal stores for the partial cache lines at either end, as partial non-temporal writes are slow. Fence once per parse pass rather than after every line. benchmark.py scrollback: +32%, in-process parsing to the main screen +26% to +62% depending on the number of columns. Builds without SSE2 keep using prefetching.
This commit is contained in:
parent
40d41ca4f5
commit
4b6783c24b
3 changed files with 54 additions and 1 deletions
|
|
@ -359,6 +359,40 @@ historybuf_push(HistoryBuf *self, ANSIBuf *as_ansi_buf, bool *needs_clear) {
|
|||
return idx;
|
||||
}
|
||||
|
||||
#ifdef __SSE2__
|
||||
#include <emmintrin.h>
|
||||
#endif
|
||||
|
||||
static void
|
||||
copy_cells_to_history(CPUCell *dest, const CPUCell *src, index_type num) {
|
||||
// The history buffer is typically much larger than the CPU cache and lines
|
||||
// added to it are not read again soon, so the cells are written with non
|
||||
// temporal stores where available. Normal stores to it stall acquiring
|
||||
// ownership of the destination cache lines, filling the store buffer, and
|
||||
// stalling the parsing and drawing of the next line, see benchmark.py.
|
||||
// Non temporal stores are only fast when they write whole cache lines, so
|
||||
// the partial cache lines at either end are written with normal stores.
|
||||
uint8_t *d = (uint8_t *)dest;
|
||||
const uint8_t *s = (const uint8_t *)src;
|
||||
size_t n = (size_t)num * sizeof(CPUCell);
|
||||
#ifdef __SSE2__
|
||||
enum { CACHE_LINE = 64 };
|
||||
const size_t head = (CACHE_LINE - ((uintptr_t)d & (CACHE_LINE - 1))) & (CACHE_LINE - 1);
|
||||
if (n >= head + CACHE_LINE) {
|
||||
memcpy(d, s, head);
|
||||
d += head;
|
||||
s += head;
|
||||
n -= head;
|
||||
// The stores are made visible to other threads by historybuf_finish_writes()
|
||||
for (; n >= CACHE_LINE; n -= CACHE_LINE, d += CACHE_LINE, s += CACHE_LINE) {
|
||||
for (unsigned i = 0; i < CACHE_LINE; i += 16) _mm_stream_si128((__m128i *)(d + i), _mm_loadu_si128((const __m128i *)(s + i)));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
memcpy(d, s, n);
|
||||
}
|
||||
|
||||
#ifndef __SSE2__
|
||||
static void
|
||||
prefetch_line_for_write(HistoryBuf *self, index_type y) {
|
||||
// The history buffer is typically much larger than the CPU cache, so the
|
||||
|
|
@ -371,18 +405,21 @@ prefetch_line_for_write(HistoryBuf *self, index_type y) {
|
|||
const size_t cache_line_sz = 64;
|
||||
for (size_t i = 0; i < self->xnum * sizeof(CPUCell); i += cache_line_sz) __builtin_prefetch(cpu + i, 1, 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
void
|
||||
historybuf_add_line(HistoryBuf *self, const Line *line, ANSIBuf *as_ansi_buf) {
|
||||
bool needs_clear;
|
||||
index_type idx = historybuf_push(self, as_ansi_buf, &needs_clear);
|
||||
init_line(self, idx, self->line);
|
||||
copy_line(line, self->line);
|
||||
copy_cells_to_history(self->line->cpu_cells, line->cpu_cells, MIN(line->xnum, self->line->xnum));
|
||||
LineAttrs *attrs = attrptr(self, idx);
|
||||
*attrs = line->attrs;
|
||||
// the render data of the line is not stored, so it must be rendered if displayed
|
||||
attrs->has_dirty_text = true;
|
||||
#ifndef __SSE2__
|
||||
prefetch_line_for_write(self, (self->start_of_data + self->count) % self->ynum);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
|
|
|
|||
|
|
@ -47,6 +47,21 @@ typedef struct {
|
|||
|
||||
|
||||
HistoryBuf *alloc_historybuf(unsigned int, unsigned int, unsigned int, TextCache *tc);
|
||||
|
||||
#ifdef __SSE2__
|
||||
#include <emmintrin.h>
|
||||
#endif
|
||||
static inline void
|
||||
historybuf_finish_writes(void) {
|
||||
// Lines are added to the history buffer with non-temporal stores, which
|
||||
// are weakly ordered. Order them before any later stores, so that they
|
||||
// are visible to other threads once later stores, such as the unlocking
|
||||
// of a mutex, are. Called once per parse pass, as fencing after every
|
||||
// line is costly. The thread that added the lines always sees them.
|
||||
#ifdef __SSE2__
|
||||
_mm_sfence();
|
||||
#endif
|
||||
}
|
||||
HistoryBuf *historybuf_alloc_for_rewrap(unsigned int columns, HistoryBuf *self);
|
||||
void historybuf_finish_rewrap(HistoryBuf *dest, HistoryBuf *src);
|
||||
void historybuf_fast_rewrap(HistoryBuf *dest, HistoryBuf *src, index_type count, bool steal);
|
||||
|
|
|
|||
|
|
@ -1715,6 +1715,7 @@ run_worker(void *p, ParseData *pd, bool flush) {
|
|||
if (compact_buffer(self)) write_space_created = true;
|
||||
}
|
||||
pd->has_pending_input = self->read.pos < self->read.sz;
|
||||
historybuf_finish_writes();
|
||||
}
|
||||
end_with_lock;
|
||||
if (write_space_created && pd->write_space_created) pd->write_space_created(pd->write_space_created_data);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue