diff --git a/kitty/data-types.c b/kitty/data-types.c
index 2775a441c..2c5d6d8e7 100644
--- a/kitty/data-types.c
+++ b/kitty/data-types.c
@@ -159,7 +159,6 @@ extern bool init_freetype_library(PyObject*);
 extern bool init_fontconfig_library(PyObject*);
 extern bool init_fonts(PyObject*);
 extern bool init_glfw(PyObject *m);
-extern bool init_sprites(PyObject *module);
 extern bool init_state(PyObject *module);
 extern bool init_keys(PyObject *module);
 extern bool init_graphics(PyObject *module);
@@ -189,7 +188,6 @@ PyInit_fast_data_types(void) {
         if (!init_ColorProfile(m)) return NULL;
         if (!init_Screen(m)) return NULL;
         if (!init_glfw(m)) return NULL;
-        if (!init_sprites(m)) return NULL;
         if (!init_state(m)) return NULL;
         if (!init_keys(m)) return NULL;
         if (!init_graphics(m)) return NULL;
diff --git a/kitty/data-types.h b/kitty/data-types.h
index ffb4cab24..b82bdf255 100644
--- a/kitty/data-types.h
+++ b/kitty/data-types.h
@@ -246,7 +246,6 @@ void cursor_reset(Cursor*);
 Cursor* cursor_copy(Cursor*);
 void cursor_copy_to(Cursor *src, Cursor *dest);
 void cursor_reset_display_attrs(Cursor*);
-void set_sprite_position(Cell *cell, Cell *previous_cell);
 
 double monotonic();
 PyObject* cm_thread_write(PyObject *self, PyObject *args);
diff --git a/kitty/fonts.c b/kitty/fonts.c
index cf9799a7a..68f0ed19f 100644
--- a/kitty/fonts.c
+++ b/kitty/fonts.c
@@ -10,50 +10,190 @@
 
 typedef uint16_t glyph_index;
 
-typedef struct {
+typedef struct SpritePosition SpritePosition;
+
+struct SpritePosition {
+    SpritePosition *next;
+    bool filled, rendered, is_second;
     sprite_index x, y, z;
-} SpriteIndex;
+    glyph_index glyph;
+    uint64_t extra_glyphs;
+};
+
+
+typedef struct {
+    size_t max_array_len, max_texture_size, max_y;
+    unsigned int x, y, z, xnum, ynum;
+} GPUSpriteTracker;
+
+
+static GPUSpriteTracker sprite_tracker = {
+    .max_array_len = 1000,
+    .max_texture_size = 1000,
+    .max_y = 100,
+};
 
 
 typedef struct {
     PyObject *face;
     hb_font_t *hb_font;
-    // Map glyph ids to sprite map co-ords
-    SpriteIndex *sprite_map; 
+    // Map glyphs to sprite map co-ords
+    SpritePosition sprite_map[1024]; 
     bool bold, italic;
 } Font;
 
-static Font medium_font = {0}, bold_font = {0}, italic_font = {0}, bi_font = {0}, box_font = {0}, missing_font = {0}, blank_font = {0};
-static Font fallback_fonts[256] = {{0}};
-static PyObject *get_fallback_font = NULL;
+
+static inline void 
+sprite_map_set_error(int error) {
+    switch(error) {
+        case 1:
+            PyErr_NoMemory(); break;
+        case 2:
+            PyErr_SetString(PyExc_RuntimeError, "Out of texture space for sprites"); break;
+        default:
+            PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while allocating sprites"); break;
+    }
+}
+
+void
+sprite_tracker_set_limits(size_t max_texture_size, size_t max_array_len) {
+    sprite_tracker.max_texture_size = max_texture_size;
+    sprite_tracker.max_array_len = max_array_len;
+}
+
+static inline void
+do_increment(int *error) {
+    sprite_tracker.x++;
+    if (sprite_tracker.x >= sprite_tracker.xnum) {
+        sprite_tracker.x = 0; sprite_tracker.y++;
+        sprite_tracker.ynum = MIN(MAX(sprite_tracker.ynum, sprite_tracker.y + 1), sprite_tracker.max_y);
+        if (sprite_tracker.y >= sprite_tracker.max_y) {
+            sprite_tracker.y = 0; sprite_tracker.z++;
+            if (sprite_tracker.z >= MIN(UINT16_MAX, sprite_tracker.max_array_len)) *error = 2;
+        }
+    }
+}
+
+
+SpritePosition*
+sprite_position_for(Font *font, glyph_index glyph, uint64_t extra_glyphs, bool is_second, int *error) {
+    glyph_index idx = glyph & 0x3ff;
+    SpritePosition *s = font->sprite_map + idx;
+    // Optimize for the common case of glyph under 1024 already in the cache
+    if (LIKELY(s->glyph == glyph && s->filled && s->extra_glyphs == extra_glyphs && s->is_second == is_second)) return s;  // Cache hit
+    while(true) {
+        if (s->filled) {
+            if (s->glyph == glyph && s->extra_glyphs == extra_glyphs && s->is_second == is_second) return s;  // Cache hit
+        } else {
+            break;
+        }
+        if (!s->next) {
+            s->next = calloc(1, sizeof(SpritePosition));
+            if (s->next == NULL) { *error = 1; return NULL; }
+        }
+        s = s->next;
+    }
+    s->glyph = glyph;
+    s->extra_glyphs = extra_glyphs;
+    s->is_second = is_second;
+    s->filled = true;
+    s->rendered = false;
+    s->x = sprite_tracker.x; s->y = sprite_tracker.y; s->z = sprite_tracker.z;
+    do_increment(error);
+    return s;
+}
+
+void
+sprite_tracker_current_layout(unsigned int *x, unsigned int *y, unsigned int *z) {
+    *x = sprite_tracker.xnum; *y = sprite_tracker.ynum; *z = sprite_tracker.z;
+}
+
+int 
+sprite_tracker_increment(sprite_index *x, sprite_index *y, sprite_index *z) {
+    int error = 0;
+    *x = sprite_tracker.x; *y = sprite_tracker.y; *z = sprite_tracker.z;
+    do_increment(&error);
+    return error;
+}
+
+void
+sprite_map_free(Font *font) {
+    SpritePosition *s, *t;
+    for (size_t i = 0; i < sizeof(font->sprite_map)/sizeof(font->sprite_map[0]); i++) {
+        s = font->sprite_map + i;
+        s = s->next;
+        while (s) {
+            t = s;
+            s = s->next;
+            free(t);
+        }
+    }
+}
+
+void
+clear_sprite_map(Font *font) {
+#define CLEAR(s) s->filled = false; s->rendered = false; s->glyph = 0; s->extra_glyphs = 0; s->x = 0; s->y = 0; s->z = 0; s->is_second = false;
+    SpritePosition *s;
+    for (size_t i = 0; i < sizeof(font->sprite_map)/sizeof(font->sprite_map[0]); i++) {
+        s = font->sprite_map + i;
+        CLEAR(s);
+        while ((s = s->next)) {
+            CLEAR(s);
+        }
+    }
+#undef CLEAR
+}
+
+void
+sprite_tracker_set_layout(unsigned int cell_width, unsigned int cell_height) {
+    sprite_tracker.xnum = MIN(MAX(1, sprite_tracker.max_texture_size / cell_width), UINT16_MAX);
+    sprite_tracker.max_y = MIN(MAX(1, sprite_tracker.max_texture_size / cell_height), UINT16_MAX);
+    sprite_tracker.ynum = 1;
+    sprite_tracker.x = 0; sprite_tracker.y = 0; sprite_tracker.z = 0;
+}
+
 
 static inline bool
 alloc_font(Font *f, PyObject *face, bool bold, bool italic) {
-    f->sprite_map = calloc(1 << (sizeof(glyph_index) * 8), sizeof(SpriteIndex));
-    if (f->sprite_map == NULL) return false;
     f->face = face; Py_INCREF(face);
     f->hb_font = harfbuzz_font_for_face(face);
+    if (f->hb_font == NULL) return false;
     f->bold = bold; f->italic = italic;
     return true;
 }
 
 static inline void
-clear_font(Font *f) { 
-    Py_CLEAR(f->face); 
-    free(f->sprite_map); f->sprite_map = NULL;
+free_font(Font *f) { 
     f->hb_font = NULL;
+    Py_CLEAR(f->face); 
+    sprite_map_free(f);
     f->bold = false; f->italic = false;
 }
 
+static Font medium_font = {0}, bold_font = {0}, italic_font = {0}, bi_font = {0}, box_font = {0}, missing_font = {0}, blank_font = {0};
+static Font fallback_fonts[256] = {{0}};
+static PyObject *get_fallback_font = NULL;
+
+typedef struct {
+    char_type left, right;
+    size_t font_idx;
+} SymbolMap;
+static SymbolMap* symbol_maps = NULL;
+static Font *symbol_map_fonts = NULL;
+static size_t symbol_maps_count = 0, symbol_map_fonts_count = 0;
+
 static unsigned int cell_width = 0, cell_height = 0, baseline = 0, underline_position = 0, underline_thickness = 0;
 
 static inline PyObject*
 update_cell_metrics(float pt_sz, float xdpi, float ydpi) {
-#define CALL(f) { if ((f)->face && !set_size_for_face((f)->face, pt_sz, xdpi, ydpi)) return NULL; }
+#define CALL(f) { if ((f)->face) { if(!set_size_for_face((f)->face, pt_sz, xdpi, ydpi)) return NULL; clear_sprite_map(f); } }
     CALL(&medium_font); CALL(&bold_font); CALL(&italic_font); CALL(&bi_font);
     for (size_t i = 0; fallback_fonts[i].face != NULL; i++)  {
         CALL(fallback_fonts + i);
     }
+    for (size_t i = 0; i < symbol_map_fonts_count; i++)  {
+        CALL(symbol_map_fonts + i);
+    }
 #undef CALL
     cell_metrics(medium_font.face, &cell_width, &cell_height, &baseline, &underline_position, &underline_thickness);
     if (!cell_width) { PyErr_SetString(PyExc_ValueError, "Failed to calculate cell width for the specified font."); return NULL; }
@@ -62,6 +202,7 @@ update_cell_metrics(float pt_sz, float xdpi, float ydpi) {
     if (cell_height < 4) { PyErr_SetString(PyExc_ValueError, "line height too small after adjustment"); return NULL; }
     if (cell_height > 1000) { PyErr_SetString(PyExc_ValueError, "line height too large after adjustment"); return NULL; }
     underline_position = MIN(cell_height - 1, underline_position);
+    sprite_tracker_set_layout(cell_width, cell_height);
     return Py_BuildValue("IIIII", cell_width, cell_height, baseline, underline_position, underline_thickness);
 }
 
@@ -105,14 +246,6 @@ fallback_font(Cell *cell) {
     return fallback_fonts + i;
 }
 
-typedef struct {
-    char_type left, right;
-    size_t font_idx;
-} SymbolMap;
-static SymbolMap* symbol_maps = NULL;
-static Font *symbol_map_fonts = NULL;
-static size_t symbol_maps_count = 0, symbol_map_fonts_count = 0;
-
 static inline Font*
 in_symbol_maps(char_type ch) {
     for (size_t i = 0; i < symbol_maps_count; i++) {
@@ -190,18 +323,51 @@ set_font(PyObject UNUSED *m, PyObject *args) {
     return update_cell_metrics(pt_sz, xdpi, ydpi);
 }
 
+static hb_buffer_t *harfbuzz_buffer = NULL;
+
 static void
 finalize(void) {
     Py_CLEAR(get_fallback_font);
-    clear_font(&medium_font); clear_font(&bold_font); clear_font(&italic_font); clear_font(&bi_font);
-    for (size_t i = 0; fallback_fonts[i].face != NULL; i++)  clear_font(fallback_fonts + i);
-    for (size_t i = 0; symbol_map_fonts_count; i++) clear_font(symbol_map_fonts + i);
+    free_font(&medium_font); free_font(&bold_font); free_font(&italic_font); free_font(&bi_font);
+    for (size_t i = 0; fallback_fonts[i].face != NULL; i++)  free_font(fallback_fonts + i);
+    for (size_t i = 0; symbol_map_fonts_count; i++) free_font(symbol_map_fonts + i);
     free(symbol_maps); free(symbol_map_fonts);
+    if (harfbuzz_buffer) hb_buffer_destroy(harfbuzz_buffer);
+}
+
+static PyObject*
+sprite_map_set_limits(PyObject UNUSED *self, PyObject *args) {
+    unsigned int w, h;
+    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
+    sprite_tracker_set_limits(w, h);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+sprite_map_set_layout(PyObject UNUSED *self, PyObject *args) {
+    unsigned int w, h;
+    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
+    sprite_tracker_set_layout(w, h);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+test_sprite_position_for(PyObject UNUSED *self, PyObject *args) {
+    glyph_index glyph;
+    uint64_t extra_glyphs = 0;
+    if (!PyArg_ParseTuple(args, "H|I", &glyph, &extra_glyphs)) return NULL;
+    int error;
+    SpritePosition *pos = sprite_position_for(&box_font, glyph, extra_glyphs, false, &error);
+    if (pos == NULL) { sprite_map_set_error(error); return NULL; }
+    return Py_BuildValue("III", pos->x, pos->y, pos->z);
 }
 
 static PyMethodDef module_methods[] = {
     METHODB(set_font_size, METH_VARARGS),
     METHODB(set_font, METH_VARARGS),
+    METHODB(sprite_map_set_limits, METH_VARARGS),
+    METHODB(sprite_map_set_layout, METH_VARARGS),
+    METHODB(test_sprite_position_for, METH_VARARGS),
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
@@ -211,6 +377,8 @@ init_fonts(PyObject *module) {
         PyErr_SetString(PyExc_RuntimeError, "Failed to register the fonts module at exit handler");
         return false;
     }
+    harfbuzz_buffer = hb_buffer_create();
+    if (harfbuzz_buffer == NULL || !hb_buffer_allocation_successful(harfbuzz_buffer) || !hb_buffer_pre_allocate(harfbuzz_buffer, 2000)) { PyErr_NoMemory(); return false; }
     if (PyModule_AddFunctions(module, module_methods) != 0) return false;
     return true;
 }
diff --git a/kitty/fonts.h b/kitty/fonts.h
index b93283919..0ea2d0152 100644
--- a/kitty/fonts.h
+++ b/kitty/fonts.h
@@ -18,3 +18,5 @@ bool face_has_codepoint(PyObject *, char_type);
 hb_font_t* harfbuzz_font_for_face(PyObject*);
 bool set_size_for_face(PyObject*, float, float, float);
 void cell_metrics(PyObject*, unsigned int*, unsigned int*, unsigned int*, unsigned int*, unsigned int*); 
+void sprite_tracker_current_layout(unsigned int *x, unsigned int *y, unsigned int *z);
+int sprite_tracker_increment(sprite_index *x, sprite_index *y, sprite_index *z);
diff --git a/kitty/gl.h b/kitty/gl.h
index ae2fbae26..0ce223bcc 100644
--- a/kitty/gl.h
+++ b/kitty/gl.h
@@ -9,7 +9,6 @@
 #include "gl-wrapper.h"
 #include "state.h"
 #include "screen.h"
-#include "sprites.h"
 #include <string.h>
 #include <stddef.h>
 #include <GLFW/glfw3.h>
diff --git a/kitty/history.c b/kitty/history.c
index 1e716bfc5..f3d769466 100644
--- a/kitty/history.c
+++ b/kitty/history.c
@@ -58,13 +58,6 @@ dealloc(HistoryBuf* self) {
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
-void 
-historybuf_refresh_sprite_positions(HistoryBuf *self) {
-    for (index_type i = 0; i < self->ynum; i++) {
-        update_sprites_in_line(lineptr(self, i), self->xnum);
-    }
-}
-
 static inline index_type 
 index_of(HistoryBuf *self, index_type lnum) {
     // The index (buffer position) of the line with line number lnum
diff --git a/kitty/line-buf.c b/kitty/line-buf.c
index 79f79d1c2..af7e2b675 100644
--- a/kitty/line-buf.c
+++ b/kitty/line-buf.c
@@ -363,13 +363,6 @@ as_ansi(LineBuf *self, PyObject *callback) {
     Py_RETURN_NONE;
 }
 
-void 
-linebuf_refresh_sprite_positions(LineBuf *self) {
-    for (index_type i = 0; i < self->ynum; i++) {
-        update_sprites_in_line(lineptr(self, i), self->xnum);
-    }
-}
-
 static PyObject*
 __str__(LineBuf *self) {
     PyObject *lines = PyTuple_New(self->ynum);
diff --git a/kitty/line.c b/kitty/line.c
index 0ed16c284..c6629ab4a 100644
--- a/kitty/line.c
+++ b/kitty/line.c
@@ -324,15 +324,12 @@ width(Line *self, PyObject *val) {
     return PyLong_FromUnsignedLong((unsigned long) (attrs & WIDTH_MASK));
 }
 
-#define set_sprite_position_at(x) set_sprite_position(self->cells + x, x == 0 ? NULL : self->cells + x - 1);
-
 void 
 line_add_combining_char(Line *self, uint32_t ch, unsigned int x) {
     if (!self->cells[x].ch) return;  // dont allow adding combining chars to a null cell
     combining_type c = self->cells[x].cc;
     if (c & CC_MASK) self->cells[x].cc = (c & CC_MASK) | ( (ch & CC_MASK) << CC_SHIFT );
     else self->cells[x].cc = ch & CC_MASK;
-    set_sprite_position_at(x);
 }
 
 static PyObject*
@@ -383,7 +380,6 @@ set_text(Line* self, PyObject *args) {
         self->cells[i].bg = bg;
         self->cells[i].decoration_fg = dfg;
         self->cells[i].cc = 0;
-        set_sprite_position_at(i);
     }
 
     Py_RETURN_NONE;
@@ -416,13 +412,12 @@ line_clear_text(Line *self, unsigned int at, unsigned int num, char_type ch) {
 #define PREFIX \
     for (index_type i = at; i < MIN(self->xnum, at + num); i++) { \
         self->cells[i].ch = ch; self->cells[i].cc = 0; \
-        self->cells[i].attrs = (self->cells[i].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width;
+        self->cells[i].attrs = (self->cells[i].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width; \
+    }
     if (CHAR_IS_BLANK(ch)) {
         PREFIX
-        clear_sprite_position(self->cells[i]); }
     } else {
         PREFIX
-        set_sprite_position_at(i)}
     }
 }
 
@@ -452,7 +447,6 @@ line_apply_cursor(Line *self, Cursor *cursor, unsigned int at, unsigned int num,
         } else {
             attrs_type w = self->cells[i].attrs & WIDTH_MASK;
             self->cells[i].attrs = attrs | w;
-            set_sprite_position_at(i);
         }
         self->cells[i].fg = fg; self->cells[i].bg = bg;
         self->cells[i].decoration_fg = dfg;
@@ -512,7 +506,7 @@ left_shift(Line *self, PyObject *args) {
 }
  
 void 
-line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Cursor *cursor, bool is_second) {
+line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Cursor *cursor, bool UNUSED is_second) {
     if (cursor == NULL) {
         self->cells[at].attrs = (self->cells[at].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width;
     } else {
@@ -523,8 +517,6 @@ line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Curs
     }
     self->cells[at].ch = ch;
     self->cells[at].cc = 0;
-    if (!is_second && CHAR_IS_BLANK(ch)) { clear_sprite_position(self->cells[at]); }
-    else set_sprite_position_at(at);
 }
 
 static PyObject*
diff --git a/kitty/lineops.h b/kitty/lineops.h
index e0fea2403..546b9f644 100644
--- a/kitty/lineops.h
+++ b/kitty/lineops.h
@@ -8,18 +8,6 @@
 
 #include "data-types.h"
 
-static inline void
-update_sprites_in_line(Cell *cells, index_type xnum) {
-    if (LIKELY(xnum > 0)) {
-        if (CHAR_IS_BLANK(cells->ch)) { clear_sprite_position(cells[0]); }
-        else set_sprite_position(cells, NULL);
-        for (index_type i = 1; i < xnum; i++) {
-            if (CHAR_IS_BLANK(cells[i].ch)) { clear_sprite_position(cells[i]); }
-            else set_sprite_position(cells + i, cells + i - 1);
-        }
-    }
-}
-
 static inline void
 set_attribute_on_line(Cell *cells, uint32_t shift, uint32_t val, index_type xnum) {
     // Set a single attribute on all cells in the line
@@ -27,7 +15,6 @@ set_attribute_on_line(Cell *cells, uint32_t shift, uint32_t val, index_type xnum
     attrs_type aval = (val & mask) << shift; 
     mask = ~(mask << shift);
     for (index_type i = 0; i < xnum; i++) cells[i].attrs = (cells[i].attrs & mask) | aval;
-    if (shift == BOLD_SHIFT || shift == ITALIC_SHIFT) update_sprites_in_line(cells, xnum);
 }
 
 static inline void
diff --git a/kitty/screen.c b/kitty/screen.c
index 35697edae..fc408b538 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -188,13 +188,6 @@ screen_rescale_images(Screen *self, unsigned int old_cell_width, unsigned int ol
     grman_rescale(self->alt_grman, old_cell_width, old_cell_height);
 }
 
-static void
-screen_refresh_sprite_positions(Screen *self) {
-    linebuf_refresh_sprite_positions(self->main_linebuf);
-    linebuf_refresh_sprite_positions(self->alt_linebuf);
-    historybuf_refresh_sprite_positions(self->historybuf);
-}
-
 
 static bool 
 screen_change_scrollback_size(Screen *self, unsigned int size) {
@@ -1356,7 +1349,6 @@ cursor_up(Screen *self, PyObject *args) {
 
 WRAP0x(index)
 WRAP0(reverse_index)
-WRAP0(refresh_sprite_positions)
 WRAP0(reset)
 WRAP0(set_tab_stop)
 WRAP1(clear_tab_stop, 0)
@@ -1575,7 +1567,6 @@ static PyMethodDef methods[] = {
     MND(set_tab_stop, METH_NOARGS)
     MND(clear_tab_stop, METH_VARARGS)
     MND(reverse_index, METH_NOARGS)
-    MND(refresh_sprite_positions, METH_NOARGS)
     MND(mark_as_dirty, METH_NOARGS)
     MND(resize, METH_VARARGS)
     MND(set_margins, METH_VARARGS)
diff --git a/kitty/shaders.c b/kitty/shaders.c
index 60a0254d1..86545e23b 100644
--- a/kitty/shaders.c
+++ b/kitty/shaders.c
@@ -6,6 +6,7 @@
  */
 
 #include "gl.h"
+#include "fonts.h"
 #include <sys/sysctl.h>
 
 enum { CELL_PROGRAM, CELL_BACKGROUND_PROGRAM, CELL_SPECIAL_PROGRAM, CELL_FOREGROUND_PROGRAM, CURSOR_PROGRAM, BORDERS_PROGRAM, GRAPHICS_PROGRAM, NUM_PROGRAMS };
@@ -61,7 +62,7 @@ realloc_sprite_texture() {
     glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 
     unsigned int xnum, ynum, z, znum, width, height, src_ynum;
-    sprite_map_current_layout(&xnum, &ynum, &z);
+    sprite_tracker_current_layout(&xnum, &ynum, &z);
     znum = z + 1;
     width = xnum * sprite_map.cell_width; height = ynum * sprite_map.cell_height;
     glTexStorage3D(GL_TEXTURE_2D_ARRAY, 1, GL_R8, width, height, znum); 
@@ -100,7 +101,7 @@ ensure_sprite_map() {
 static void 
 sprite_send_to_gpu(unsigned int x, unsigned int y, unsigned int z, PyObject *buf) {
     unsigned int xnum, ynum, znum;
-    sprite_map_current_layout(&xnum, &ynum, &znum);
+    sprite_tracker_current_layout(&xnum, &ynum, &znum);
     if ((int)znum >= sprite_map.last_num_of_layers || (znum == 0 && (int)ynum > sprite_map.last_ynum)) realloc_sprite_texture();
     glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); 
     glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 
@@ -113,13 +114,6 @@ sprite_send_to_gpu(unsigned int x, unsigned int y, unsigned int z, PyObject *buf
     Py_DECREF(buf);
 }
 
-static void 
-render_and_send_dirty_sprites(PyObject *text, bool bold, bool italic, bool is_second, sprite_index x, sprite_index y, sprite_index z) {
-    if (text == NULL) { fatal("The text for a sprite was NULL, probably out of memory."); }
-    PyObject *buf = render_cell(text, bold, italic, false, false, is_second);
-    sprite_send_to_gpu(x, y, z, buf);
-}
-
 static inline sprite_index
 send_prerendered(unsigned int underline, bool strikethrough) {
     sprite_index x, y, z;
@@ -127,7 +121,7 @@ send_prerendered(unsigned int underline, bool strikethrough) {
     if (blank == NULL) { fatal("Out of memory"); }
     PyObject *buf = render_cell(blank, false, false, underline, strikethrough, false);
     Py_CLEAR(blank);
-    if (sprite_map_increment(&x, &y, &z) != 0) { fatal("Failed to increment sprite map for prerendering"); }
+    if (sprite_tracker_increment(&x, &y, &z) != 0) { fatal("Failed to increment sprite map for prerendering"); }
     sprite_send_to_gpu(x, y, z, buf);
     return x;
 }
@@ -141,9 +135,9 @@ layout_sprite_map(unsigned int cell_width, unsigned int cell_height, PyObject *r
     if (sprite_map.max_texture_size == 0) {
         glGetIntegerv(GL_MAX_TEXTURE_SIZE, &(sprite_map.max_texture_size)); 
         glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &(sprite_map.max_array_texture_layers)); 
-        sprite_map_set_limits(sprite_map.max_texture_size, sprite_map.max_array_texture_layers);
+        /* sprite_map_set_limits(sprite_map.max_texture_size, sprite_map.max_array_texture_layers); */
     }
-    sprite_map_set_layout(sprite_map.cell_width, sprite_map.cell_height);
+    /* sprite_map_set_layout(sprite_map.cell_width, sprite_map.cell_height); */
     Py_CLEAR(sprite_map.render_cell);
     sprite_map.render_cell = render_cell; Py_INCREF(sprite_map.render_cell);
     if (sprite_map.texture_id) { glDeleteTextures(1, &(sprite_map.texture_id)); sprite_map.texture_id = 0; }
@@ -157,7 +151,7 @@ layout_sprite_map(unsigned int cell_width, unsigned int cell_height, PyObject *r
 
 static void
 destroy_sprite_map() {
-    sprite_map_free();
+    /* sprite_map_free(); */
     Py_CLEAR(sprite_map.render_cell);
     if (sprite_map.texture_id) {
         glDeleteTextures(1, &(sprite_map.texture_id));
@@ -259,7 +253,7 @@ cell_update_uniform_block(ssize_t vao_idx, Screen *screen, int uniform_buffer, G
     
     rd->xstart = xstart; rd->ystart = ystart; rd->dx = dx; rd->dy = dy;
     unsigned int x, y, z;
-    sprite_map_current_layout(&x, &y, &z);
+    sprite_tracker_current_layout(&x, &y, &z);
     rd->sprite_dx = 1.0f / (float)x; rd->sprite_dy = 1.0f / (float)y;
     rd->color1 = inverted & 1; rd->color2 = 1 - (inverted & 1);
 
@@ -300,7 +294,7 @@ cell_prepare_to_render(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen, GLfloa
     cell_update_uniform_block(vao_idx, screen, uniform_buffer, xstart, ystart, dx, dy, cursor);
 
     ensure_sprite_map();
-    render_dirty_sprites(render_and_send_dirty_sprites);
+    /* render_dirty_sprites(render_and_send_dirty_sprites); */
 
     bind_vao_uniform_buffer(vao_idx, uniform_buffer, cell_program_layouts[CELL_PROGRAM].render_data.index);
     bind_vertex_array(vao_idx);
diff --git a/kitty/sprites.c b/kitty/sprites.c
deleted file mode 100644
index 95ff15b0d..000000000
--- a/kitty/sprites.c
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * sprites.c
- * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "data-types.h"
-#include "lineops.h"
-#include <structmember.h>
-
-typedef struct SpritePosition SpritePosition;
-struct SpritePosition {
-    SpritePosition *next;
-    sprite_index x, y, z;
-    char_type ch;
-    combining_type cc;
-    bool bold, italic;
-    bool is_second;
-    bool filled;
-    bool rendered;
-};
-
-typedef struct {
-    size_t max_array_len, max_texture_size, max_y;
-    unsigned int x, y, z, xnum, ynum;
-    SpritePosition cache[1024];
-    bool dirty;
-} SpriteMap;
-
-static SpriteMap sprite_map = {
-    .max_array_len = 1000,
-    .max_texture_size = 1000,
-    .max_y = 100,
-    .dirty = true
-};
-
-static inline void 
-sprite_map_set_error(int error) {
-    switch(error) {
-        case 1:
-            PyErr_NoMemory(); break;
-        case 2:
-            PyErr_SetString(PyExc_RuntimeError, "Out of texture space for sprites"); break;
-        default:
-            PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while allocating sprites"); break;
-    }
-}
-
-void
-sprite_map_set_limits(size_t max_texture_size, size_t max_array_len) {
-    sprite_map.max_texture_size = max_texture_size;
-    sprite_map.max_array_len = max_array_len;
-}
-
-static PyObject*
-sprite_map_set_limits_py(PyObject UNUSED *self, PyObject *args) {
-    unsigned int w, h;
-    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
-    sprite_map_set_limits(w, h);
-    Py_RETURN_NONE;
-}
-
-void
-sprite_map_free() {
-    SpritePosition *s, *t;
-    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
-        s = &(sprite_map.cache[i]);
-        s = s->next;
-        while (s) {
-            t = s;
-            s = s->next;
-            PyMem_Free(t);
-        }
-    }
-}
-
-static inline void
-do_increment(int *error) {
-    sprite_map.x++;
-    if (sprite_map.x >= sprite_map.xnum) {
-        sprite_map.x = 0; sprite_map.y++;
-        sprite_map.ynum = MIN(MAX(sprite_map.ynum, sprite_map.y + 1), sprite_map.max_y);
-        if (sprite_map.y >= sprite_map.max_y) {
-            sprite_map.y = 0; sprite_map.z++;
-            if (sprite_map.z >= MIN(UINT16_MAX, sprite_map.max_array_len)) *error = 2;
-        }
-    }
-}
-
-SpritePosition*
-sprite_map_position_for(char_type ch, attrs_type attrs, combining_type cc, bool is_second, int *error) {
-    uint8_t bold_italic = (attrs >> BOLD_SHIFT) & 3;
-    unsigned int idx = (ch & 0xFF) | (bold_italic << 8); // Only bold italic bits and lowest byte of char
-    SpritePosition *s = &(sprite_map.cache[idx]);
-    // Optimize for the common case of an ASCII char already in the cache
-    if (LIKELY(s->ch == ch && s->filled && s->cc == cc && s->is_second == is_second)) return s;  // Cache hit
-    while(true) {
-        if (s->filled) {
-            if (s->ch == ch && s->cc == cc && s->is_second == is_second) return s;  // Cache hit
-        } else {
-            break;
-        }
-        if (!s->next) {
-            s->next = PyMem_Calloc(1, sizeof(SpritePosition));
-            if (s->next == NULL) { *error = 1; return NULL; }
-        }
-        s = s->next;
-    }
-    s->ch = ch;
-    s->cc = cc;
-    s->is_second = is_second;
-    s->filled = true;
-    s->rendered = false;
-    s->bold = bold_italic & 1;
-    s->italic = bold_italic >> 1;
-    s->x = sprite_map.x; s->y = sprite_map.y; s->z = sprite_map.z;
-    do_increment(error);
-    sprite_map.dirty = true;
-    return s;
-}
-
-
-void
-set_sprite_position(Cell *cell, Cell *previous_cell) {
-    SpritePosition *sp;
-    static int error;
-    if (UNLIKELY(previous_cell != NULL && ((previous_cell->attrs) & WIDTH_MASK) == 2)) {
-        sp = sprite_map_position_for(previous_cell->ch, previous_cell->attrs, 0, true, &error);
-    } else {
-        sp = sprite_map_position_for(cell->ch, cell->attrs, cell->cc, false, &error);
-    }
-    cell->sprite_x = sp->x;
-    cell->sprite_y = sp->y;
-    cell->sprite_z = sp->z;
-}
-
-int 
-sprite_map_increment(sprite_index *x, sprite_index *y, sprite_index *z) {
-    int error = 0;
-    *x = sprite_map.x; *y = sprite_map.y; *z = sprite_map.z;
-    do_increment(&error);
-    return error;
-}
-
-void
-sprite_map_set_layout(unsigned int cell_width, unsigned int cell_height) {
-    // Invalidate cache since cell size has changed.
-    SpritePosition *s;
-    sprite_map.xnum = MIN(MAX(1, sprite_map.max_texture_size / cell_width), UINT16_MAX);
-    sprite_map.max_y = MIN(MAX(1, sprite_map.max_texture_size / cell_height), UINT16_MAX);
-    sprite_map.ynum = 1;
-    sprite_map.x = 0; sprite_map.y = 0; sprite_map.z = 0;
-
-    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
-        s = &(sprite_map.cache[i]);
-        do {
-            s->filled = false;
-            s->is_second = false;
-            s->rendered = false;
-            s->ch = 0; s->cc = 0;
-            s->x = 0; s->y = 0; s->z = 0;
-            s = s->next;
-        } while (s != NULL);
-    }
-    sprite_map.dirty = true;
-}
-
-static PyObject*
-sprite_map_set_layout_py(PyObject UNUSED *self, PyObject *args) {
-    unsigned int w, h;
-    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
-    sprite_map_set_layout(w, h);
-    Py_RETURN_NONE;
-}
-
-void
-sprite_map_current_layout(unsigned int *x, unsigned int *y, unsigned int *z) {
-    *x = sprite_map.xnum; *y = sprite_map.ynum; *z = sprite_map.z;
-}
-
-PyObject*
-sprite_position_for(PyObject UNUSED *self, PyObject *args) {
-#define sprite_position_for_doc "sprite_position_for(ch, cc, is_second, attrs) -> x, y, z the sprite position for the specified text"
-    unsigned long ch = 0;
-    unsigned long long cc = 0;
-    unsigned int attrs = 1;
-    int is_second = 0, error = 0;
-    if (!PyArg_ParseTuple(args, "|kKpI", &ch, &cc, &is_second, &attrs)) return NULL;
-    SpritePosition *pos = sprite_map_position_for(ch, attrs, cc, is_second, &error);
-    if (pos == NULL) { sprite_map_set_error(error); return NULL; }
-    return Py_BuildValue("III", pos->x, pos->y, pos->z);
-}
-
-void
-render_dirty_sprites(void (*render)(PyObject*, bool, bool, bool, sprite_index, sprite_index, sprite_index)) {
-#define render_dirty_cells_doc "Render all cells that are marked as dirty"
-    if (!sprite_map.dirty) return;
-
-    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
-        SpritePosition *sp = &(sprite_map.cache[i]);
-        do {
-            if (sp->filled && !sp->rendered) {
-                PyObject *text = line_text_at(sp->ch, sp->cc);
-                render(text, sp->bold, sp->italic, sp->is_second, sp->x, sp->y, sp->z);
-                Py_CLEAR(text);
-                sp->rendered = true; 
-            }
-            sp = sp->next;
-        } while(sp);
-    }
-    sprite_map.dirty = false;
-}
-
-
-static PyMethodDef module_methods[] = {
-    {"sprite_position_for", (PyCFunction)sprite_position_for, METH_VARARGS, ""},
-    {"sprite_map_set_layout", (PyCFunction)sprite_map_set_layout_py, METH_VARARGS, ""},
-    {"sprite_map_set_limits", (PyCFunction)sprite_map_set_limits_py, METH_VARARGS, ""},
-
-    {NULL, NULL, 0, NULL}        /* Sentinel */
-};
-
-bool 
-init_sprites(PyObject *module) {
-    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
-    return true;
-}
diff --git a/kitty/sprites.h b/kitty/sprites.h
deleted file mode 100644
index 0ac115d71..000000000
--- a/kitty/sprites.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#pragma once
-
-
-void sprite_map_current_layout(unsigned int *x, unsigned int *y, unsigned int*);
-void sprite_map_set_layout(unsigned int cell_width, unsigned int cell_height);
-void sprite_map_set_limits(size_t max_texture_size, size_t max_array_len);
-void sprite_map_free();
-int sprite_map_increment(sprite_index *x, sprite_index *y, sprite_index *z);
-void render_dirty_sprites(void (*render)(PyObject*, bool, bool, bool, sprite_index, sprite_index, sprite_index));
diff --git a/kitty_tests/datatypes.py b/kitty_tests/datatypes.py
index 7cdc9dfdf..2632c01b8 100644
--- a/kitty_tests/datatypes.py
+++ b/kitty_tests/datatypes.py
@@ -8,7 +8,7 @@ from unittest import skipIf
 from kitty.config import build_ansi_color_table, defaults
 from kitty.fast_data_types import (
     REVERSE, ColorProfile, Cursor as C, HistoryBuf, LineBuf,
-    sprite_map_set_layout, sprite_map_set_limits, sprite_position_for
+    sprite_map_set_layout, sprite_map_set_limits, test_sprite_position_for
 )
 from kitty.utils import sanitize_title, wcwidth
 
@@ -329,15 +329,15 @@ class TestDataTypes(BaseTest):
     def test_sprite_map(self):
         sprite_map_set_limits(10, 2)
         sprite_map_set_layout(5, 5)
-        self.ae(sprite_position_for(0), (0, 0, 0))
-        self.ae(sprite_position_for(1), (1, 0, 0))
-        self.ae(sprite_position_for(2), (0, 1, 0))
-        self.ae(sprite_position_for(3), (1, 1, 0))
-        self.ae(sprite_position_for(4), (0, 0, 1))
-        self.ae(sprite_position_for(5), (1, 0, 1))
-        self.ae(sprite_position_for(0, 1), (0, 1, 1))
-        self.ae(sprite_position_for(0, 2), (1, 1, 1))
-        self.ae(sprite_position_for(0, 2), (1, 1, 1))
+        self.ae(test_sprite_position_for(0), (0, 0, 0))
+        self.ae(test_sprite_position_for(1), (1, 0, 0))
+        self.ae(test_sprite_position_for(2), (0, 1, 0))
+        self.ae(test_sprite_position_for(3), (1, 1, 0))
+        self.ae(test_sprite_position_for(4), (0, 0, 1))
+        self.ae(test_sprite_position_for(5), (1, 0, 1))
+        self.ae(test_sprite_position_for(0, 1), (0, 1, 1))
+        self.ae(test_sprite_position_for(0, 2), (1, 1, 1))
+        self.ae(test_sprite_position_for(0, 2), (1, 1, 1))
         sprite_map_set_limits(1000, 1000)
 
     def test_historybuf(self):