From d835cfa62f955d70d91e0e67d5eace52f28f5b8e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 3 Nov 2026 10:22:41 +0530 Subject: [PATCH] Make shader targets driver defined and program metadata reflection based Each GPU driver now declares the shader target it consumes (glsl-140 for the OpenGL driver), a fixed property available as soon as the driver is loaded. setup.py compiles the built-in shaders for the targets of the drivers it builds, into a directory per target, and custom shaders are compiled at runtime for the target of the loaded driver. The GLSL_VERSION constant is no longer exported to Python. The program metadata passed to the C code is now generated from the reflection data produced by slangc and is backend neutral: vertex input locations, uniform names, uniform block sizes and array layouts, all identified by their names in the Slang source. Block sizes are verified against the driver. The only target specific part is a map from Slang names to the identifiers in the compiled code, which the OpenGL driver needs to look up parameters by name and which is passed to the driver when creating a pipeline. The driver no longer needs to be queried for uniform array layouts. The debug config output now uses the name reported by the driver. --- gpu/api.h | 40 ++++-- gpu/opengl/driver.c | 91 +++++++++---- kitty/debug_config.py | 13 +- kitty/fast_data_types.pyi | 4 +- kitty/gpu.c | 138 ++++++++++++-------- kitty/gpu.h | 22 ++-- kitty/shaders.c | 74 ++++++++++- kitty/shaders/slang.py | 259 +++++++++++++++++++++++++------------ kitty_tests/check_build.py | 12 ++ kitty_tests/slang.py | 22 +++- setup.py | 14 ++ 11 files changed, 500 insertions(+), 189 deletions(-) diff --git a/gpu/api.h b/gpu/api.h index 32a5241da..c8977ce1e 100644 --- a/gpu/api.h +++ b/gpu/api.h @@ -49,7 +49,7 @@ #include #include -#define KITTY_GPU_API_VERSION 2u +#define KITTY_GPU_API_VERSION 3u #define KITTY_GPU_DRIVER_ENTRY_POINT "kitty_gpu_driver_load" #define GPU_NULL_HANDLE ((uint64_t)0) @@ -227,11 +227,18 @@ typedef struct GPUUniformBlockInfo { uint32_t size; // size in bytes of the data in the block } GPUUniformBlockInfo; -typedef struct GPUUniformArrayInfo { - uint32_t offset; // byte offset of the array from the start of its uniform block - uint32_t stride; // bytes between consecutive elements - uint32_t count; // number of elements -} GPUUniformArrayInfo; +// Shaders are written in Slang and compiled ahead of time, or for user +// supplied shaders at runtime, to the shader target of the driver, see +// shader_target(). Everything about the parameters of a pipeline, such as the +// layout of uniform blocks, comes from the reflection data produced by the +// Slang compiler and is identified by the names used in the Slang source. +// Some targets, such as GLSL for OpenGL, look up parameters by the identifier +// used in the compiled code, which can differ from the Slang name. For those +// the mapping is supplied when creating the pipeline. +typedef struct GPUNativeName { + const char *name; // the name of a parameter, vertex input or uniform block type in the Slang source + const char *native_name; // the corresponding identifier in the compiled code +} GPUNativeName; // }}} // Drawing {{{ @@ -251,6 +258,14 @@ typedef enum { typedef struct GPUDriverAPI { uint32_t api_version; + // A human readable name for the driver, such as "OpenGL" + const char *name; + // The identifier of the shader target this driver consumes, for example + // glsl-140. This is a fixed property of the driver, available as soon as + // it is loaded. kitty compiles its built-in shaders for the targets of the + // drivers it builds at build time, and user supplied shaders for the target + // of the loaded driver at runtime. + const char *(*shader_target)(void *driver); // Lifecycle {{{ // Called once, when the first surface has been created and made current. @@ -304,18 +319,23 @@ typedef struct GPUDriverAPI { // }}} // Pipelines {{{ - // Create a pipeline from shader sources, on failure last_error() contains - // the compiler output - gpu_pipeline_t (*pipeline_create)(void *driver, const GPUShaderSource *sources, size_t num_sources); + // Create a pipeline from shader sources compiled for the shader target of + // the driver. native_names maps Slang names to identifiers in the compiled + // code, see GPUNativeName, and may be empty for targets that do not need + // it. On failure last_error() contains the compiler output. + gpu_pipeline_t (*pipeline_create)( + void *driver, const GPUShaderSource *sources, size_t num_sources, const GPUNativeName *native_names, size_t num_native_names); void (*pipeline_destroy)(void *driver, gpu_pipeline_t pipeline); // Make a pipeline current, GPU_NULL_HANDLE for none void (*use_pipeline)(void *driver, gpu_pipeline_t pipeline); + // All names below are names in the Slang source. // The slot used to set the named loose uniform with the set_uniform_*() // functions, or -1 if the pipeline does not use that uniform. For arrays, // name must not include a subscript. int32_t (*pipeline_uniform_slot)(void *driver, gpu_pipeline_t pipeline, const char *name); + // name is the name of the type of the uniform block. The size reported + // by the driver is used to verify the layout from the reflection data. bool (*pipeline_uniform_block)(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformBlockInfo *ans); - bool (*pipeline_uniform_array)(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformArrayInfo *ans); // Associate a uniform block of the pipeline with a binding point, see bind_uniform_buffer() void (*pipeline_bind_uniform_block)(void *driver, gpu_pipeline_t pipeline, uint32_t block_index, uint32_t binding_point); // The location of the named vertex input, -1 if it is not used diff --git a/gpu/opengl/driver.c b/gpu/opengl/driver.c index 707bd60c9..eadf51399 100644 --- a/gpu/opengl/driver.c +++ b/gpu/opengl/driver.c @@ -42,10 +42,17 @@ typedef struct Uniform { GLint location; } Uniform; +typedef struct NativeName { + char *name, *native_name; +} NativeName; + typedef struct Pipeline { GLuint id; Uniform *uniforms; GLint num_uniforms; + // Maps Slang names to GLSL identifiers + NativeName *native_names; + size_t num_native_names; } Pipeline; typedef struct Driver { @@ -153,6 +160,14 @@ version_string(void *driver) { return d->version_string; } +#define STRINGIFY_(x) #x +#define STRINGIFY(x) STRINGIFY_(x) + +static const char * +shader_target(void *driver UNUSED) { + return "glsl-" STRINGIFY(GLSL_VERSION); +} + static bool initialize(void *driver) { Driver *d = driver; @@ -645,6 +660,11 @@ pipeline_destroy(void *driver UNUSED, gpu_pipeline_t pipeline) { if (!p) return; if (p->id) glDeleteProgram(p->id); free(p->uniforms); + for (size_t i = 0; i < p->num_native_names; i++) { + free(p->native_names[i].name); + free(p->native_names[i].native_name); + } + free(p->native_names); free(p); } @@ -669,14 +689,55 @@ read_active_uniforms(Driver *d, Pipeline *p) { return true; } +static char * +dup_string(const char *src) { + const size_t n = strlen(src) + 1; + char *ans = malloc(n); + if (ans) memcpy(ans, src, n); + return ans; +} + +static bool +copy_native_names(Driver *d, Pipeline *p, const GPUNativeName *names, size_t count) { + if (!count) return true; + p->native_names = calloc(count, sizeof(NativeName)); + if (!p->native_names) { + set_error(d, "Out of memory allocating a GPU program"); + return false; + } + for (size_t i = 0; i < count; i++) { + p->native_names[i].name = dup_string(names[i].name); + p->native_names[i].native_name = dup_string(names[i].native_name); + p->num_native_names++; + if (!p->native_names[i].name || !p->native_names[i].native_name) { + set_error(d, "Out of memory allocating a GPU program"); + return false; + } + } + return true; +} + +// The GLSL identifier for a Slang name +static const char * +native_name(const Pipeline *p, const char *name) { + for (size_t i = 0; i < p->num_native_names; i++) { + if (strcmp(p->native_names[i].name, name) == 0) return p->native_names[i].native_name; + } + return name; +} + static gpu_pipeline_t -pipeline_create(void *driver, const GPUShaderSource *sources, size_t num_sources) { +pipeline_create(void *driver, const GPUShaderSource *sources, size_t num_sources, const GPUNativeName *native_names, size_t num_native_names) { Driver *d = driver; Pipeline *p = calloc(1, sizeof(Pipeline)); if (!p) { set_error(d, "Out of memory allocating a GPU program"); return GPU_NULL_HANDLE; } + if (!copy_native_names(d, p, native_names, num_native_names)) { + pipeline_destroy(d, AS_HANDLE(p)); + return GPU_NULL_HANDLE; + } p->id = glCreateProgram(); for (size_t i = 0; i < num_sources; i++) { GLuint shader_id = compile_shader(d, sources + i); @@ -714,6 +775,7 @@ use_pipeline(void *driver UNUSED, gpu_pipeline_t pipeline) { static int32_t pipeline_uniform_slot(void *driver UNUSED, gpu_pipeline_t pipeline, const char *name) { const Pipeline *p = AS_PIPELINE(pipeline); + name = native_name(p, name); const size_t n = strlen(name) + 1; for (GLint i = 0; i < p->num_uniforms; i++) { const Uniform *u = p->uniforms + i; @@ -725,6 +787,7 @@ pipeline_uniform_slot(void *driver UNUSED, gpu_pipeline_t pipeline, const char * static bool pipeline_uniform_block(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformBlockInfo *ans) { const Pipeline *p = AS_PIPELINE(pipeline); + name = native_name(p, name); GLuint index = glGetUniformBlockIndex(p->id, name); if (index == GL_INVALID_INDEX) { set_error(driver, "Could not find block index for %s", name); @@ -737,26 +800,6 @@ pipeline_uniform_block(void *driver, gpu_pipeline_t pipeline, const char *name, return true; } -static bool -pipeline_uniform_array(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformArrayInfo *ans) { - const Pipeline *p = AS_PIPELINE(pipeline); - const GLchar *names[1] = {name}; - GLuint t; - glGetUniformIndices(p->id, 1, names, &t); - if (t == GL_INVALID_INDEX) { - set_error(driver, "Could not find the index for uniform array: %s", name); - return false; - } - GLint stride, offset, size; - glGetActiveUniformsiv(p->id, 1, &t, GL_UNIFORM_ARRAY_STRIDE, &stride); - glGetActiveUniformsiv(p->id, 1, &t, GL_UNIFORM_OFFSET, &offset); - glGetActiveUniformsiv(p->id, 1, &t, GL_UNIFORM_SIZE, &size); - ans->stride = stride > 0 ? (uint32_t)stride : 0; - ans->offset = offset > 0 ? (uint32_t)offset : 0; - ans->count = size > 0 ? (uint32_t)size : 0; - return true; -} - static void pipeline_bind_uniform_block(void *driver UNUSED, gpu_pipeline_t pipeline, uint32_t block_index, uint32_t binding_point) { glUniformBlockBinding(AS_PIPELINE(pipeline)->id, block_index, binding_point); @@ -764,7 +807,8 @@ pipeline_bind_uniform_block(void *driver UNUSED, gpu_pipeline_t pipeline, uint32 static int32_t pipeline_attribute_location(void *driver UNUSED, gpu_pipeline_t pipeline, const char *name) { - return glGetAttribLocation(AS_PIPELINE(pipeline)->id, name); + const Pipeline *p = AS_PIPELINE(pipeline); + return glGetAttribLocation(p->id, native_name(p, name)); } static void @@ -1195,7 +1239,9 @@ kitty_gpu_driver_load(uint32_t api_version, const GPUDriverInitParams *params, G the_driver = d; *api = (GPUDriverAPI){ .api_version = KITTY_GPU_API_VERSION, + .name = "OpenGL", #define F(name) .name = name + F(shader_target), F(initialize), F(destroy), F(last_error), @@ -1224,7 +1270,6 @@ kitty_gpu_driver_load(uint32_t api_version, const GPUDriverInitParams *params, G F(use_pipeline), F(pipeline_uniform_slot), F(pipeline_uniform_block), - F(pipeline_uniform_array), F(pipeline_bind_uniform_block), F(pipeline_attribute_location), F(set_uniform_floats), diff --git a/kitty/debug_config.py b/kitty/debug_config.py index e7bf24a8e..de1b35d66 100644 --- a/kitty/debug_config.py +++ b/kitty/debug_config.py @@ -19,7 +19,16 @@ from .child import cmdline_of_pid from .cli import version from .colors import theme_colors from .constants import extensions_dir, is_macos, is_wayland, kitty_base_dir, kitty_exe, shell_path -from .fast_data_types import Color, SingleKey, current_fonts, glfw_get_system_color_theme, gpu_driver_version_string, num_users, wayland_compositor_data +from .fast_data_types import ( + Color, + SingleKey, + current_fonts, + glfw_get_system_color_theme, + gpu_driver_name, + gpu_driver_version_string, + num_users, + wayland_compositor_data, +) from .options.types import Options as KittyOpts from .options.types import defaults, secret_options from .options.utils import KeyboardMode, KeyDefinition @@ -310,7 +319,7 @@ def debug_config(opts: KittyOpts | None = None, global_shortcuts: dict[str, Sing p(f.read().strip()) if not is_macos: p('Running under:', green(compositor_name())) - p(green('OpenGL:'), gpu_driver_version_string()) + p(green(f'{gpu_driver_name() or "GPU"}:'), gpu_driver_version_string()) p(green('Frozen:'), 'True' if getattr(sys, 'frozen', False) else 'False') p(green('Fonts:')) for k, font in current_fonts().items(): diff --git a/kitty/fast_data_types.pyi b/kitty/fast_data_types.pyi index b3d570f0e..bc29b0f39 100644 --- a/kitty/fast_data_types.pyi +++ b/kitty/fast_data_types.pyi @@ -87,7 +87,6 @@ QUIT_CAUSE_SIGNAL: int QUIT_CAUSE_FATAL_ERROR: int CLOSE_BEING_CONFIRMED: int ERROR_PREFIX: str -GLSL_VERSION: int # start glfw functional keys (auto generated by gen-key-constants.py do not edit) GLFW_FKEY_ESCAPE: int GLFW_FKEY_ENTER: int @@ -1682,6 +1681,9 @@ def wayland_compositor_data() -> Tuple[int, Optional[str]]: ... def monotonic() -> float: ... def timed_debug_print(x: str) -> None: ... def gpu_driver_version_string() -> str: ... +def gpu_driver_name() -> str: ... +def gpu_shader_target() -> str: ... +def load_gpu_driver(path: str, debug_rendering: bool = False) -> None: ... def systemd_move_pid_into_new_scope(pid: int, scope_name: str, description: str) -> str: ... def mime_type_for_filename(path: str) -> str | None: ... def mime_type_aliases(mime: str) -> tuple[str, ...]: ... diff --git a/kitty/gpu.c b/kitty/gpu.c index 93476bbf7..90ffcdacd 100644 --- a/kitty/gpu.c +++ b/kitty/gpu.c @@ -261,20 +261,6 @@ attrib_location(int program, const char *name) { return gpu_pipeline_attribute_location(programs[program], name); } -static UniformBlock -block_info(int program, const char *name) { - GPUUniformBlockInfo info; - if (!gpu_pipeline_uniform_block(programs[program], name, &info)) fatal("%s", gpu_last_error()); - return (UniformBlock){.index = (int32_t)info.index, .size = (int32_t)info.size}; -} - -static ArrayInformation -get_uniform_array_information(int program, const char *name) { - GPUUniformArrayInfo info; - if (!gpu_pipeline_uniform_array(programs[program], name, &info)) fatal("%s", gpu_last_error()); - return (ArrayInformation){.offset = (int32_t)info.offset, .stride = (int32_t)info.stride, .size = (int32_t)info.count}; -} - // Program metadata (uniform/attribute locations and UBO/array info, looked up by name) {{{ #define NAME program_metadata_map #define KEY_TY const char * @@ -312,62 +298,106 @@ program_metadata_entry(int program, const char *name, ProgramMetadataKind expect return &itr.data->val; } -void +// Fetches a required item from a dict in the program metadata, setting a +// Python exception if it is missing or of the wrong type +static PyObject * +metadata_item(PyObject *d, const char *key, PyTypeObject *type) { + PyObject *ans = PyDict_GetItemString(d, key); + if (!ans) { + PyErr_Format(PyExc_KeyError, "The shader program metadata has no %s entry", key); + return NULL; + } + if (!PyObject_TypeCheck(ans, type)) { + PyErr_Format(PyExc_TypeError, "The %s entry in the shader program metadata is not a %s", key, type->tp_name); + return NULL; + } + return ans; +} + +static bool +metadata_uint(PyObject *d, const char *key, int32_t *ans) { + PyObject *v = metadata_item(d, key, &PyLong_Type); + if (!v) return false; + long val = PyLong_AsLong(v); + if (PyErr_Occurred()) return false; + *ans = (int32_t)val; + return true; +} + +// The metadata is generated from the reflection data produced by the Slang +// compiler, see program_metadata() in shaders/slang.py. It is backend neutral, +// all names are names from the Slang source. +bool set_program_layout(int program, PyObject *metadata) { clear_program_layout(program); vt_init(program_layouts + program); program_layouts_initialized[program] = true; + const gpu_pipeline_t pipeline = programs[program]; PyObject *key, *val; Py_ssize_t pos; - PyObject *loose_uniforms = PyDict_GetItemString(metadata, "loose_uniforms"); + PyObject *inputs = metadata_item(metadata, "inputs", &PyDict_Type); + if (!inputs) return false; pos = 0; - while (PyDict_Next(loose_uniforms, &pos, &key, &val)) { - const char *actual_name = PyUnicode_AsUTF8(val); - // The metadata records array uniforms with a [0] subscript to mark them - // as arrays, but the driver looks them up by their bare name. - char without_subscript[256]; - const char *bracket = strchr(actual_name, '['); - if (bracket) { - size_t n = MIN(sizeof(without_subscript) - 1, (size_t)(bracket - actual_name)); - memcpy(without_subscript, actual_name, n); - without_subscript[n] = 0; - actual_name = without_subscript; - } - ProgramMetadataEntry e = {.kind = PROGRAM_UNIFORM, .location = gpu_pipeline_uniform_slot(programs[program], actual_name)}; - insert_program_metadata(program, PyUnicode_AsUTF8(key), e); - } - - PyObject *input_locations = PyDict_GetItemString(metadata, "input_locations"); - pos = 0; - while (PyDict_Next(input_locations, &pos, &key, &val)) { + while (PyDict_Next(inputs, &pos, &key, &val)) { ProgramMetadataEntry e = {.kind = PROGRAM_ATTRIBUTE, .location = (int32_t)PyLong_AsLong(val)}; + if (PyErr_Occurred()) return false; insert_program_metadata(program, PyUnicode_AsUTF8(key), e); } - PyObject *uniform_structs = PyDict_GetItemString(metadata, "uniform_structs"); - PyObject *uniform_struct_names = PyDict_GetItemString(metadata, "uniform_struct_names"); - pos = 0; - while (PyDict_Next(uniform_structs, &pos, &key, &val)) { - const char *glsl_block_name = PyUnicode_AsUTF8(PyDict_GetItem(uniform_struct_names, key)); - ProgramMetadataEntry be = {.kind = PROGRAM_BLOCK, .block = block_info(program, glsl_block_name)}; - insert_program_metadata(program, PyUnicode_AsUTF8(key), be); + PyObject *uniforms = metadata_item(metadata, "uniforms", &PyList_Type); + if (!uniforms) return false; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(uniforms); i++) { + const char *name = PyUnicode_AsUTF8(PyList_GET_ITEM(uniforms, i)); + if (!name) return false; + ProgramMetadataEntry e = {.kind = PROGRAM_UNIFORM, .location = gpu_pipeline_uniform_slot(pipeline, name)}; + insert_program_metadata(program, name, e); + } - PyObject *mkey, *mval; - Py_ssize_t mpos = 0; - while (PyDict_Next(val, &mpos, &mkey, &mval)) { - const char *decl = PyUnicode_AsUTF8(mval); - const char *bracket = strchr(decl, '['); - if (!bracket) continue; // scalar block members are hand-mirrored in C structs, not queried here - char actual_name[256]; - size_t n = MIN(sizeof(actual_name) - 1, (size_t)(bracket - decl)); - memcpy(actual_name, decl, n); - actual_name[n] = 0; - ProgramMetadataEntry ae = {.kind = PROGRAM_ARRAY, .array = get_uniform_array_information(program, actual_name)}; - insert_program_metadata(program, PyUnicode_AsUTF8(mkey), ae); + PyObject *blocks = metadata_item(metadata, "blocks", &PyDict_Type); + if (!blocks) return false; + pos = 0; + while (PyDict_Next(blocks, &pos, &key, &val)) { + const char *block_name = PyUnicode_AsUTF8(key); + if (!block_name) return false; + int32_t size; + if (!metadata_uint(val, "size", &size)) return false; + GPUUniformBlockInfo info; + if (!gpu_pipeline_uniform_block(pipeline, block_name, &info)) { + PyErr_Format(PyExc_ValueError, "The GPU program: %d has no uniform block: %s with error: %s", program, block_name, gpu_last_error()); + return false; + } + // The layout of the block is taken from the reflection data, so + // verify that it matches what the driver actually uses + if ((uint32_t)size != info.size) { + PyErr_Format( + PyExc_ValueError, + "The uniform block: %s in GPU program: %d has size %u according to the GPU driver but %d according to the shader reflection data", + block_name, + program, + info.size, + size); + return false; + } + ProgramMetadataEntry be = {.kind = PROGRAM_BLOCK, .block = {.index = (int32_t)info.index, .size = size}}; + insert_program_metadata(program, block_name, be); + + PyObject *arrays = metadata_item(val, "arrays", &PyDict_Type); + if (!arrays) return false; + PyObject *akey, *aval; + Py_ssize_t apos = 0; + while (PyDict_Next(arrays, &apos, &akey, &aval)) { + const char *array_name = PyUnicode_AsUTF8(akey); + if (!array_name) return false; + ArrayInformation ai; + if (!metadata_uint(aval, "offset", &ai.offset) || !metadata_uint(aval, "stride", &ai.stride) || !metadata_uint(aval, "count", &ai.size)) + return false; + ProgramMetadataEntry ae = {.kind = PROGRAM_ARRAY, .array = ai}; + insert_program_metadata(program, array_name, ae); } } + return true; } int32_t diff --git a/kitty/gpu.h b/kitty/gpu.h index 7ae1d51bd..92d5ea4e7 100644 --- a/kitty/gpu.h +++ b/kitty/gpu.h @@ -34,6 +34,16 @@ void unload_gpu_driver(void); #define D gpu_driver.driver #define A gpu_driver.api +static inline const char * +gpu_driver_name(void) { + return gpu_driver.driver ? A.name : ""; +} + +static inline const char * +gpu_shader_target(void) { + return gpu_driver.driver ? A.shader_target(D) : ""; +} + static inline const char * gpu_last_error(void) { return A.last_error(D); @@ -180,8 +190,8 @@ gpu_bind_uniform_buffer(uint32_t binding_point, gpu_buffer_t buffer) { } static inline gpu_pipeline_t -gpu_pipeline_create(const GPUShaderSource *sources, size_t num_sources) { - return A.pipeline_create(D, sources, num_sources); +gpu_pipeline_create(const GPUShaderSource *sources, size_t num_sources, const GPUNativeName *native_names, size_t num_native_names) { + return A.pipeline_create(D, sources, num_sources, native_names, num_native_names); } static inline void @@ -204,11 +214,6 @@ gpu_pipeline_uniform_block(gpu_pipeline_t pipeline, const char *name, GPUUniform return A.pipeline_uniform_block(D, pipeline, name, ans); } -static inline bool -gpu_pipeline_uniform_array(gpu_pipeline_t pipeline, const char *name, GPUUniformArrayInfo *ans) { - return A.pipeline_uniform_array(D, pipeline, name, ans); -} - static inline void gpu_pipeline_bind_uniform_block(gpu_pipeline_t pipeline, uint32_t block_index, uint32_t binding_point) { A.pipeline_bind_uniform_block(D, pipeline, block_index, binding_point); @@ -405,7 +410,8 @@ gpu_pipeline_t program_pipeline(int program); // Replace the pipeline of program, freeing any existing pipeline. Steals the reference to pipeline. void set_program_pipeline(int program, gpu_pipeline_t pipeline); int32_t attrib_location(int program, const char *name); -void set_program_layout(int program, PyObject *metadata); +// Returns false with a Python exception set on failure +bool set_program_layout(int program, PyObject *metadata); void free_program_layouts(void); int32_t program_uniform_location(int program, const char *name); int32_t try_program_uniform_location(int program, const char *name); diff --git a/kitty/shaders.c b/kitty/shaders.c index e4eed8802..e8bf5db07 100644 --- a/kitty/shaders.c +++ b/kitty/shaders.c @@ -3439,6 +3439,29 @@ pygpu_driver_version_string(PyObject *self UNUSED, PyObject *args UNUSED) { return PyUnicode_FromString(gpu_version_string()); } +static PyObject * +pygpu_driver_name(PyObject *self UNUSED, PyObject *args UNUSED) { + return PyUnicode_FromString(gpu_driver_name()); +} + +static PyObject * +pygpu_shader_target(PyObject *self UNUSED, PyObject *args UNUSED) { + return PyUnicode_FromString(gpu_shader_target()); +} + +static PyObject * +pyload_gpu_driver(PyObject *self UNUSED, PyObject *args) { + const char *path; + int debug_rendering = 0; + if (!PyArg_ParseTuple(args, "s|p", &path, &debug_rendering)) return NULL; + const char *err = load_gpu_driver(path, debug_rendering != 0); + if (err) { + PyErr_SetString(PyExc_OSError, err); + return NULL; + } + Py_RETURN_NONE; +} + static void free_custom_shader_pipeline(CustomShaderPipeline *p) { for (size_t i = 0; i < p->num_groups; i++) { @@ -3638,6 +3661,42 @@ compile_program(PyObject UNUSED *self, PyObject *args) { return NULL; } } + // The shaders must have been compiled for the target of the loaded driver + PyObject *target = PyDict_GetItemString(metadata, "target"); + const char *target_name = target && PyUnicode_Check(target) ? PyUnicode_AsUTF8(target) : NULL; + if (!target_name || strcmp(target_name, gpu_shader_target()) != 0) { + PyErr_Format( + PyExc_ValueError, + "The shaders were compiled for the target: %s but the GPU driver needs: %s", + target_name ? target_name : "unknown", + gpu_shader_target()); + return NULL; + } + GPUShaderLanguage language; + if (strncmp(target_name, "glsl-", 5) == 0) language = GPU_SHADER_LANGUAGE_GLSL; + else if (strncmp(target_name, "spirv-", 6) == 0) language = GPU_SHADER_LANGUAGE_SPIRV; + else if (strncmp(target_name, "msl-", 4) == 0) language = GPU_SHADER_LANGUAGE_MSL; + else { + PyErr_Format(PyExc_ValueError, "Unknown shader target: %s", target_name); + return NULL; + } + PyObject *native_names = PyDict_GetItemString(metadata, "native_names"); + if (!native_names || !PyDict_Check(native_names)) { + PyErr_SetString(PyExc_KeyError, "The shader program metadata has no native_names dict"); + return NULL; + } + RAII_ALLOC(GPUNativeName, names, calloc(MAX(1, PyDict_GET_SIZE(native_names)), sizeof(GPUNativeName))); + if (!names) return PyErr_NoMemory(); + size_t num_names = 0; + { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(native_names, &pos, &key, &val)) { + // The strings remain valid as long as the metadata dict does + if (!(names[num_names].name = PyUnicode_AsUTF8(key)) || !(names[num_names].native_name = PyUnicode_AsUTF8(val))) return NULL; + num_names++; + } + } RAII_PyObject(vertex_source, join_shader_sources(vertex_shaders)); if (!vertex_source) return NULL; RAII_PyObject(fragment_source, join_shader_sources(fragment_shaders)); @@ -3648,10 +3707,10 @@ compile_program(PyObject UNUSED *self, PyObject *args) { const char *fcode = PyUnicode_AsUTF8AndSize(fragment_source, &fsz); if (!fcode) return NULL; const GPUShaderSource sources[2] = { - {.stage = GPU_SHADER_VERTEX, .language = GPU_SHADER_LANGUAGE_GLSL, .code = vcode, .size = (size_t)vsz}, - {.stage = GPU_SHADER_FRAGMENT, .language = GPU_SHADER_LANGUAGE_GLSL, .code = fcode, .size = (size_t)fsz}, + {.stage = GPU_SHADER_VERTEX, .language = language, .code = vcode, .size = (size_t)vsz}, + {.stage = GPU_SHADER_FRAGMENT, .language = language, .code = fcode, .size = (size_t)fsz}, }; - gpu_pipeline_t pipeline = gpu_pipeline_create(sources, arraysz(sources)); + gpu_pipeline_t pipeline = gpu_pipeline_create(sources, arraysz(sources), names, num_names); if (!pipeline) { PyErr_SetString(PyExc_ValueError, gpu_last_error()); return NULL; @@ -3681,7 +3740,10 @@ compile_program(PyObject UNUSED *self, PyObject *args) { } custom_shaders.generation++; } - set_program_layout(which, metadata); + if (!set_program_layout(which, metadata)) { + set_program_pipeline(which, GPU_NULL_HANDLE); + return NULL; + } Py_RETURN_NONE; } @@ -3863,6 +3925,9 @@ static PyMethodDef module_methods[] = { M(compile_program, METH_VARARGS), M(sprite_map_set_limits, METH_VARARGS), MW(gpu_driver_version_string, METH_NOARGS), + MW(gpu_driver_name, METH_NOARGS), + MW(gpu_shader_target, METH_NOARGS), + MW(load_gpu_driver, METH_VARARGS), MW(custom_shader_needs_render, METH_VARARGS), MW(decorations_map_new_size, METH_VARARGS), MW(simulate_custom_shader_render_ticks, METH_VARARGS), @@ -3932,7 +3997,6 @@ init_shaders(PyObject *module) { C(MAX_SHADER_ANIM_USER_EVENTS); C(MAX_SHADER_ANIM_USER_EVENT_NAME_LEN); C(POINTER_MAP_SIZE); - C(GLSL_VERSION); C(ANIMATION_SAMPLE_WAIT); #undef C diff --git a/kitty/shaders/slang.py b/kitty/shaders/slang.py index 38b3ba19d..29ddb258e 100644 --- a/kitty/shaders/slang.py +++ b/kitty/shaders/slang.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # License: GPLv3 Copyright: 2026, Kovid Goyal -import glob import hashlib import json import os @@ -40,7 +39,6 @@ from kitty.fast_data_types import ( DECORATION, DECORATION_MASK, DIM, - GLSL_VERSION, GRAPHICS_ALPHA_MASK_PROGRAM, GRAPHICS_INDEXED_PROGRAM, GRAPHICS_INDEXED_UNSCALED_PROGRAM, @@ -62,6 +60,7 @@ from kitty.fast_data_types import ( TRAIL_PROGRAM, compile_program, get_options, + gpu_shader_target, ) from kitty.options.types import Options, defaults from kitty.options.utils import USER_ANIMATION_EVENT_PREFIX, EasingFunction, is_valid_user_animation_event_name @@ -206,14 +205,52 @@ def variant_name(variant: dict[str, str], default: dict[str, str]) -> str: return key.hexdigest()[:5] -def glsl_shaders(name: str, variant_name: str = '') -> tuple[str, str]: +class ShaderTarget(NamedTuple): + # The shader target consumed by a GPU driver, identified by strings of the + # form family-version, for example glsl-140. Built-in shaders are compiled + # for the targets of the drivers being built, at build time, and custom + # shaders for the target of the loaded driver, at runtime. + family: str + version: int + + @property + def id(self) -> str: + return f'{self.family}-{self.version}' + + @classmethod + def parse(cls, spec: str) -> 'ShaderTarget': + family, sep, version = spec.partition('-') + if not sep or not version.isdigit(): + raise ValueError(f'Invalid shader target: {spec!r}') + if family not in ('glsl',): + raise ValueError(f'Unsupported shader target: {spec!r}') + return cls(family, int(version)) + + @property + def slangc_args(self) -> list[str]: + # slangc fails with glsl_140 https://github.com/shader-slang/slang/issues/11898 + return ['-target', 'glsl', '-profile', f'glsl_{max(150, self.version)}'] + + +def current_shader_target() -> ShaderTarget: + spec = gpu_shader_target() + if not spec: + raise RuntimeError('No GPU driver has been loaded, cannot determine the shader target') + return ShaderTarget.parse(spec) + + +def builtin_program(target: ShaderTarget, name: str, variant_name: str = '') -> tuple[str, str, dict[str, Any]]: + # Returns the vertex and fragment shaders and the metadata for a built-in program if variant_name: variant_name = '.' + variant_name - with open(os.path.join(shaders_dir, f'{name}{variant_name}.vert.glsl')) as f: + base = os.path.join(shaders_dir, target.id, f'{name}{variant_name}') + with open(f'{base}.vert.glsl') as f: vert = f.read() - with open(os.path.join(shaders_dir, f'{name}{variant_name}.frag.glsl')) as f: + with open(f'{base}.frag.glsl') as f: frag = f.read() - return vert, frag + with open(f'{base}.json') as f: + metadata = json.load(f) + return vert, frag, metadata class LoadShaderPrograms: @@ -264,12 +301,12 @@ class LoadShaderPrograms: self.text_old_gamma = opts.text_composition_strategy == 'legacy' self.text_contrast_identity = text_contrast_is_identity(opts.text_composition_strategy) self.text_fg_override_threshold = opts.text_fg_override_threshold - metadata = load_glsl_metadata() + target = current_shader_target() def cell(prog: int) -> None: v = cell_variant(opts, program=prog) - vert, frag = glsl_shaders('cell', variant_name(v, default_cell_variant)) - compile_program(prog, (vert,), (frag,), metadata['cell'], allow_recompile) + vert, frag, metadata = builtin_program(target, 'cell', variant_name(v, default_cell_variant)) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) cell(CELL_PROGRAM), cell(CELL_BG_PROGRAM), cell(CELL_FG_PROGRAM) for prog, vname in { @@ -277,11 +314,11 @@ class LoadShaderPrograms: GRAPHICS_ALPHA_MASK_PROGRAM: 'alpha_mask', GRAPHICS_PREMULT_PROGRAM: 'premult', }.items(): - vert, frag = glsl_shaders('graphics', vname) - compile_program(prog, (vert,), (frag,), metadata['graphics'], allow_recompile) + vert, frag, metadata = builtin_program(target, 'graphics', vname) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) for prog, vname in {GRAPHICS_INDEXED_PROGRAM: '', GRAPHICS_INDEXED_UNSCALED_PROGRAM: 'unscaled'}.items(): - vert, frag = glsl_shaders('indexed_graphics', vname) - compile_program(prog, (vert,), (frag,), metadata['indexed_graphics'], allow_recompile) + vert, frag, metadata = builtin_program(target, 'indexed_graphics', vname) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) for name, prog in { 'bgimage': BGIMAGE_PROGRAM, 'tint': TINT_PROGRAM, @@ -292,8 +329,8 @@ class LoadShaderPrograms: 'border': BORDERS_PROGRAM, 'padding': PADDING_PROGRAM, }.items(): - vert, frag = glsl_shaders(name) - compile_program(prog, (vert,), (frag,), metadata[name], allow_recompile) + vert, frag, metadata = builtin_program(target, name) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) compile_program(-1, (), (), {}) # initialize programs self.compile_custom_shaders(allow_recompile) @@ -349,7 +386,7 @@ class LoadShaderPrograms: else: try: pipeline = merge_pipelines(slot_pipelines) - vert, frag, metadata = build_custom_shader_pipeline_glsl(pipeline) + vert, frag, metadata = build_custom_shader_pipeline_glsl(pipeline, current_shader_target()) # print(vert, file=open('/tmp/sample.vert', 'w')) # print(frag, file=open('/tmp/sample.frag', 'w')) except FileNotFoundError as e: @@ -1035,13 +1072,23 @@ def commands_to_compile_to_spirv(sources: dict[str, SlangFile], build_dir: str, # GLSL {{{ -glsl_version = max(150, GLSL_VERSION) # slangc fails with glsl_140 https://github.com/shader-slang/slang/issues/11898 -def commands_to_compile_to_glsl(sources: dict[str, SlangFile], build_dir: str, dest_dir: str, built_glsl_files: list[str]) -> Iterator[Command]: - for base_dest, base_build, slang_module, cmd, sfile in iter_entry_point_shaders(sources, build_dir, dest_dir): +def reflection_path(target: ShaderTarget, reflection_dir: str, shader_path: str) -> str: + # The reflection data produced by slangc for a compiled shader. It is used + # only at build time to generate the program metadata, so it lives in the + # build directory and is not installed. + return os.path.join(reflection_dir, target.id, os.path.basename(shader_path) + '.reflection.json') + + +def commands_to_compile_to_glsl( + sources: dict[str, SlangFile], build_dir: str, dest_dir: str, target: ShaderTarget, built_glsl_files: list[str] +) -> Iterator[Command]: + os.makedirs(os.path.join(dest_dir, target.id), exist_ok=True) + os.makedirs(os.path.join(build_dir, target.id), exist_ok=True) + for base_dest, base_build, slang_module, cmd, sfile in iter_entry_point_shaders(sources, build_dir, os.path.join(dest_dir, target.id)): module_mtime = os.path.getmtime(slang_module) - extra_cmd = ['-line-directive-mode', 'none', '-target', 'glsl', '-profile', f'glsl_{glsl_version}'] + extra_cmd = ['-line-directive-mode', 'none'] + target.slangc_args for ep in sfile.entry_points: for sp in sfile.specializations: v = {Stage.vertex: 'vert', Stage.fragment: 'frag'}[ep.stage] @@ -1049,7 +1096,7 @@ def commands_to_compile_to_glsl(sources: dict[str, SlangFile], build_dir: str, d dest = f'{base_dest}{sp.filename_insert}.{v}.glsl' if sp.name else f'{base_dest}.{v}.glsl' if sp.variables: c.insert(-1, f'{base_build}{sp.filename_insert}.slang-module') - c += extra_cmd + ['-entry', ep.name, '-stage', ep.stage.name, '-o', dest] + c += extra_cmd + ['-entry', ep.name, '-stage', ep.stage.name, '-o', dest, '-reflection-json', reflection_path(target, build_dir, dest)] output_mtime = safe_mtime(dest) needs_build = output_mtime < module_mtime if needs_build: @@ -1063,6 +1110,7 @@ class GLSLMetadata: input_locations: dict[str, int] uniform_struct_names: dict[str, str] fragment_inputs: dict[int, str] + native_inputs: dict[str, str] def __init__(self) -> None: self.loose_uniforms = {} @@ -1070,12 +1118,14 @@ class GLSLMetadata: self.input_locations = {} self.uniform_struct_names = {} self.fragment_inputs = {} + self.native_inputs = {} def merge(self, other: 'GLSLMetadata') -> None: self.loose_uniforms.update(other.loose_uniforms) self.uniform_structs.update(other.uniform_structs) self.uniform_struct_names.update(other.uniform_struct_names) self.input_locations.update(other.input_locations) + self.native_inputs.update(other.native_inputs) def asdict(self) -> dict[str, Any]: return { @@ -1084,6 +1134,7 @@ class GLSLMetadata: 'input_locations': self.input_locations, 'uniform_struct_names': self.uniform_struct_names, 'fragment_inputs': self.fragment_inputs, + 'native_inputs': self.native_inputs, } @classmethod @@ -1094,6 +1145,7 @@ class GLSLMetadata: ans.input_locations = d['input_locations'] ans.uniform_struct_names = d['uniform_struct_names'] ans.fragment_inputs = d['fragment_inputs'] + ans.native_inputs = d['native_inputs'] return ans @@ -1109,7 +1161,7 @@ class GLSLMetadata: GLSL_LOOSE_UNIFORM_BLOCKS = frozenset({'GammaLUT'}) -def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLMetadata | None) -> tuple[str, GLSLMetadata]: +def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLMetadata | None, glsl_version: int) -> tuple[str, GLSLMetadata]: is_fragment_shader = existing_metadata is None shader_name += '.frag.glsl' if is_fragment_shader else '.vert.glsl' lines: list[str] = [] @@ -1124,6 +1176,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM uniform_structs = {} uniform_struct_names = {} input_locations = {} + native_inputs = {} named_interface_blocks = set() pipeline_io_vars: dict[int, str] = {} replacements = { @@ -1187,7 +1240,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM current_uniform_names = [] else: if line.startswith('#version '): - line = f'#version {GLSL_VERSION}' + line = f'#version {glsl_version}' if not is_fragment_shader: line += '\n#extension GL_ARB_explicit_attrib_location : require' elif line.startswith('#extension ') or line in ('layout(row_major) buffer;', 'layout(push_constant)'): @@ -1220,6 +1273,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM elif not is_fragment_shader and words[0] == 'in': name = words[-1].rstrip(';') input_locations[name.rpartition('_')[0]] = int(lines[-1].split()[-1].rstrip(')')) + native_inputs[name.rpartition('_')[0]] = name lines.append(line) if fragment_inputs: raise ValueError( @@ -1250,6 +1304,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM m.uniform_structs = uniform_structs m.input_locations = input_locations m.uniform_struct_names = uniform_struct_names + m.native_inputs = native_inputs if is_fragment_shader: m.fragment_inputs = pipeline_io_vars return ans, m @@ -1262,17 +1317,80 @@ def shader_name_from_path(path: str) -> str: return '.'.join(parts[:2]) -def fixup_opengl_files(paths: Iterable[str]) -> None: - "Convert the GLSL output of slangc to something that will work with OpenGL 3.1" +def program_metadata(target: ShaderTarget, reflections: Sequence[dict[str, Any]], native: GLSLMetadata) -> dict[str, Any]: + """ + The metadata about a program passed to the C code. It is generated from the + slangc reflection data for every stage of the program and is backend + neutral, using only names from the Slang source. The only target specific + part is native_names, which maps those names to the identifiers in the + compiled code, for targets that look up parameters by identifier. + """ + inputs: dict[str, int] = {} + uniforms: set[str] = set() + blocks: dict[str, dict[str, Any]] = {} + params: dict[str, dict[str, dict[str, int]]] = {} + # Uniform blocks that are emitted as uniform blocks rather than as loose uniforms by the target + real_blocks = set(native.uniform_struct_names) + for refl in reflections: + for p in refl['parameters']: + t = p['type'] + match t['kind']: + case 'constantBuffer': + evl = t['elementVarLayout'] + et = evl['type'] + tname = et['name'] + if tname in real_blocks: + arrays = {} + for f in et['fields']: + ft = f['type'] + if ft['kind'] == 'array': + arrays[f['name']] = {'offset': f['binding']['offset'], 'stride': ft['uniformStride'], 'count': ft['elementCount']} + blocks[tname] = {'size': evl['binding']['size'], 'arrays': arrays} + else: + uniforms.update(f['name'] for f in et['fields']) + case 'resource': + uniforms.add(p['name']) + case _: + raise ValueError(f'Unsupported shader parameter: {p["name"]} of kind: {t["kind"]}') + for ep in refl['entryPoints']: + stage = ep['stage'] + fields: dict[str, dict[str, int]] = {} + for p in ep.get('parameters', ()): + b = p.get('binding') or {} + match b.get('kind'): + case 'varyingInput': + if stage == 'vertex': + inputs[p['name']] = b['index'] + case 'uniform': + uniforms.add(p['name']) + fields[p['name']] = {'offset': b['offset'], 'size': b['size']} + params[stage] = fields + native_names = {k: v.partition('[')[0] for k, v in native.loose_uniforms.items()} # ] + native_names.update(native.uniform_struct_names) + native_names.update(native.native_inputs) + return { + 'target': target.id, + 'inputs': inputs, + 'uniforms': sorted(uniforms), + 'blocks': blocks, + 'params': params, + 'native_names': native_names, + } + + +def fixup_opengl_files(paths: Iterable[str], target: ShaderTarget, reflection_dir: str) -> None: + "Convert the GLSL output of slangc to something that will work with OpenGL 3.1 and write the program metadata" metadata_map: dict[str, GLSLMetadata] = {} + reflections: dict[str, list[dict[str, Any]]] = {} dest_dir = '' + # The fragment shader must be processed before the vertex shader, which sorting ensures for path in sorted(paths): dest_dir = os.path.dirname(path) with open(path, 'r') as f: glsl_code = f.read() shader_name = shader_name_from_path(path) try: - fixed, metadata = fixup_opengl_code(glsl_code, shader_name, metadata_map.get(shader_name)) + fixed, metadata = fixup_opengl_code(glsl_code, shader_name, metadata_map.get(shader_name), target.version) except Exception: os.unlink(path) raise @@ -1281,9 +1399,10 @@ def fixup_opengl_files(paths: Iterable[str]) -> None: metadata_map[shader_name].merge(metadata) else: metadata_map[shader_name] = metadata + with open(reflection_path(target, reflection_dir, path)) as f: + reflections.setdefault(shader_name, []).append(json.load(f)) for name, gm in metadata_map.items(): - with open(os.path.join(dest_dir, f'{name}.glsl.json'), 'w') as f: - f.write(json.dumps(gm.asdict())) + write_if_changed(os.path.join(dest_dir, f'{name}.json'), json.dumps(program_metadata(target, reflections[name], gm), indent=2, sort_keys=True)) def write_if_changed(dest: str, text: str) -> None: @@ -1295,34 +1414,6 @@ def write_if_changed(dest: str, text: str) -> None: f.write(text) -def glsl_metadata_for_shader(path: str) -> dict[str, Any]: - with open(path) as f: - d = json.load(f) - m = GLSLMetadata.fromdict(d) - return { - 'loose_uniforms': m.loose_uniforms, - 'uniform_structs': m.uniform_structs, - 'input_locations': m.input_locations, - 'uniform_struct_names': m.uniform_struct_names, - } - - -def write_glsl_metadata(dest_dir: str, dest: str = 'glsl-uniforms.json') -> None: - metadata_map = {} - for x in glob.glob(os.path.join(dest_dir, '*.glsl.json')): - shader_name = shader_name_from_path(x) - if '.' in shader_name: - continue - metadata_map[shader_name] = glsl_metadata_for_shader(x) - write_if_changed(os.path.join(dest_dir, dest), json.dumps(metadata_map, indent=2, sort_keys=True)) - - -@lru_cache(maxsize=1) -def load_glsl_metadata() -> dict[str, dict[str, Any]]: - with open(os.path.join(shaders_dir, 'glsl-uniforms.json')) as f: - return dict(json.load(f)) - - # }}} @@ -1357,7 +1448,7 @@ def create_specialisations(sources: dict[str, SlangFile], build_dir: str) -> Ite yield Command(needs_build, f'Compiling specialisation |{os.path.basename(dest)}| ...', list(slangc()) + [dest, '-o', dest + '-module']) -def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: ParallelRun) -> None: +def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: ParallelRun, targets: Sequence[ShaderTarget]) -> None: ensure_cache_dir(build_dir) ensure_cache_dir(dest_dir) src_dir = os.path.abspath('kitty/shaders') @@ -1373,24 +1464,33 @@ def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: Paralle # Now Vulkan shaders built_spirv_files: list[str] = [] spirv_commands = commands_to_compile_to_spirv(source_tree, build_dir, dest_dir, built_spirv_files) - # Now glsl files - built_glsl_files: list[str] = [] - glsl_commands = commands_to_compile_to_glsl(source_tree, build_dir, dest_dir, built_glsl_files) + # Now the shaders for the targets of the GPU drivers + built_glsl_files: dict[ShaderTarget, list[str]] = {} + target_commands: list[Iterator[Command]] = [] + for target in targets: + built_glsl_files[target] = [] + target_commands.append(commands_to_compile_to_glsl(source_tree, build_dir, dest_dir, target, built_glsl_files[target])) # Now run all commands - parallel_run(chain(spirv_commands, glsl_commands)) - fixup_opengl_files(built_glsl_files) - if shutil.which('glslangValidator'): - from kitty.shaders.validate_shaders import validation_command_for_file + parallel_run(chain(spirv_commands, *target_commands)) + for target, built in built_glsl_files.items(): + fixup_opengl_files(built, target, build_dir) + if shutil.which('glslangValidator'): + from kitty.shaders.validate_shaders import validation_command_for_file - parallel_run((True, f'Validating |{os.path.basename(x)}| ...', validation_command_for_file(x)) for x in built_glsl_files) - write_glsl_metadata(dest_dir) + parallel_run((True, f'Validating |{os.path.basename(x)}| ...', validation_command_for_file(x)) for x in built) def main() -> None: if not shutil.which(slangc()[0]): raise SystemExit(f'The shader slang compiler ({slangc()[0]}) not in PATH: {os.environ.get("PATH")}') setup = runpy.run_path('setup.py') - verbose = sys.argv[-3] == 'verbose' + verbose = 'verbose' in sys.argv + targets = [] + for x in sys.argv: + if x.startswith('--targets='): + targets = [ShaderTarget.parse(t) for t in x.partition('=')[2].split(',') if t] + if not targets: + raise SystemExit('No shader targets specified, use --targets=') Command = setup['Command'] parallel_run = setup['parallel_run'] emphasis = setup['emphasis'] @@ -1404,7 +1504,7 @@ def main() -> None: needed.append(Command(desc, cmd, lambda: True)) parallel_run(needed, verbose) - compile_builtin_shaders(sys.argv[-2], sys.argv[-1], prun) + compile_builtin_shaders(sys.argv[-2], sys.argv[-1], prun, targets) class SlangFailed(Exception): @@ -2093,7 +2193,7 @@ float4 fmain_wrap( def build_custom_shader_pipeline_glsl( - pipeline: Pipeline, cache_dir: str = '', invocation_tracker: set[tuple[str, ...]] | None = None + pipeline: Pipeline, target: ShaderTarget, cache_dir: str = '', invocation_tracker: set[tuple[str, ...]] | None = None ) -> tuple[str, str, dict[str, Any]]: import kitty.constants as kc @@ -2107,13 +2207,14 @@ def build_custom_shader_pipeline_glsl( ): pipeline_ir = build_custom_shader_pipeline_ir(pipeline, cache_dir, invocation_tracker) import_dirs, slang_module_path = pipeline_ir.import_dirs, pipeline_ir.module_path - glsl_dir = os.path.join(os.path.dirname(os.path.dirname(slang_module_path)), 'glsl') + glsl_dir = os.path.join(os.path.dirname(os.path.dirname(slang_module_path)), target.id) os.makedirs(glsl_dir, exist_ok=True) + os.makedirs(os.path.join(glsl_dir, target.id), exist_ok=True) module_mtime = safe_mtime(slang_module_path) slot = pipeline['slot'] vertex = os.path.join(glsl_dir, f'{slot}.vert.glsl') fragment = os.path.join(glsl_dir, f'{slot}.frag.glsl') - metadata = os.path.join(glsl_dir, f'{slot}.glsl.json') + metadata = os.path.join(glsl_dir, f'{slot}.json') if module_mtime > safe_mtime(metadata): inc = [] for x in import_dirs: @@ -2128,14 +2229,12 @@ def build_custom_shader_pipeline_glsl( 'none', '-lang', 'slang', - '-target', - 'glsl', - '-profile', - f'glsl_{glsl_version}', ] + + target.slangc_args ) - vcmd = cmd + ['-stage', 'vertex', '-entry', 'vmain_wrap', '-o', vertex, '--', '-'] - fcmd = cmd + ['-stage', 'fragment', '-entry', 'fmain_wrap', '-o', fragment, '--', '-'] + vrefl, frefl = reflection_path(target, glsl_dir, vertex), reflection_path(target, glsl_dir, fragment) + vcmd = cmd + ['-stage', 'vertex', '-entry', 'vmain_wrap', '-o', vertex, '-reflection-json', vrefl, '--', '-'] + fcmd = cmd + ['-stage', 'fragment', '-entry', 'fmain_wrap', '-o', fragment, '-reflection-json', frefl, '--', '-'] src = module_wrapper_for_slot(slot) invocation_tracker.add(tuple(vcmd)) invocation_tracker.add(tuple(fcmd)) @@ -2153,9 +2252,9 @@ def build_custom_shader_pipeline_glsl( finally: v.stderr.close() f.stderr.close() - fixup_opengl_files((fragment, vertex)) - with open(vertex) as vf, open(fragment) as ff: - m = glsl_metadata_for_shader(metadata) + fixup_opengl_files((fragment, vertex), target, glsl_dir) + with open(vertex) as vf, open(fragment) as ff, open(metadata) as mf: + m = json.load(mf) m['pipeline'] = pipeline m['has_pointer_map'] = bool(pipeline_ir.pointer_map_shaders) m['pointer_map_groups'] = sorted({g for g, _ in pipeline_ir.pointer_map_shaders}) diff --git a/kitty_tests/check_build.py b/kitty_tests/check_build.py index 04f64752f..d42efe8f5 100644 --- a/kitty_tests/check_build.py +++ b/kitty_tests/check_build.py @@ -161,6 +161,18 @@ class TestBuild(BaseTest): lib = ctypes.CDLL(path) self.assertTrue(hasattr(lib, 'kitty_gpu_driver_load'), f'{path} does not export kitty_gpu_driver_load') + def test_gpu_driver_shaders(self) -> None: + # The built-in shaders must have been compiled for the shader target of the driver + from kitty.constants import gpu_driver_path, shaders_dir + from kitty.fast_data_types import gpu_shader_target, load_gpu_driver + + if not gpu_shader_target(): + load_gpu_driver(gpu_driver_path('opengl')) + target = gpu_shader_target() + self.assertTrue(target, 'The GPU driver did not report a shader target') + q = os.path.join(shaders_dir, target, 'cell.json') + self.assertTrue(os.path.isfile(q), f'The built-in shaders have not been compiled for the {target} shader target of the GPU driver') + def test_all_kitten_names(self) -> None: from kittens.runner import all_kitten_names diff --git a/kitty_tests/slang.py b/kitty_tests/slang.py index cdf1b124d..d29e1161a 100644 --- a/kitty_tests/slang.py +++ b/kitty_tests/slang.py @@ -11,10 +11,12 @@ import sys import tempfile from unittest.mock import patch -from kitty.constants import slangc +from kitty.constants import gpu_driver_path, slangc +from kitty.fast_data_types import gpu_shader_target, load_gpu_driver from kitty.shaders.slang import ( EntryPoint, LoadShaderPrograms, + ShaderTarget, SlangFile, SlangShaderQuery, Stage, @@ -45,6 +47,13 @@ from .base import BaseTest _SUPPORT_SHADER_NAMES = frozenset(('types', 'pipeline')) +def driver_shader_target() -> ShaderTarget: + # The shader target of the GPU driver, loading the driver does not need a window + if not gpu_shader_target(): + load_gpu_driver(gpu_driver_path()) + return ShaderTarget.parse(gpu_shader_target()) + + class TestSlang(BaseTest): def test_animation_step_parsing(self): def step(val): @@ -301,6 +310,7 @@ class TestSlang(BaseTest): with ( patch('kitty.shaders.slang.parse_pipeline', return_value=pipeline), patch('kitty.shaders.slang.build_custom_shader_pipeline_glsl', return_value=('vertex', 'fragment', {})), + patch('kitty.shaders.slang.current_shader_target', return_value=driver_shader_target()), patch('kitty.shaders.slang.compile_program') as compile_program, ): for shaders in (('test',), (), ('test',)): @@ -792,9 +802,9 @@ float f() { static const float SCAN = 2; return SCAN; } 'test', ) try: - vert_src, frag_src, metadata = build_custom_shader_pipeline_glsl(p, cache_dir=cache_dir) + vert_src, frag_src, metadata = build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir) invocation_tracker = set() - build_custom_shader_pipeline_glsl(p, cache_dir=cache_dir, invocation_tracker=invocation_tracker) + build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir, invocation_tracker=invocation_tracker) finally: clear_caches() @@ -825,7 +835,7 @@ float f() { static const float SCAN = 2; return SCAN; } 'test', ) try: - _, pm_frag_src, pm_metadata = build_custom_shader_pipeline_glsl(p, cache_dir=cache_dir) + _, pm_frag_src, pm_metadata = build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir) finally: clear_caches() self.assertTrue(pm_metadata['has_pointer_map']) @@ -899,7 +909,7 @@ float f() { static const float SCAN = 2; return SCAN; } name, ) try: - vert_src, frag_src, _ = build_custom_shader_pipeline_glsl(pipeline, cache_dir=cache_dir) + vert_src, frag_src, _ = build_custom_shader_pipeline_glsl(pipeline, driver_shader_target(), cache_dir=cache_dir) except Exception as e: failures.append(f'{name}: {e}') continue @@ -914,7 +924,7 @@ float f() { static const float SCAN = 2; return SCAN; } for name in pipeline_names: try: pipeline = parse_pipeline_definition(get_custom_pipeline_src(name).decode().splitlines(), name) - build_custom_shader_pipeline_glsl(pipeline, cache_dir=cache_dir) + build_custom_shader_pipeline_glsl(pipeline, driver_shader_target(), cache_dir=cache_dir) except Exception as e: failures.append(f'{name}.pipeline: {e}') clear_caches() diff --git a/setup.py b/setup.py index bc729900f..72e4f6103 100755 --- a/setup.py +++ b/setup.py @@ -1326,6 +1326,19 @@ def gpu_driver_init_env(env: Env, driver: str = 'opengl') -> Env: return ans +def gpu_driver_shader_targets(driver: str = 'opengl') -> List[str]: + # The shader targets consumed by a GPU driver, the built-in shaders are + # compiled for these. Must match what the driver returns from its + # shader_target() function. + if driver == 'opengl': + with open('gpu/opengl/version.h') as f: + m = re.search(r'^#define GLSL_VERSION (\d+)', f.read(), flags=re.MULTILINE) + if m is None: + raise SystemExit('Could not find GLSL_VERSION in gpu/opengl/version.h') + return [f'glsl-{m.group(1)}'] + raise KeyError(f'Unknown GPU driver: {driver}') + + def gpu_driver_sources(driver: str) -> List[str]: if driver == 'opengl': common = ['driver.c', 'gl-wrapper.c'] @@ -1491,6 +1504,7 @@ def generate_shaders(kitty_exe: str) -> None: '+launch', os.path.join(src_base, 'kitty/shaders/slang.py'), 'verbose' if verbose else '', + '--targets=' + ','.join(t for driver in ('opengl',) for t in gpu_driver_shader_targets(driver)), 'build/shaders', 'shaders', ],