diff --git a/gpu/api.h b/gpu/api.h index 32a5241da..c8977ce1e 100644 --- a/gpu/api.h +++ b/gpu/api.h @@ -49,7 +49,7 @@ #include #include -#define KITTY_GPU_API_VERSION 2u +#define KITTY_GPU_API_VERSION 3u #define KITTY_GPU_DRIVER_ENTRY_POINT "kitty_gpu_driver_load" #define GPU_NULL_HANDLE ((uint64_t)0) @@ -227,11 +227,18 @@ typedef struct GPUUniformBlockInfo { uint32_t size; // size in bytes of the data in the block } GPUUniformBlockInfo; -typedef struct GPUUniformArrayInfo { - uint32_t offset; // byte offset of the array from the start of its uniform block - uint32_t stride; // bytes between consecutive elements - uint32_t count; // number of elements -} GPUUniformArrayInfo; +// Shaders are written in Slang and compiled ahead of time, or for user +// supplied shaders at runtime, to the shader target of the driver, see +// shader_target(). Everything about the parameters of a pipeline, such as the +// layout of uniform blocks, comes from the reflection data produced by the +// Slang compiler and is identified by the names used in the Slang source. +// Some targets, such as GLSL for OpenGL, look up parameters by the identifier +// used in the compiled code, which can differ from the Slang name. For those +// the mapping is supplied when creating the pipeline. +typedef struct GPUNativeName { + const char *name; // the name of a parameter, vertex input or uniform block type in the Slang source + const char *native_name; // the corresponding identifier in the compiled code +} GPUNativeName; // }}} // Drawing {{{ @@ -251,6 +258,14 @@ typedef enum { typedef struct GPUDriverAPI { uint32_t api_version; + // A human readable name for the driver, such as "OpenGL" + const char *name; + // The identifier of the shader target this driver consumes, for example + // glsl-140. This is a fixed property of the driver, available as soon as + // it is loaded. kitty compiles its built-in shaders for the targets of the + // drivers it builds at build time, and user supplied shaders for the target + // of the loaded driver at runtime. + const char *(*shader_target)(void *driver); // Lifecycle {{{ // Called once, when the first surface has been created and made current. @@ -304,18 +319,23 @@ typedef struct GPUDriverAPI { // }}} // Pipelines {{{ - // Create a pipeline from shader sources, on failure last_error() contains - // the compiler output - gpu_pipeline_t (*pipeline_create)(void *driver, const GPUShaderSource *sources, size_t num_sources); + // Create a pipeline from shader sources compiled for the shader target of + // the driver. native_names maps Slang names to identifiers in the compiled + // code, see GPUNativeName, and may be empty for targets that do not need + // it. On failure last_error() contains the compiler output. + gpu_pipeline_t (*pipeline_create)( + void *driver, const GPUShaderSource *sources, size_t num_sources, const GPUNativeName *native_names, size_t num_native_names); void (*pipeline_destroy)(void *driver, gpu_pipeline_t pipeline); // Make a pipeline current, GPU_NULL_HANDLE for none void (*use_pipeline)(void *driver, gpu_pipeline_t pipeline); + // All names below are names in the Slang source. // The slot used to set the named loose uniform with the set_uniform_*() // functions, or -1 if the pipeline does not use that uniform. For arrays, // name must not include a subscript. int32_t (*pipeline_uniform_slot)(void *driver, gpu_pipeline_t pipeline, const char *name); + // name is the name of the type of the uniform block. The size reported + // by the driver is used to verify the layout from the reflection data. bool (*pipeline_uniform_block)(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformBlockInfo *ans); - bool (*pipeline_uniform_array)(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformArrayInfo *ans); // Associate a uniform block of the pipeline with a binding point, see bind_uniform_buffer() void (*pipeline_bind_uniform_block)(void *driver, gpu_pipeline_t pipeline, uint32_t block_index, uint32_t binding_point); // The location of the named vertex input, -1 if it is not used diff --git a/gpu/opengl/driver.c b/gpu/opengl/driver.c index 707bd60c9..eadf51399 100644 --- a/gpu/opengl/driver.c +++ b/gpu/opengl/driver.c @@ -42,10 +42,17 @@ typedef struct Uniform { GLint location; } Uniform; +typedef struct NativeName { + char *name, *native_name; +} NativeName; + typedef struct Pipeline { GLuint id; Uniform *uniforms; GLint num_uniforms; + // Maps Slang names to GLSL identifiers + NativeName *native_names; + size_t num_native_names; } Pipeline; typedef struct Driver { @@ -153,6 +160,14 @@ version_string(void *driver) { return d->version_string; } +#define STRINGIFY_(x) #x +#define STRINGIFY(x) STRINGIFY_(x) + +static const char * +shader_target(void *driver UNUSED) { + return "glsl-" STRINGIFY(GLSL_VERSION); +} + static bool initialize(void *driver) { Driver *d = driver; @@ -645,6 +660,11 @@ pipeline_destroy(void *driver UNUSED, gpu_pipeline_t pipeline) { if (!p) return; if (p->id) glDeleteProgram(p->id); free(p->uniforms); + for (size_t i = 0; i < p->num_native_names; i++) { + free(p->native_names[i].name); + free(p->native_names[i].native_name); + } + free(p->native_names); free(p); } @@ -669,14 +689,55 @@ read_active_uniforms(Driver *d, Pipeline *p) { return true; } +static char * +dup_string(const char *src) { + const size_t n = strlen(src) + 1; + char *ans = malloc(n); + if (ans) memcpy(ans, src, n); + return ans; +} + +static bool +copy_native_names(Driver *d, Pipeline *p, const GPUNativeName *names, size_t count) { + if (!count) return true; + p->native_names = calloc(count, sizeof(NativeName)); + if (!p->native_names) { + set_error(d, "Out of memory allocating a GPU program"); + return false; + } + for (size_t i = 0; i < count; i++) { + p->native_names[i].name = dup_string(names[i].name); + p->native_names[i].native_name = dup_string(names[i].native_name); + p->num_native_names++; + if (!p->native_names[i].name || !p->native_names[i].native_name) { + set_error(d, "Out of memory allocating a GPU program"); + return false; + } + } + return true; +} + +// The GLSL identifier for a Slang name +static const char * +native_name(const Pipeline *p, const char *name) { + for (size_t i = 0; i < p->num_native_names; i++) { + if (strcmp(p->native_names[i].name, name) == 0) return p->native_names[i].native_name; + } + return name; +} + static gpu_pipeline_t -pipeline_create(void *driver, const GPUShaderSource *sources, size_t num_sources) { +pipeline_create(void *driver, const GPUShaderSource *sources, size_t num_sources, const GPUNativeName *native_names, size_t num_native_names) { Driver *d = driver; Pipeline *p = calloc(1, sizeof(Pipeline)); if (!p) { set_error(d, "Out of memory allocating a GPU program"); return GPU_NULL_HANDLE; } + if (!copy_native_names(d, p, native_names, num_native_names)) { + pipeline_destroy(d, AS_HANDLE(p)); + return GPU_NULL_HANDLE; + } p->id = glCreateProgram(); for (size_t i = 0; i < num_sources; i++) { GLuint shader_id = compile_shader(d, sources + i); @@ -714,6 +775,7 @@ use_pipeline(void *driver UNUSED, gpu_pipeline_t pipeline) { static int32_t pipeline_uniform_slot(void *driver UNUSED, gpu_pipeline_t pipeline, const char *name) { const Pipeline *p = AS_PIPELINE(pipeline); + name = native_name(p, name); const size_t n = strlen(name) + 1; for (GLint i = 0; i < p->num_uniforms; i++) { const Uniform *u = p->uniforms + i; @@ -725,6 +787,7 @@ pipeline_uniform_slot(void *driver UNUSED, gpu_pipeline_t pipeline, const char * static bool pipeline_uniform_block(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformBlockInfo *ans) { const Pipeline *p = AS_PIPELINE(pipeline); + name = native_name(p, name); GLuint index = glGetUniformBlockIndex(p->id, name); if (index == GL_INVALID_INDEX) { set_error(driver, "Could not find block index for %s", name); @@ -737,26 +800,6 @@ pipeline_uniform_block(void *driver, gpu_pipeline_t pipeline, const char *name, return true; } -static bool -pipeline_uniform_array(void *driver, gpu_pipeline_t pipeline, const char *name, GPUUniformArrayInfo *ans) { - const Pipeline *p = AS_PIPELINE(pipeline); - const GLchar *names[1] = {name}; - GLuint t; - glGetUniformIndices(p->id, 1, names, &t); - if (t == GL_INVALID_INDEX) { - set_error(driver, "Could not find the index for uniform array: %s", name); - return false; - } - GLint stride, offset, size; - glGetActiveUniformsiv(p->id, 1, &t, GL_UNIFORM_ARRAY_STRIDE, &stride); - glGetActiveUniformsiv(p->id, 1, &t, GL_UNIFORM_OFFSET, &offset); - glGetActiveUniformsiv(p->id, 1, &t, GL_UNIFORM_SIZE, &size); - ans->stride = stride > 0 ? (uint32_t)stride : 0; - ans->offset = offset > 0 ? (uint32_t)offset : 0; - ans->count = size > 0 ? (uint32_t)size : 0; - return true; -} - static void pipeline_bind_uniform_block(void *driver UNUSED, gpu_pipeline_t pipeline, uint32_t block_index, uint32_t binding_point) { glUniformBlockBinding(AS_PIPELINE(pipeline)->id, block_index, binding_point); @@ -764,7 +807,8 @@ pipeline_bind_uniform_block(void *driver UNUSED, gpu_pipeline_t pipeline, uint32 static int32_t pipeline_attribute_location(void *driver UNUSED, gpu_pipeline_t pipeline, const char *name) { - return glGetAttribLocation(AS_PIPELINE(pipeline)->id, name); + const Pipeline *p = AS_PIPELINE(pipeline); + return glGetAttribLocation(p->id, native_name(p, name)); } static void @@ -1195,7 +1239,9 @@ kitty_gpu_driver_load(uint32_t api_version, const GPUDriverInitParams *params, G the_driver = d; *api = (GPUDriverAPI){ .api_version = KITTY_GPU_API_VERSION, + .name = "OpenGL", #define F(name) .name = name + F(shader_target), F(initialize), F(destroy), F(last_error), @@ -1224,7 +1270,6 @@ kitty_gpu_driver_load(uint32_t api_version, const GPUDriverInitParams *params, G F(use_pipeline), F(pipeline_uniform_slot), F(pipeline_uniform_block), - F(pipeline_uniform_array), F(pipeline_bind_uniform_block), F(pipeline_attribute_location), F(set_uniform_floats), diff --git a/kitty/debug_config.py b/kitty/debug_config.py index e7bf24a8e..de1b35d66 100644 --- a/kitty/debug_config.py +++ b/kitty/debug_config.py @@ -19,7 +19,16 @@ from .child import cmdline_of_pid from .cli import version from .colors import theme_colors from .constants import extensions_dir, is_macos, is_wayland, kitty_base_dir, kitty_exe, shell_path -from .fast_data_types import Color, SingleKey, current_fonts, glfw_get_system_color_theme, gpu_driver_version_string, num_users, wayland_compositor_data +from .fast_data_types import ( + Color, + SingleKey, + current_fonts, + glfw_get_system_color_theme, + gpu_driver_name, + gpu_driver_version_string, + num_users, + wayland_compositor_data, +) from .options.types import Options as KittyOpts from .options.types import defaults, secret_options from .options.utils import KeyboardMode, KeyDefinition @@ -310,7 +319,7 @@ def debug_config(opts: KittyOpts | None = None, global_shortcuts: dict[str, Sing p(f.read().strip()) if not is_macos: p('Running under:', green(compositor_name())) - p(green('OpenGL:'), gpu_driver_version_string()) + p(green(f'{gpu_driver_name() or "GPU"}:'), gpu_driver_version_string()) p(green('Frozen:'), 'True' if getattr(sys, 'frozen', False) else 'False') p(green('Fonts:')) for k, font in current_fonts().items(): diff --git a/kitty/fast_data_types.pyi b/kitty/fast_data_types.pyi index b3d570f0e..bc29b0f39 100644 --- a/kitty/fast_data_types.pyi +++ b/kitty/fast_data_types.pyi @@ -87,7 +87,6 @@ QUIT_CAUSE_SIGNAL: int QUIT_CAUSE_FATAL_ERROR: int CLOSE_BEING_CONFIRMED: int ERROR_PREFIX: str -GLSL_VERSION: int # start glfw functional keys (auto generated by gen-key-constants.py do not edit) GLFW_FKEY_ESCAPE: int GLFW_FKEY_ENTER: int @@ -1682,6 +1681,9 @@ def wayland_compositor_data() -> Tuple[int, Optional[str]]: ... def monotonic() -> float: ... def timed_debug_print(x: str) -> None: ... def gpu_driver_version_string() -> str: ... +def gpu_driver_name() -> str: ... +def gpu_shader_target() -> str: ... +def load_gpu_driver(path: str, debug_rendering: bool = False) -> None: ... def systemd_move_pid_into_new_scope(pid: int, scope_name: str, description: str) -> str: ... def mime_type_for_filename(path: str) -> str | None: ... def mime_type_aliases(mime: str) -> tuple[str, ...]: ... diff --git a/kitty/gpu.c b/kitty/gpu.c index 93476bbf7..90ffcdacd 100644 --- a/kitty/gpu.c +++ b/kitty/gpu.c @@ -261,20 +261,6 @@ attrib_location(int program, const char *name) { return gpu_pipeline_attribute_location(programs[program], name); } -static UniformBlock -block_info(int program, const char *name) { - GPUUniformBlockInfo info; - if (!gpu_pipeline_uniform_block(programs[program], name, &info)) fatal("%s", gpu_last_error()); - return (UniformBlock){.index = (int32_t)info.index, .size = (int32_t)info.size}; -} - -static ArrayInformation -get_uniform_array_information(int program, const char *name) { - GPUUniformArrayInfo info; - if (!gpu_pipeline_uniform_array(programs[program], name, &info)) fatal("%s", gpu_last_error()); - return (ArrayInformation){.offset = (int32_t)info.offset, .stride = (int32_t)info.stride, .size = (int32_t)info.count}; -} - // Program metadata (uniform/attribute locations and UBO/array info, looked up by name) {{{ #define NAME program_metadata_map #define KEY_TY const char * @@ -312,62 +298,106 @@ program_metadata_entry(int program, const char *name, ProgramMetadataKind expect return &itr.data->val; } -void +// Fetches a required item from a dict in the program metadata, setting a +// Python exception if it is missing or of the wrong type +static PyObject * +metadata_item(PyObject *d, const char *key, PyTypeObject *type) { + PyObject *ans = PyDict_GetItemString(d, key); + if (!ans) { + PyErr_Format(PyExc_KeyError, "The shader program metadata has no %s entry", key); + return NULL; + } + if (!PyObject_TypeCheck(ans, type)) { + PyErr_Format(PyExc_TypeError, "The %s entry in the shader program metadata is not a %s", key, type->tp_name); + return NULL; + } + return ans; +} + +static bool +metadata_uint(PyObject *d, const char *key, int32_t *ans) { + PyObject *v = metadata_item(d, key, &PyLong_Type); + if (!v) return false; + long val = PyLong_AsLong(v); + if (PyErr_Occurred()) return false; + *ans = (int32_t)val; + return true; +} + +// The metadata is generated from the reflection data produced by the Slang +// compiler, see program_metadata() in shaders/slang.py. It is backend neutral, +// all names are names from the Slang source. +bool set_program_layout(int program, PyObject *metadata) { clear_program_layout(program); vt_init(program_layouts + program); program_layouts_initialized[program] = true; + const gpu_pipeline_t pipeline = programs[program]; PyObject *key, *val; Py_ssize_t pos; - PyObject *loose_uniforms = PyDict_GetItemString(metadata, "loose_uniforms"); + PyObject *inputs = metadata_item(metadata, "inputs", &PyDict_Type); + if (!inputs) return false; pos = 0; - while (PyDict_Next(loose_uniforms, &pos, &key, &val)) { - const char *actual_name = PyUnicode_AsUTF8(val); - // The metadata records array uniforms with a [0] subscript to mark them - // as arrays, but the driver looks them up by their bare name. - char without_subscript[256]; - const char *bracket = strchr(actual_name, '['); - if (bracket) { - size_t n = MIN(sizeof(without_subscript) - 1, (size_t)(bracket - actual_name)); - memcpy(without_subscript, actual_name, n); - without_subscript[n] = 0; - actual_name = without_subscript; - } - ProgramMetadataEntry e = {.kind = PROGRAM_UNIFORM, .location = gpu_pipeline_uniform_slot(programs[program], actual_name)}; - insert_program_metadata(program, PyUnicode_AsUTF8(key), e); - } - - PyObject *input_locations = PyDict_GetItemString(metadata, "input_locations"); - pos = 0; - while (PyDict_Next(input_locations, &pos, &key, &val)) { + while (PyDict_Next(inputs, &pos, &key, &val)) { ProgramMetadataEntry e = {.kind = PROGRAM_ATTRIBUTE, .location = (int32_t)PyLong_AsLong(val)}; + if (PyErr_Occurred()) return false; insert_program_metadata(program, PyUnicode_AsUTF8(key), e); } - PyObject *uniform_structs = PyDict_GetItemString(metadata, "uniform_structs"); - PyObject *uniform_struct_names = PyDict_GetItemString(metadata, "uniform_struct_names"); - pos = 0; - while (PyDict_Next(uniform_structs, &pos, &key, &val)) { - const char *glsl_block_name = PyUnicode_AsUTF8(PyDict_GetItem(uniform_struct_names, key)); - ProgramMetadataEntry be = {.kind = PROGRAM_BLOCK, .block = block_info(program, glsl_block_name)}; - insert_program_metadata(program, PyUnicode_AsUTF8(key), be); + PyObject *uniforms = metadata_item(metadata, "uniforms", &PyList_Type); + if (!uniforms) return false; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(uniforms); i++) { + const char *name = PyUnicode_AsUTF8(PyList_GET_ITEM(uniforms, i)); + if (!name) return false; + ProgramMetadataEntry e = {.kind = PROGRAM_UNIFORM, .location = gpu_pipeline_uniform_slot(pipeline, name)}; + insert_program_metadata(program, name, e); + } - PyObject *mkey, *mval; - Py_ssize_t mpos = 0; - while (PyDict_Next(val, &mpos, &mkey, &mval)) { - const char *decl = PyUnicode_AsUTF8(mval); - const char *bracket = strchr(decl, '['); - if (!bracket) continue; // scalar block members are hand-mirrored in C structs, not queried here - char actual_name[256]; - size_t n = MIN(sizeof(actual_name) - 1, (size_t)(bracket - decl)); - memcpy(actual_name, decl, n); - actual_name[n] = 0; - ProgramMetadataEntry ae = {.kind = PROGRAM_ARRAY, .array = get_uniform_array_information(program, actual_name)}; - insert_program_metadata(program, PyUnicode_AsUTF8(mkey), ae); + PyObject *blocks = metadata_item(metadata, "blocks", &PyDict_Type); + if (!blocks) return false; + pos = 0; + while (PyDict_Next(blocks, &pos, &key, &val)) { + const char *block_name = PyUnicode_AsUTF8(key); + if (!block_name) return false; + int32_t size; + if (!metadata_uint(val, "size", &size)) return false; + GPUUniformBlockInfo info; + if (!gpu_pipeline_uniform_block(pipeline, block_name, &info)) { + PyErr_Format(PyExc_ValueError, "The GPU program: %d has no uniform block: %s with error: %s", program, block_name, gpu_last_error()); + return false; + } + // The layout of the block is taken from the reflection data, so + // verify that it matches what the driver actually uses + if ((uint32_t)size != info.size) { + PyErr_Format( + PyExc_ValueError, + "The uniform block: %s in GPU program: %d has size %u according to the GPU driver but %d according to the shader reflection data", + block_name, + program, + info.size, + size); + return false; + } + ProgramMetadataEntry be = {.kind = PROGRAM_BLOCK, .block = {.index = (int32_t)info.index, .size = size}}; + insert_program_metadata(program, block_name, be); + + PyObject *arrays = metadata_item(val, "arrays", &PyDict_Type); + if (!arrays) return false; + PyObject *akey, *aval; + Py_ssize_t apos = 0; + while (PyDict_Next(arrays, &apos, &akey, &aval)) { + const char *array_name = PyUnicode_AsUTF8(akey); + if (!array_name) return false; + ArrayInformation ai; + if (!metadata_uint(aval, "offset", &ai.offset) || !metadata_uint(aval, "stride", &ai.stride) || !metadata_uint(aval, "count", &ai.size)) + return false; + ProgramMetadataEntry ae = {.kind = PROGRAM_ARRAY, .array = ai}; + insert_program_metadata(program, array_name, ae); } } + return true; } int32_t diff --git a/kitty/gpu.h b/kitty/gpu.h index 7ae1d51bd..92d5ea4e7 100644 --- a/kitty/gpu.h +++ b/kitty/gpu.h @@ -34,6 +34,16 @@ void unload_gpu_driver(void); #define D gpu_driver.driver #define A gpu_driver.api +static inline const char * +gpu_driver_name(void) { + return gpu_driver.driver ? A.name : ""; +} + +static inline const char * +gpu_shader_target(void) { + return gpu_driver.driver ? A.shader_target(D) : ""; +} + static inline const char * gpu_last_error(void) { return A.last_error(D); @@ -180,8 +190,8 @@ gpu_bind_uniform_buffer(uint32_t binding_point, gpu_buffer_t buffer) { } static inline gpu_pipeline_t -gpu_pipeline_create(const GPUShaderSource *sources, size_t num_sources) { - return A.pipeline_create(D, sources, num_sources); +gpu_pipeline_create(const GPUShaderSource *sources, size_t num_sources, const GPUNativeName *native_names, size_t num_native_names) { + return A.pipeline_create(D, sources, num_sources, native_names, num_native_names); } static inline void @@ -204,11 +214,6 @@ gpu_pipeline_uniform_block(gpu_pipeline_t pipeline, const char *name, GPUUniform return A.pipeline_uniform_block(D, pipeline, name, ans); } -static inline bool -gpu_pipeline_uniform_array(gpu_pipeline_t pipeline, const char *name, GPUUniformArrayInfo *ans) { - return A.pipeline_uniform_array(D, pipeline, name, ans); -} - static inline void gpu_pipeline_bind_uniform_block(gpu_pipeline_t pipeline, uint32_t block_index, uint32_t binding_point) { A.pipeline_bind_uniform_block(D, pipeline, block_index, binding_point); @@ -405,7 +410,8 @@ gpu_pipeline_t program_pipeline(int program); // Replace the pipeline of program, freeing any existing pipeline. Steals the reference to pipeline. void set_program_pipeline(int program, gpu_pipeline_t pipeline); int32_t attrib_location(int program, const char *name); -void set_program_layout(int program, PyObject *metadata); +// Returns false with a Python exception set on failure +bool set_program_layout(int program, PyObject *metadata); void free_program_layouts(void); int32_t program_uniform_location(int program, const char *name); int32_t try_program_uniform_location(int program, const char *name); diff --git a/kitty/shaders.c b/kitty/shaders.c index e4eed8802..e8bf5db07 100644 --- a/kitty/shaders.c +++ b/kitty/shaders.c @@ -3439,6 +3439,29 @@ pygpu_driver_version_string(PyObject *self UNUSED, PyObject *args UNUSED) { return PyUnicode_FromString(gpu_version_string()); } +static PyObject * +pygpu_driver_name(PyObject *self UNUSED, PyObject *args UNUSED) { + return PyUnicode_FromString(gpu_driver_name()); +} + +static PyObject * +pygpu_shader_target(PyObject *self UNUSED, PyObject *args UNUSED) { + return PyUnicode_FromString(gpu_shader_target()); +} + +static PyObject * +pyload_gpu_driver(PyObject *self UNUSED, PyObject *args) { + const char *path; + int debug_rendering = 0; + if (!PyArg_ParseTuple(args, "s|p", &path, &debug_rendering)) return NULL; + const char *err = load_gpu_driver(path, debug_rendering != 0); + if (err) { + PyErr_SetString(PyExc_OSError, err); + return NULL; + } + Py_RETURN_NONE; +} + static void free_custom_shader_pipeline(CustomShaderPipeline *p) { for (size_t i = 0; i < p->num_groups; i++) { @@ -3638,6 +3661,42 @@ compile_program(PyObject UNUSED *self, PyObject *args) { return NULL; } } + // The shaders must have been compiled for the target of the loaded driver + PyObject *target = PyDict_GetItemString(metadata, "target"); + const char *target_name = target && PyUnicode_Check(target) ? PyUnicode_AsUTF8(target) : NULL; + if (!target_name || strcmp(target_name, gpu_shader_target()) != 0) { + PyErr_Format( + PyExc_ValueError, + "The shaders were compiled for the target: %s but the GPU driver needs: %s", + target_name ? target_name : "unknown", + gpu_shader_target()); + return NULL; + } + GPUShaderLanguage language; + if (strncmp(target_name, "glsl-", 5) == 0) language = GPU_SHADER_LANGUAGE_GLSL; + else if (strncmp(target_name, "spirv-", 6) == 0) language = GPU_SHADER_LANGUAGE_SPIRV; + else if (strncmp(target_name, "msl-", 4) == 0) language = GPU_SHADER_LANGUAGE_MSL; + else { + PyErr_Format(PyExc_ValueError, "Unknown shader target: %s", target_name); + return NULL; + } + PyObject *native_names = PyDict_GetItemString(metadata, "native_names"); + if (!native_names || !PyDict_Check(native_names)) { + PyErr_SetString(PyExc_KeyError, "The shader program metadata has no native_names dict"); + return NULL; + } + RAII_ALLOC(GPUNativeName, names, calloc(MAX(1, PyDict_GET_SIZE(native_names)), sizeof(GPUNativeName))); + if (!names) return PyErr_NoMemory(); + size_t num_names = 0; + { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(native_names, &pos, &key, &val)) { + // The strings remain valid as long as the metadata dict does + if (!(names[num_names].name = PyUnicode_AsUTF8(key)) || !(names[num_names].native_name = PyUnicode_AsUTF8(val))) return NULL; + num_names++; + } + } RAII_PyObject(vertex_source, join_shader_sources(vertex_shaders)); if (!vertex_source) return NULL; RAII_PyObject(fragment_source, join_shader_sources(fragment_shaders)); @@ -3648,10 +3707,10 @@ compile_program(PyObject UNUSED *self, PyObject *args) { const char *fcode = PyUnicode_AsUTF8AndSize(fragment_source, &fsz); if (!fcode) return NULL; const GPUShaderSource sources[2] = { - {.stage = GPU_SHADER_VERTEX, .language = GPU_SHADER_LANGUAGE_GLSL, .code = vcode, .size = (size_t)vsz}, - {.stage = GPU_SHADER_FRAGMENT, .language = GPU_SHADER_LANGUAGE_GLSL, .code = fcode, .size = (size_t)fsz}, + {.stage = GPU_SHADER_VERTEX, .language = language, .code = vcode, .size = (size_t)vsz}, + {.stage = GPU_SHADER_FRAGMENT, .language = language, .code = fcode, .size = (size_t)fsz}, }; - gpu_pipeline_t pipeline = gpu_pipeline_create(sources, arraysz(sources)); + gpu_pipeline_t pipeline = gpu_pipeline_create(sources, arraysz(sources), names, num_names); if (!pipeline) { PyErr_SetString(PyExc_ValueError, gpu_last_error()); return NULL; @@ -3681,7 +3740,10 @@ compile_program(PyObject UNUSED *self, PyObject *args) { } custom_shaders.generation++; } - set_program_layout(which, metadata); + if (!set_program_layout(which, metadata)) { + set_program_pipeline(which, GPU_NULL_HANDLE); + return NULL; + } Py_RETURN_NONE; } @@ -3863,6 +3925,9 @@ static PyMethodDef module_methods[] = { M(compile_program, METH_VARARGS), M(sprite_map_set_limits, METH_VARARGS), MW(gpu_driver_version_string, METH_NOARGS), + MW(gpu_driver_name, METH_NOARGS), + MW(gpu_shader_target, METH_NOARGS), + MW(load_gpu_driver, METH_VARARGS), MW(custom_shader_needs_render, METH_VARARGS), MW(decorations_map_new_size, METH_VARARGS), MW(simulate_custom_shader_render_ticks, METH_VARARGS), @@ -3932,7 +3997,6 @@ init_shaders(PyObject *module) { C(MAX_SHADER_ANIM_USER_EVENTS); C(MAX_SHADER_ANIM_USER_EVENT_NAME_LEN); C(POINTER_MAP_SIZE); - C(GLSL_VERSION); C(ANIMATION_SAMPLE_WAIT); #undef C diff --git a/kitty/shaders/slang.py b/kitty/shaders/slang.py index 38b3ba19d..29ddb258e 100644 --- a/kitty/shaders/slang.py +++ b/kitty/shaders/slang.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # License: GPLv3 Copyright: 2026, Kovid Goyal -import glob import hashlib import json import os @@ -40,7 +39,6 @@ from kitty.fast_data_types import ( DECORATION, DECORATION_MASK, DIM, - GLSL_VERSION, GRAPHICS_ALPHA_MASK_PROGRAM, GRAPHICS_INDEXED_PROGRAM, GRAPHICS_INDEXED_UNSCALED_PROGRAM, @@ -62,6 +60,7 @@ from kitty.fast_data_types import ( TRAIL_PROGRAM, compile_program, get_options, + gpu_shader_target, ) from kitty.options.types import Options, defaults from kitty.options.utils import USER_ANIMATION_EVENT_PREFIX, EasingFunction, is_valid_user_animation_event_name @@ -206,14 +205,52 @@ def variant_name(variant: dict[str, str], default: dict[str, str]) -> str: return key.hexdigest()[:5] -def glsl_shaders(name: str, variant_name: str = '') -> tuple[str, str]: +class ShaderTarget(NamedTuple): + # The shader target consumed by a GPU driver, identified by strings of the + # form family-version, for example glsl-140. Built-in shaders are compiled + # for the targets of the drivers being built, at build time, and custom + # shaders for the target of the loaded driver, at runtime. + family: str + version: int + + @property + def id(self) -> str: + return f'{self.family}-{self.version}' + + @classmethod + def parse(cls, spec: str) -> 'ShaderTarget': + family, sep, version = spec.partition('-') + if not sep or not version.isdigit(): + raise ValueError(f'Invalid shader target: {spec!r}') + if family not in ('glsl',): + raise ValueError(f'Unsupported shader target: {spec!r}') + return cls(family, int(version)) + + @property + def slangc_args(self) -> list[str]: + # slangc fails with glsl_140 https://github.com/shader-slang/slang/issues/11898 + return ['-target', 'glsl', '-profile', f'glsl_{max(150, self.version)}'] + + +def current_shader_target() -> ShaderTarget: + spec = gpu_shader_target() + if not spec: + raise RuntimeError('No GPU driver has been loaded, cannot determine the shader target') + return ShaderTarget.parse(spec) + + +def builtin_program(target: ShaderTarget, name: str, variant_name: str = '') -> tuple[str, str, dict[str, Any]]: + # Returns the vertex and fragment shaders and the metadata for a built-in program if variant_name: variant_name = '.' + variant_name - with open(os.path.join(shaders_dir, f'{name}{variant_name}.vert.glsl')) as f: + base = os.path.join(shaders_dir, target.id, f'{name}{variant_name}') + with open(f'{base}.vert.glsl') as f: vert = f.read() - with open(os.path.join(shaders_dir, f'{name}{variant_name}.frag.glsl')) as f: + with open(f'{base}.frag.glsl') as f: frag = f.read() - return vert, frag + with open(f'{base}.json') as f: + metadata = json.load(f) + return vert, frag, metadata class LoadShaderPrograms: @@ -264,12 +301,12 @@ class LoadShaderPrograms: self.text_old_gamma = opts.text_composition_strategy == 'legacy' self.text_contrast_identity = text_contrast_is_identity(opts.text_composition_strategy) self.text_fg_override_threshold = opts.text_fg_override_threshold - metadata = load_glsl_metadata() + target = current_shader_target() def cell(prog: int) -> None: v = cell_variant(opts, program=prog) - vert, frag = glsl_shaders('cell', variant_name(v, default_cell_variant)) - compile_program(prog, (vert,), (frag,), metadata['cell'], allow_recompile) + vert, frag, metadata = builtin_program(target, 'cell', variant_name(v, default_cell_variant)) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) cell(CELL_PROGRAM), cell(CELL_BG_PROGRAM), cell(CELL_FG_PROGRAM) for prog, vname in { @@ -277,11 +314,11 @@ class LoadShaderPrograms: GRAPHICS_ALPHA_MASK_PROGRAM: 'alpha_mask', GRAPHICS_PREMULT_PROGRAM: 'premult', }.items(): - vert, frag = glsl_shaders('graphics', vname) - compile_program(prog, (vert,), (frag,), metadata['graphics'], allow_recompile) + vert, frag, metadata = builtin_program(target, 'graphics', vname) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) for prog, vname in {GRAPHICS_INDEXED_PROGRAM: '', GRAPHICS_INDEXED_UNSCALED_PROGRAM: 'unscaled'}.items(): - vert, frag = glsl_shaders('indexed_graphics', vname) - compile_program(prog, (vert,), (frag,), metadata['indexed_graphics'], allow_recompile) + vert, frag, metadata = builtin_program(target, 'indexed_graphics', vname) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) for name, prog in { 'bgimage': BGIMAGE_PROGRAM, 'tint': TINT_PROGRAM, @@ -292,8 +329,8 @@ class LoadShaderPrograms: 'border': BORDERS_PROGRAM, 'padding': PADDING_PROGRAM, }.items(): - vert, frag = glsl_shaders(name) - compile_program(prog, (vert,), (frag,), metadata[name], allow_recompile) + vert, frag, metadata = builtin_program(target, name) + compile_program(prog, (vert,), (frag,), metadata, allow_recompile) compile_program(-1, (), (), {}) # initialize programs self.compile_custom_shaders(allow_recompile) @@ -349,7 +386,7 @@ class LoadShaderPrograms: else: try: pipeline = merge_pipelines(slot_pipelines) - vert, frag, metadata = build_custom_shader_pipeline_glsl(pipeline) + vert, frag, metadata = build_custom_shader_pipeline_glsl(pipeline, current_shader_target()) # print(vert, file=open('/tmp/sample.vert', 'w')) # print(frag, file=open('/tmp/sample.frag', 'w')) except FileNotFoundError as e: @@ -1035,13 +1072,23 @@ def commands_to_compile_to_spirv(sources: dict[str, SlangFile], build_dir: str, # GLSL {{{ -glsl_version = max(150, GLSL_VERSION) # slangc fails with glsl_140 https://github.com/shader-slang/slang/issues/11898 -def commands_to_compile_to_glsl(sources: dict[str, SlangFile], build_dir: str, dest_dir: str, built_glsl_files: list[str]) -> Iterator[Command]: - for base_dest, base_build, slang_module, cmd, sfile in iter_entry_point_shaders(sources, build_dir, dest_dir): +def reflection_path(target: ShaderTarget, reflection_dir: str, shader_path: str) -> str: + # The reflection data produced by slangc for a compiled shader. It is used + # only at build time to generate the program metadata, so it lives in the + # build directory and is not installed. + return os.path.join(reflection_dir, target.id, os.path.basename(shader_path) + '.reflection.json') + + +def commands_to_compile_to_glsl( + sources: dict[str, SlangFile], build_dir: str, dest_dir: str, target: ShaderTarget, built_glsl_files: list[str] +) -> Iterator[Command]: + os.makedirs(os.path.join(dest_dir, target.id), exist_ok=True) + os.makedirs(os.path.join(build_dir, target.id), exist_ok=True) + for base_dest, base_build, slang_module, cmd, sfile in iter_entry_point_shaders(sources, build_dir, os.path.join(dest_dir, target.id)): module_mtime = os.path.getmtime(slang_module) - extra_cmd = ['-line-directive-mode', 'none', '-target', 'glsl', '-profile', f'glsl_{glsl_version}'] + extra_cmd = ['-line-directive-mode', 'none'] + target.slangc_args for ep in sfile.entry_points: for sp in sfile.specializations: v = {Stage.vertex: 'vert', Stage.fragment: 'frag'}[ep.stage] @@ -1049,7 +1096,7 @@ def commands_to_compile_to_glsl(sources: dict[str, SlangFile], build_dir: str, d dest = f'{base_dest}{sp.filename_insert}.{v}.glsl' if sp.name else f'{base_dest}.{v}.glsl' if sp.variables: c.insert(-1, f'{base_build}{sp.filename_insert}.slang-module') - c += extra_cmd + ['-entry', ep.name, '-stage', ep.stage.name, '-o', dest] + c += extra_cmd + ['-entry', ep.name, '-stage', ep.stage.name, '-o', dest, '-reflection-json', reflection_path(target, build_dir, dest)] output_mtime = safe_mtime(dest) needs_build = output_mtime < module_mtime if needs_build: @@ -1063,6 +1110,7 @@ class GLSLMetadata: input_locations: dict[str, int] uniform_struct_names: dict[str, str] fragment_inputs: dict[int, str] + native_inputs: dict[str, str] def __init__(self) -> None: self.loose_uniforms = {} @@ -1070,12 +1118,14 @@ class GLSLMetadata: self.input_locations = {} self.uniform_struct_names = {} self.fragment_inputs = {} + self.native_inputs = {} def merge(self, other: 'GLSLMetadata') -> None: self.loose_uniforms.update(other.loose_uniforms) self.uniform_structs.update(other.uniform_structs) self.uniform_struct_names.update(other.uniform_struct_names) self.input_locations.update(other.input_locations) + self.native_inputs.update(other.native_inputs) def asdict(self) -> dict[str, Any]: return { @@ -1084,6 +1134,7 @@ class GLSLMetadata: 'input_locations': self.input_locations, 'uniform_struct_names': self.uniform_struct_names, 'fragment_inputs': self.fragment_inputs, + 'native_inputs': self.native_inputs, } @classmethod @@ -1094,6 +1145,7 @@ class GLSLMetadata: ans.input_locations = d['input_locations'] ans.uniform_struct_names = d['uniform_struct_names'] ans.fragment_inputs = d['fragment_inputs'] + ans.native_inputs = d['native_inputs'] return ans @@ -1109,7 +1161,7 @@ class GLSLMetadata: GLSL_LOOSE_UNIFORM_BLOCKS = frozenset({'GammaLUT'}) -def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLMetadata | None) -> tuple[str, GLSLMetadata]: +def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLMetadata | None, glsl_version: int) -> tuple[str, GLSLMetadata]: is_fragment_shader = existing_metadata is None shader_name += '.frag.glsl' if is_fragment_shader else '.vert.glsl' lines: list[str] = [] @@ -1124,6 +1176,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM uniform_structs = {} uniform_struct_names = {} input_locations = {} + native_inputs = {} named_interface_blocks = set() pipeline_io_vars: dict[int, str] = {} replacements = { @@ -1187,7 +1240,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM current_uniform_names = [] else: if line.startswith('#version '): - line = f'#version {GLSL_VERSION}' + line = f'#version {glsl_version}' if not is_fragment_shader: line += '\n#extension GL_ARB_explicit_attrib_location : require' elif line.startswith('#extension ') or line in ('layout(row_major) buffer;', 'layout(push_constant)'): @@ -1220,6 +1273,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM elif not is_fragment_shader and words[0] == 'in': name = words[-1].rstrip(';') input_locations[name.rpartition('_')[0]] = int(lines[-1].split()[-1].rstrip(')')) + native_inputs[name.rpartition('_')[0]] = name lines.append(line) if fragment_inputs: raise ValueError( @@ -1250,6 +1304,7 @@ def fixup_opengl_code(glsl_code: str, shader_name: str, existing_metadata: GLSLM m.uniform_structs = uniform_structs m.input_locations = input_locations m.uniform_struct_names = uniform_struct_names + m.native_inputs = native_inputs if is_fragment_shader: m.fragment_inputs = pipeline_io_vars return ans, m @@ -1262,17 +1317,80 @@ def shader_name_from_path(path: str) -> str: return '.'.join(parts[:2]) -def fixup_opengl_files(paths: Iterable[str]) -> None: - "Convert the GLSL output of slangc to something that will work with OpenGL 3.1" +def program_metadata(target: ShaderTarget, reflections: Sequence[dict[str, Any]], native: GLSLMetadata) -> dict[str, Any]: + """ + The metadata about a program passed to the C code. It is generated from the + slangc reflection data for every stage of the program and is backend + neutral, using only names from the Slang source. The only target specific + part is native_names, which maps those names to the identifiers in the + compiled code, for targets that look up parameters by identifier. + """ + inputs: dict[str, int] = {} + uniforms: set[str] = set() + blocks: dict[str, dict[str, Any]] = {} + params: dict[str, dict[str, dict[str, int]]] = {} + # Uniform blocks that are emitted as uniform blocks rather than as loose uniforms by the target + real_blocks = set(native.uniform_struct_names) + for refl in reflections: + for p in refl['parameters']: + t = p['type'] + match t['kind']: + case 'constantBuffer': + evl = t['elementVarLayout'] + et = evl['type'] + tname = et['name'] + if tname in real_blocks: + arrays = {} + for f in et['fields']: + ft = f['type'] + if ft['kind'] == 'array': + arrays[f['name']] = {'offset': f['binding']['offset'], 'stride': ft['uniformStride'], 'count': ft['elementCount']} + blocks[tname] = {'size': evl['binding']['size'], 'arrays': arrays} + else: + uniforms.update(f['name'] for f in et['fields']) + case 'resource': + uniforms.add(p['name']) + case _: + raise ValueError(f'Unsupported shader parameter: {p["name"]} of kind: {t["kind"]}') + for ep in refl['entryPoints']: + stage = ep['stage'] + fields: dict[str, dict[str, int]] = {} + for p in ep.get('parameters', ()): + b = p.get('binding') or {} + match b.get('kind'): + case 'varyingInput': + if stage == 'vertex': + inputs[p['name']] = b['index'] + case 'uniform': + uniforms.add(p['name']) + fields[p['name']] = {'offset': b['offset'], 'size': b['size']} + params[stage] = fields + native_names = {k: v.partition('[')[0] for k, v in native.loose_uniforms.items()} # ] + native_names.update(native.uniform_struct_names) + native_names.update(native.native_inputs) + return { + 'target': target.id, + 'inputs': inputs, + 'uniforms': sorted(uniforms), + 'blocks': blocks, + 'params': params, + 'native_names': native_names, + } + + +def fixup_opengl_files(paths: Iterable[str], target: ShaderTarget, reflection_dir: str) -> None: + "Convert the GLSL output of slangc to something that will work with OpenGL 3.1 and write the program metadata" metadata_map: dict[str, GLSLMetadata] = {} + reflections: dict[str, list[dict[str, Any]]] = {} dest_dir = '' + # The fragment shader must be processed before the vertex shader, which sorting ensures for path in sorted(paths): dest_dir = os.path.dirname(path) with open(path, 'r') as f: glsl_code = f.read() shader_name = shader_name_from_path(path) try: - fixed, metadata = fixup_opengl_code(glsl_code, shader_name, metadata_map.get(shader_name)) + fixed, metadata = fixup_opengl_code(glsl_code, shader_name, metadata_map.get(shader_name), target.version) except Exception: os.unlink(path) raise @@ -1281,9 +1399,10 @@ def fixup_opengl_files(paths: Iterable[str]) -> None: metadata_map[shader_name].merge(metadata) else: metadata_map[shader_name] = metadata + with open(reflection_path(target, reflection_dir, path)) as f: + reflections.setdefault(shader_name, []).append(json.load(f)) for name, gm in metadata_map.items(): - with open(os.path.join(dest_dir, f'{name}.glsl.json'), 'w') as f: - f.write(json.dumps(gm.asdict())) + write_if_changed(os.path.join(dest_dir, f'{name}.json'), json.dumps(program_metadata(target, reflections[name], gm), indent=2, sort_keys=True)) def write_if_changed(dest: str, text: str) -> None: @@ -1295,34 +1414,6 @@ def write_if_changed(dest: str, text: str) -> None: f.write(text) -def glsl_metadata_for_shader(path: str) -> dict[str, Any]: - with open(path) as f: - d = json.load(f) - m = GLSLMetadata.fromdict(d) - return { - 'loose_uniforms': m.loose_uniforms, - 'uniform_structs': m.uniform_structs, - 'input_locations': m.input_locations, - 'uniform_struct_names': m.uniform_struct_names, - } - - -def write_glsl_metadata(dest_dir: str, dest: str = 'glsl-uniforms.json') -> None: - metadata_map = {} - for x in glob.glob(os.path.join(dest_dir, '*.glsl.json')): - shader_name = shader_name_from_path(x) - if '.' in shader_name: - continue - metadata_map[shader_name] = glsl_metadata_for_shader(x) - write_if_changed(os.path.join(dest_dir, dest), json.dumps(metadata_map, indent=2, sort_keys=True)) - - -@lru_cache(maxsize=1) -def load_glsl_metadata() -> dict[str, dict[str, Any]]: - with open(os.path.join(shaders_dir, 'glsl-uniforms.json')) as f: - return dict(json.load(f)) - - # }}} @@ -1357,7 +1448,7 @@ def create_specialisations(sources: dict[str, SlangFile], build_dir: str) -> Ite yield Command(needs_build, f'Compiling specialisation |{os.path.basename(dest)}| ...', list(slangc()) + [dest, '-o', dest + '-module']) -def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: ParallelRun) -> None: +def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: ParallelRun, targets: Sequence[ShaderTarget]) -> None: ensure_cache_dir(build_dir) ensure_cache_dir(dest_dir) src_dir = os.path.abspath('kitty/shaders') @@ -1373,24 +1464,33 @@ def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: Paralle # Now Vulkan shaders built_spirv_files: list[str] = [] spirv_commands = commands_to_compile_to_spirv(source_tree, build_dir, dest_dir, built_spirv_files) - # Now glsl files - built_glsl_files: list[str] = [] - glsl_commands = commands_to_compile_to_glsl(source_tree, build_dir, dest_dir, built_glsl_files) + # Now the shaders for the targets of the GPU drivers + built_glsl_files: dict[ShaderTarget, list[str]] = {} + target_commands: list[Iterator[Command]] = [] + for target in targets: + built_glsl_files[target] = [] + target_commands.append(commands_to_compile_to_glsl(source_tree, build_dir, dest_dir, target, built_glsl_files[target])) # Now run all commands - parallel_run(chain(spirv_commands, glsl_commands)) - fixup_opengl_files(built_glsl_files) - if shutil.which('glslangValidator'): - from kitty.shaders.validate_shaders import validation_command_for_file + parallel_run(chain(spirv_commands, *target_commands)) + for target, built in built_glsl_files.items(): + fixup_opengl_files(built, target, build_dir) + if shutil.which('glslangValidator'): + from kitty.shaders.validate_shaders import validation_command_for_file - parallel_run((True, f'Validating |{os.path.basename(x)}| ...', validation_command_for_file(x)) for x in built_glsl_files) - write_glsl_metadata(dest_dir) + parallel_run((True, f'Validating |{os.path.basename(x)}| ...', validation_command_for_file(x)) for x in built) def main() -> None: if not shutil.which(slangc()[0]): raise SystemExit(f'The shader slang compiler ({slangc()[0]}) not in PATH: {os.environ.get("PATH")}') setup = runpy.run_path('setup.py') - verbose = sys.argv[-3] == 'verbose' + verbose = 'verbose' in sys.argv + targets = [] + for x in sys.argv: + if x.startswith('--targets='): + targets = [ShaderTarget.parse(t) for t in x.partition('=')[2].split(',') if t] + if not targets: + raise SystemExit('No shader targets specified, use --targets=') Command = setup['Command'] parallel_run = setup['parallel_run'] emphasis = setup['emphasis'] @@ -1404,7 +1504,7 @@ def main() -> None: needed.append(Command(desc, cmd, lambda: True)) parallel_run(needed, verbose) - compile_builtin_shaders(sys.argv[-2], sys.argv[-1], prun) + compile_builtin_shaders(sys.argv[-2], sys.argv[-1], prun, targets) class SlangFailed(Exception): @@ -2093,7 +2193,7 @@ float4 fmain_wrap( def build_custom_shader_pipeline_glsl( - pipeline: Pipeline, cache_dir: str = '', invocation_tracker: set[tuple[str, ...]] | None = None + pipeline: Pipeline, target: ShaderTarget, cache_dir: str = '', invocation_tracker: set[tuple[str, ...]] | None = None ) -> tuple[str, str, dict[str, Any]]: import kitty.constants as kc @@ -2107,13 +2207,14 @@ def build_custom_shader_pipeline_glsl( ): pipeline_ir = build_custom_shader_pipeline_ir(pipeline, cache_dir, invocation_tracker) import_dirs, slang_module_path = pipeline_ir.import_dirs, pipeline_ir.module_path - glsl_dir = os.path.join(os.path.dirname(os.path.dirname(slang_module_path)), 'glsl') + glsl_dir = os.path.join(os.path.dirname(os.path.dirname(slang_module_path)), target.id) os.makedirs(glsl_dir, exist_ok=True) + os.makedirs(os.path.join(glsl_dir, target.id), exist_ok=True) module_mtime = safe_mtime(slang_module_path) slot = pipeline['slot'] vertex = os.path.join(glsl_dir, f'{slot}.vert.glsl') fragment = os.path.join(glsl_dir, f'{slot}.frag.glsl') - metadata = os.path.join(glsl_dir, f'{slot}.glsl.json') + metadata = os.path.join(glsl_dir, f'{slot}.json') if module_mtime > safe_mtime(metadata): inc = [] for x in import_dirs: @@ -2128,14 +2229,12 @@ def build_custom_shader_pipeline_glsl( 'none', '-lang', 'slang', - '-target', - 'glsl', - '-profile', - f'glsl_{glsl_version}', ] + + target.slangc_args ) - vcmd = cmd + ['-stage', 'vertex', '-entry', 'vmain_wrap', '-o', vertex, '--', '-'] - fcmd = cmd + ['-stage', 'fragment', '-entry', 'fmain_wrap', '-o', fragment, '--', '-'] + vrefl, frefl = reflection_path(target, glsl_dir, vertex), reflection_path(target, glsl_dir, fragment) + vcmd = cmd + ['-stage', 'vertex', '-entry', 'vmain_wrap', '-o', vertex, '-reflection-json', vrefl, '--', '-'] + fcmd = cmd + ['-stage', 'fragment', '-entry', 'fmain_wrap', '-o', fragment, '-reflection-json', frefl, '--', '-'] src = module_wrapper_for_slot(slot) invocation_tracker.add(tuple(vcmd)) invocation_tracker.add(tuple(fcmd)) @@ -2153,9 +2252,9 @@ def build_custom_shader_pipeline_glsl( finally: v.stderr.close() f.stderr.close() - fixup_opengl_files((fragment, vertex)) - with open(vertex) as vf, open(fragment) as ff: - m = glsl_metadata_for_shader(metadata) + fixup_opengl_files((fragment, vertex), target, glsl_dir) + with open(vertex) as vf, open(fragment) as ff, open(metadata) as mf: + m = json.load(mf) m['pipeline'] = pipeline m['has_pointer_map'] = bool(pipeline_ir.pointer_map_shaders) m['pointer_map_groups'] = sorted({g for g, _ in pipeline_ir.pointer_map_shaders}) diff --git a/kitty_tests/check_build.py b/kitty_tests/check_build.py index 04f64752f..d42efe8f5 100644 --- a/kitty_tests/check_build.py +++ b/kitty_tests/check_build.py @@ -161,6 +161,18 @@ class TestBuild(BaseTest): lib = ctypes.CDLL(path) self.assertTrue(hasattr(lib, 'kitty_gpu_driver_load'), f'{path} does not export kitty_gpu_driver_load') + def test_gpu_driver_shaders(self) -> None: + # The built-in shaders must have been compiled for the shader target of the driver + from kitty.constants import gpu_driver_path, shaders_dir + from kitty.fast_data_types import gpu_shader_target, load_gpu_driver + + if not gpu_shader_target(): + load_gpu_driver(gpu_driver_path('opengl')) + target = gpu_shader_target() + self.assertTrue(target, 'The GPU driver did not report a shader target') + q = os.path.join(shaders_dir, target, 'cell.json') + self.assertTrue(os.path.isfile(q), f'The built-in shaders have not been compiled for the {target} shader target of the GPU driver') + def test_all_kitten_names(self) -> None: from kittens.runner import all_kitten_names diff --git a/kitty_tests/slang.py b/kitty_tests/slang.py index cdf1b124d..d29e1161a 100644 --- a/kitty_tests/slang.py +++ b/kitty_tests/slang.py @@ -11,10 +11,12 @@ import sys import tempfile from unittest.mock import patch -from kitty.constants import slangc +from kitty.constants import gpu_driver_path, slangc +from kitty.fast_data_types import gpu_shader_target, load_gpu_driver from kitty.shaders.slang import ( EntryPoint, LoadShaderPrograms, + ShaderTarget, SlangFile, SlangShaderQuery, Stage, @@ -45,6 +47,13 @@ from .base import BaseTest _SUPPORT_SHADER_NAMES = frozenset(('types', 'pipeline')) +def driver_shader_target() -> ShaderTarget: + # The shader target of the GPU driver, loading the driver does not need a window + if not gpu_shader_target(): + load_gpu_driver(gpu_driver_path()) + return ShaderTarget.parse(gpu_shader_target()) + + class TestSlang(BaseTest): def test_animation_step_parsing(self): def step(val): @@ -301,6 +310,7 @@ class TestSlang(BaseTest): with ( patch('kitty.shaders.slang.parse_pipeline', return_value=pipeline), patch('kitty.shaders.slang.build_custom_shader_pipeline_glsl', return_value=('vertex', 'fragment', {})), + patch('kitty.shaders.slang.current_shader_target', return_value=driver_shader_target()), patch('kitty.shaders.slang.compile_program') as compile_program, ): for shaders in (('test',), (), ('test',)): @@ -792,9 +802,9 @@ float f() { static const float SCAN = 2; return SCAN; } 'test', ) try: - vert_src, frag_src, metadata = build_custom_shader_pipeline_glsl(p, cache_dir=cache_dir) + vert_src, frag_src, metadata = build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir) invocation_tracker = set() - build_custom_shader_pipeline_glsl(p, cache_dir=cache_dir, invocation_tracker=invocation_tracker) + build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir, invocation_tracker=invocation_tracker) finally: clear_caches() @@ -825,7 +835,7 @@ float f() { static const float SCAN = 2; return SCAN; } 'test', ) try: - _, pm_frag_src, pm_metadata = build_custom_shader_pipeline_glsl(p, cache_dir=cache_dir) + _, pm_frag_src, pm_metadata = build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir) finally: clear_caches() self.assertTrue(pm_metadata['has_pointer_map']) @@ -899,7 +909,7 @@ float f() { static const float SCAN = 2; return SCAN; } name, ) try: - vert_src, frag_src, _ = build_custom_shader_pipeline_glsl(pipeline, cache_dir=cache_dir) + vert_src, frag_src, _ = build_custom_shader_pipeline_glsl(pipeline, driver_shader_target(), cache_dir=cache_dir) except Exception as e: failures.append(f'{name}: {e}') continue @@ -914,7 +924,7 @@ float f() { static const float SCAN = 2; return SCAN; } for name in pipeline_names: try: pipeline = parse_pipeline_definition(get_custom_pipeline_src(name).decode().splitlines(), name) - build_custom_shader_pipeline_glsl(pipeline, cache_dir=cache_dir) + build_custom_shader_pipeline_glsl(pipeline, driver_shader_target(), cache_dir=cache_dir) except Exception as e: failures.append(f'{name}.pipeline: {e}') clear_caches() diff --git a/setup.py b/setup.py index bc729900f..72e4f6103 100755 --- a/setup.py +++ b/setup.py @@ -1326,6 +1326,19 @@ def gpu_driver_init_env(env: Env, driver: str = 'opengl') -> Env: return ans +def gpu_driver_shader_targets(driver: str = 'opengl') -> List[str]: + # The shader targets consumed by a GPU driver, the built-in shaders are + # compiled for these. Must match what the driver returns from its + # shader_target() function. + if driver == 'opengl': + with open('gpu/opengl/version.h') as f: + m = re.search(r'^#define GLSL_VERSION (\d+)', f.read(), flags=re.MULTILINE) + if m is None: + raise SystemExit('Could not find GLSL_VERSION in gpu/opengl/version.h') + return [f'glsl-{m.group(1)}'] + raise KeyError(f'Unknown GPU driver: {driver}') + + def gpu_driver_sources(driver: str) -> List[str]: if driver == 'opengl': common = ['driver.c', 'gl-wrapper.c'] @@ -1491,6 +1504,7 @@ def generate_shaders(kitty_exe: str) -> None: '+launch', os.path.join(src_base, 'kitty/shaders/slang.py'), 'verbose' if verbose else '', + '--targets=' + ','.join(t for driver in ('opengl',) for t in gpu_driver_shader_targets(driver)), 'build/shaders', 'shaders', ],