mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-21 23:54:57 +00:00
Add a SPIR-V shader target for the built-in and custom shaders
Shader targets of the form spirv-NN (for SPIR-V N.N) compile each program to a single SPIR-V module containing the entry points for all stages, along with the same backend neutral program metadata as for GLSL. The compiled modules are validated with spirv-val, when it is available. Custom shaders are compiled at runtime for SPIR-V targets as well, and the tests compile and validate all bundled custom shaders for SPIR-V in addition to the target of the loaded driver. The program metadata now includes the names of the entry points, which are passed to drivers in GPUShaderSource, since a module can contain the entry points of several stages. compile_program() accepts binary shaders as bytes. This replaces the unused, untargeted .spv files that were built before.
This commit is contained in:
parent
fd43ed0a88
commit
56a683b0ea
6 changed files with 237 additions and 124 deletions
|
|
@ -224,6 +224,10 @@ typedef struct GPUShaderSource {
|
|||
GPUShaderLanguage language;
|
||||
const char *code;
|
||||
size_t size; // size of code in bytes, may be zero for null terminated text
|
||||
// The name of the entry point function for this stage in code. Modules,
|
||||
// such as SPIR-V ones, can contain the entry points of several stages, in
|
||||
// which case the same code is passed for each stage.
|
||||
const char *entry_point;
|
||||
} GPUShaderSource;
|
||||
|
||||
typedef struct GPUUniformBlockInfo {
|
||||
|
|
|
|||
|
|
@ -526,7 +526,11 @@ def add_window(os_window_id: int, tab_id: int, title: str) -> int:
|
|||
pass
|
||||
|
||||
def compile_program(
|
||||
which: int, vertex_shaders: Tuple[str, ...], fragment_shaders: Tuple[str, ...], metadata: Dict[str, Any], allow_recompile: bool = False
|
||||
which: int,
|
||||
vertex_shaders: Tuple[str | bytes, ...], # all str or all bytes
|
||||
fragment_shaders: Tuple[str | bytes, ...],
|
||||
metadata: Dict[str, Any],
|
||||
allow_recompile: bool = False,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
|
|
|||
11
kitty/gpu.c
11
kitty/gpu.c
|
|
@ -691,9 +691,16 @@ compile_gpu_program(int program, const char *vertex, size_t vertex_size, const c
|
|||
(GPUParamField){.name = f->name, .offset = f->offset, .scalar = f->scalar, .components = f->components, .count = f->count, .stride = f->stride};
|
||||
}
|
||||
}
|
||||
PyObject *entry_points = metadata_item(metadata, "entry_points", &PyDict_Type);
|
||||
const char *vertex_entry = entry_points ? metadata_str(entry_points, "vertex") : NULL;
|
||||
const char *fragment_entry = entry_points ? metadata_str(entry_points, "fragment") : NULL;
|
||||
if (!vertex_entry || !fragment_entry) {
|
||||
free_program_params(&pp);
|
||||
return false;
|
||||
}
|
||||
const GPUShaderSource sources[2] = {
|
||||
{.stage = GPU_SHADER_VERTEX, .language = language, .code = vertex, .size = vertex_size},
|
||||
{.stage = GPU_SHADER_FRAGMENT, .language = language, .code = fragment, .size = fragment_size},
|
||||
{.stage = GPU_SHADER_VERTEX, .language = language, .code = vertex, .size = vertex_size, .entry_point = vertex_entry},
|
||||
{.stage = GPU_SHADER_FRAGMENT, .language = language, .code = fragment, .size = fragment_size, .entry_point = fragment_entry},
|
||||
};
|
||||
programs[program] = gpu_pipeline_create(sources, arraysz(sources), gpb, pp.num_blocks, bindings, num_bindings, names, num_names);
|
||||
free_program_params(&pp);
|
||||
|
|
|
|||
|
|
@ -3580,17 +3580,37 @@ transfer_pipeline_to_struct(PyObject *pg, CustomShaderPipeline *p) {
|
|||
|
||||
// Concatenate the tuple of source strings into a single string, the returned
|
||||
// object owns the memory the string points to
|
||||
// Join shader sources, which are either all text (str) or all binary (bytes),
|
||||
// such as SPIR-V, into a single bytes object
|
||||
static PyObject *
|
||||
join_shader_sources(PyObject *sources) {
|
||||
bool all_bytes = true, all_text = true;
|
||||
for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(sources); i++) {
|
||||
if (!PyUnicode_Check(PyTuple_GET_ITEM(sources, i))) {
|
||||
PyErr_SetString(PyExc_TypeError, "shaders must be strings");
|
||||
return NULL;
|
||||
PyObject *x = PyTuple_GET_ITEM(sources, i);
|
||||
all_bytes = all_bytes && PyBytes_Check(x);
|
||||
all_text = all_text && PyUnicode_Check(x);
|
||||
}
|
||||
if (!all_bytes && !all_text) {
|
||||
PyErr_SetString(PyExc_TypeError, "shaders must be all str or all bytes");
|
||||
return NULL;
|
||||
}
|
||||
if (all_bytes) {
|
||||
Py_ssize_t total = 0;
|
||||
for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(sources); i++) total += PyBytes_GET_SIZE(PyTuple_GET_ITEM(sources, i));
|
||||
PyObject *ans = PyBytes_FromStringAndSize(NULL, total);
|
||||
if (!ans) return NULL;
|
||||
char *dest = PyBytes_AS_STRING(ans);
|
||||
for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(sources); i++) {
|
||||
PyObject *x = PyTuple_GET_ITEM(sources, i);
|
||||
memcpy(dest, PyBytes_AS_STRING(x), PyBytes_GET_SIZE(x));
|
||||
dest += PyBytes_GET_SIZE(x);
|
||||
}
|
||||
return ans;
|
||||
}
|
||||
RAII_PyObject(empty, PyUnicode_FromString(""));
|
||||
if (!empty) return NULL;
|
||||
return PyUnicode_Join(empty, sources);
|
||||
RAII_PyObject(text, PyUnicode_Join(empty, sources));
|
||||
return text ? PyUnicode_AsUTF8String(text) : NULL;
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
|
|
@ -3630,10 +3650,8 @@ compile_program(PyObject UNUSED *self, PyObject *args) {
|
|||
RAII_PyObject(fragment_source, join_shader_sources(fragment_shaders));
|
||||
if (!fragment_source) return NULL;
|
||||
Py_ssize_t vsz, fsz;
|
||||
const char *vcode = PyUnicode_AsUTF8AndSize(vertex_source, &vsz);
|
||||
if (!vcode) return NULL;
|
||||
const char *fcode = PyUnicode_AsUTF8AndSize(fragment_source, &fsz);
|
||||
if (!fcode) return NULL;
|
||||
char *vcode, *fcode;
|
||||
if (PyBytes_AsStringAndSize(vertex_source, &vcode, &vsz) != 0 || PyBytes_AsStringAndSize(fragment_source, &fcode, &fsz) != 0) return NULL;
|
||||
if (!compile_gpu_program(which, vcode, (size_t)vsz, fcode, (size_t)fsz, metadata)) return NULL;
|
||||
if (which == CUSTOM_END_PROGRAM) {
|
||||
free_custom_shader_pipeline(&custom_shaders.end);
|
||||
|
|
|
|||
|
|
@ -207,9 +207,11 @@ def variant_name(variant: dict[str, str], default: dict[str, str]) -> str:
|
|||
|
||||
class ShaderTarget(NamedTuple):
|
||||
# The shader target consumed by a GPU driver, identified by strings of the
|
||||
# form family-version, for example glsl-140. Built-in shaders are compiled
|
||||
# for the targets of the drivers being built, at build time, and custom
|
||||
# shaders for the target of the loaded driver, at runtime.
|
||||
# form family-version, for example glsl-140 or spirv-16 (SPIR-V 1.6).
|
||||
# Built-in shaders are compiled for the targets of the drivers being built,
|
||||
# at build time, and custom shaders for the target of the loaded driver, at
|
||||
# runtime. GLSL targets produce one text file per stage, SPIR-V targets a
|
||||
# single binary module containing all the entry points of the program.
|
||||
family: str
|
||||
version: int
|
||||
|
||||
|
|
@ -222,12 +224,22 @@ class ShaderTarget(NamedTuple):
|
|||
family, sep, version = spec.partition('-')
|
||||
if not sep or not version.isdigit():
|
||||
raise ValueError(f'Invalid shader target: {spec!r}')
|
||||
if family not in ('glsl',):
|
||||
if family not in ('glsl', 'spirv'):
|
||||
raise ValueError(f'Unsupported shader target: {spec!r}')
|
||||
return cls(family, int(version))
|
||||
|
||||
@property
|
||||
def is_spirv(self) -> bool:
|
||||
return self.family == 'spirv'
|
||||
|
||||
@property
|
||||
def slangc_args(self) -> list[str]:
|
||||
if self.is_spirv:
|
||||
major, minor = divmod(self.version, 10)
|
||||
# The GLSL 450 profile provides the shader capabilities, such as
|
||||
# texture gathering, without it slangc warns that it implicitly
|
||||
# upgrades the profile
|
||||
return ['-target', 'spirv', '-profile', 'glsl_450', '-capability', f'spirv_{major}_{minor}', '-fvk-use-entrypoint-name']
|
||||
# slangc fails with glsl_140 https://github.com/shader-slang/slang/issues/11898
|
||||
return ['-target', 'glsl', '-profile', f'glsl_{max(150, self.version)}']
|
||||
|
||||
|
|
@ -239,17 +251,22 @@ def current_shader_target() -> ShaderTarget:
|
|||
return ShaderTarget.parse(spec)
|
||||
|
||||
|
||||
def builtin_program(target: ShaderTarget, name: str, variant_name: str = '') -> tuple[str, str, dict[str, Any]]:
|
||||
def builtin_program(target: ShaderTarget, name: str, variant_name: str = '') -> tuple[str | bytes, str | bytes, dict[str, Any]]:
|
||||
# Returns the vertex and fragment shaders and the metadata for a built-in program
|
||||
if variant_name:
|
||||
variant_name = '.' + variant_name
|
||||
base = os.path.join(shaders_dir, target.id, f'{name}{variant_name}')
|
||||
with open(f'{base}.json') as f:
|
||||
metadata = json.load(f)
|
||||
if target.is_spirv:
|
||||
# A single module contains the entry points for all stages
|
||||
with open(f'{base}.spv', 'rb') as b:
|
||||
spirv = b.read()
|
||||
return spirv, spirv, metadata
|
||||
with open(f'{base}.vert.glsl') as f:
|
||||
vert = f.read()
|
||||
with open(f'{base}.frag.glsl') as f:
|
||||
frag = f.read()
|
||||
with open(f'{base}.json') as f:
|
||||
metadata = json.load(f)
|
||||
return vert, frag, metadata
|
||||
|
||||
|
||||
|
|
@ -265,7 +282,7 @@ class LoadShaderPrograms:
|
|||
def __init__(self) -> None:
|
||||
# Per-instance because it is mutable: the sources most recently compiled
|
||||
# into each program, used to skip redundant recompiles.
|
||||
self.last_built_custom_shaders: dict[int, tuple[str, str, dict[str, Any]]] = {}
|
||||
self.last_built_custom_shaders: dict[int, tuple[str | bytes, str | bytes, dict[str, Any]]] = {}
|
||||
# Errors from the most recent attempt to build the shaders named by the
|
||||
# custom_shaders option. A failure here disables the shader silently as
|
||||
# far as rendering is concerned, so the Boss displays these to the user.
|
||||
|
|
@ -386,7 +403,7 @@ class LoadShaderPrograms:
|
|||
else:
|
||||
try:
|
||||
pipeline = merge_pipelines(slot_pipelines)
|
||||
vert, frag, metadata = build_custom_shader_pipeline_glsl(pipeline, current_shader_target())
|
||||
vert, frag, metadata = build_custom_shader_pipeline(pipeline, current_shader_target())
|
||||
# print(vert, file=open('/tmp/sample.vert', 'w'))
|
||||
# print(frag, file=open('/tmp/sample.frag', 'w'))
|
||||
except FileNotFoundError as e:
|
||||
|
|
@ -1053,16 +1070,19 @@ def serialize_source_metadata(sources: dict[str, SlangFile], dest_dir: str) -> N
|
|||
f.write(json.dumps(sfile.asdict(skip_source=True), indent=2, sort_keys=True))
|
||||
|
||||
|
||||
def commands_to_compile_to_spirv(sources: dict[str, SlangFile], build_dir: str, dest_dir: str, built_files: list[str]) -> Iterator[Command]:
|
||||
# glsl 450 is vulkan 1.1 and spirv 1.3 released 2008
|
||||
base_cmd = ['-target', 'spirv', '-profile', 'glsl_450', '-capability', 'vk_mem_model', '-fvk-use-entrypoint-name']
|
||||
for base_dest, base_build, slang_module, scmd, sfile in iter_entry_point_shaders(sources, build_dir, dest_dir):
|
||||
def commands_to_compile_to_spirv(
|
||||
sources: dict[str, SlangFile], build_dir: str, dest_dir: str, target: ShaderTarget, built_files: list[str]
|
||||
) -> Iterator[Command]:
|
||||
# One module containing all the entry points of each program
|
||||
os.makedirs(os.path.join(dest_dir, target.id), exist_ok=True)
|
||||
os.makedirs(os.path.join(build_dir, target.id), exist_ok=True)
|
||||
for base_dest, base_build, slang_module, scmd, sfile in iter_entry_point_shaders(sources, build_dir, os.path.join(dest_dir, target.id)):
|
||||
for x in sfile.specializations:
|
||||
cmd = list(scmd)
|
||||
dest = f'{base_dest}.{x.name}.spv' if x.name else f'{base_dest}.spv'
|
||||
dest = f'{base_dest}{x.filename_insert}.spv' if x.name else f'{base_dest}.spv'
|
||||
if x.variables:
|
||||
cmd.insert(-1, f'{base_build}{x.filename_insert}.slang-module')
|
||||
cmd += base_cmd + ['-o', dest, '-reflection-json', dest + '.json']
|
||||
cmd += target.slangc_args + ['-o', dest, '-reflection-json', reflection_path(target, build_dir, dest)]
|
||||
output_mtime = safe_mtime(dest)
|
||||
module_mtime = os.path.getmtime(slang_module)
|
||||
needs_build = output_mtime < module_mtime
|
||||
|
|
@ -1071,6 +1091,25 @@ def commands_to_compile_to_spirv(sources: dict[str, SlangFile], build_dir: str,
|
|||
yield Command(needs_build, f'Linking |{os.path.basename(dest)}| ...', cmd)
|
||||
|
||||
|
||||
def write_spirv_metadata(paths: Iterable[str], target: ShaderTarget, reflection_dir: str) -> None:
|
||||
"Write the program metadata for compiled SPIR-V modules"
|
||||
for path in paths:
|
||||
with open(reflection_path(target, reflection_dir, path)) as f:
|
||||
reflection = json.load(f)
|
||||
try:
|
||||
metadata = program_metadata(target, [reflection], GLSLMetadata())
|
||||
except Exception:
|
||||
os.unlink(path)
|
||||
raise
|
||||
write_if_changed(path.removesuffix('.spv') + '.json', json.dumps(metadata, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
def spirv_validation_command(target: ShaderTarget, path: str) -> list[str]:
|
||||
# The Vulkan version whose environment the SPIR-V is validated against
|
||||
env = {16: 'vulkan1.3', 15: 'vulkan1.2', 13: 'vulkan1.1'}.get(target.version, 'vulkan1.3')
|
||||
return ['spirv-val', '--target-env', env, path]
|
||||
|
||||
|
||||
# GLSL {{{
|
||||
|
||||
|
||||
|
|
@ -1424,8 +1463,12 @@ def program_metadata(target: ShaderTarget, reflections: Sequence[dict[str, Any]]
|
|||
# parameters of the program and the pipeline constant blocks. kitty sets their contents with set_params()
|
||||
# rather than via uniform buffers.
|
||||
param_blocks: dict[str, dict[str, Any]] = {}
|
||||
# Uniform blocks that are emitted as uniform blocks rather than as loose uniforms by the target
|
||||
# Uniform blocks that are emitted as uniform blocks rather than as loose
|
||||
# uniforms by the target. For SPIR-V all of them, other than the pipeline
|
||||
# constant blocks, and the reflection data lists only those present in the
|
||||
# compiled code.
|
||||
real_blocks = set(native.uniform_struct_names)
|
||||
entry_points: dict[str, str] = {}
|
||||
for refl in reflections:
|
||||
for p in refl['parameters']:
|
||||
t = p['type']
|
||||
|
|
@ -1453,7 +1496,7 @@ def program_metadata(target: ShaderTarget, reflections: Sequence[dict[str, Any]]
|
|||
'size': evl['binding']['size'],
|
||||
'fields': [param_field(f['name'], f['binding']['offset'], f['type']) for f in et['fields']],
|
||||
}
|
||||
elif tname in real_blocks:
|
||||
elif tname in real_blocks or target.is_spirv:
|
||||
arrays = {}
|
||||
for f in et['fields']:
|
||||
ft = f['type']
|
||||
|
|
@ -1468,6 +1511,8 @@ def program_metadata(target: ShaderTarget, reflections: Sequence[dict[str, Any]]
|
|||
raise ValueError(f'Unsupported shader parameter: {p["name"]} of kind: {t["kind"]}')
|
||||
for ep in refl['entryPoints']:
|
||||
stage = ep['stage']
|
||||
# GLSL shaders always use main(), so the name is needed only for other targets
|
||||
entry_points[stage] = 'main' if target.family == 'glsl' else ep['name']
|
||||
for p in ep.get('parameters', ()):
|
||||
b = p.get('binding') or {}
|
||||
match b.get('kind'):
|
||||
|
|
@ -1484,6 +1529,7 @@ def program_metadata(target: ShaderTarget, reflections: Sequence[dict[str, Any]]
|
|||
native_names.update(native.native_inputs)
|
||||
return {
|
||||
'target': target.id,
|
||||
'entry_points': entry_points,
|
||||
'inputs': inputs,
|
||||
'textures': textures,
|
||||
'blocks': blocks,
|
||||
|
|
@ -1575,18 +1621,23 @@ def compile_builtin_shaders(build_dir: str, dest_dir: str, parallel_run: Paralle
|
|||
parallel_run(commands_to_compile_dir_to_ir(layer_sources, src_dir, build_dir))
|
||||
# Create the specializations
|
||||
parallel_run(create_specialisations(source_tree, build_dir))
|
||||
# Now Vulkan shaders
|
||||
built_spirv_files: list[str] = []
|
||||
spirv_commands = commands_to_compile_to_spirv(source_tree, build_dir, dest_dir, built_spirv_files)
|
||||
# Now the shaders for the targets of the GPU drivers
|
||||
built_glsl_files: dict[ShaderTarget, list[str]] = {}
|
||||
built_files: dict[ShaderTarget, list[str]] = {}
|
||||
target_commands: list[Iterator[Command]] = []
|
||||
for target in targets:
|
||||
built_glsl_files[target] = []
|
||||
target_commands.append(commands_to_compile_to_glsl(source_tree, build_dir, dest_dir, target, built_glsl_files[target]))
|
||||
built_files[target] = []
|
||||
if target.is_spirv:
|
||||
target_commands.append(commands_to_compile_to_spirv(source_tree, build_dir, dest_dir, target, built_files[target]))
|
||||
else:
|
||||
target_commands.append(commands_to_compile_to_glsl(source_tree, build_dir, dest_dir, target, built_files[target]))
|
||||
# Now run all commands
|
||||
parallel_run(chain(spirv_commands, *target_commands))
|
||||
for target, built in built_glsl_files.items():
|
||||
parallel_run(chain(*target_commands))
|
||||
for target, built in built_files.items():
|
||||
if target.is_spirv:
|
||||
write_spirv_metadata(built, target, build_dir)
|
||||
if shutil.which('spirv-val'):
|
||||
parallel_run((True, f'Validating |{os.path.basename(x)}| ...', spirv_validation_command(target, x)) for x in built)
|
||||
continue
|
||||
fixup_opengl_files(built, target, build_dir)
|
||||
if shutil.which('glslangValidator'):
|
||||
from kitty.shaders.validate_shaders import validation_command_for_file
|
||||
|
|
@ -2311,9 +2362,14 @@ float4 fmain_wrap(float2 texcoord : TEXCOORD) : SV_Target {
|
|||
""".replace('MODULE', slot.replace('-', '_')).encode()
|
||||
|
||||
|
||||
def build_custom_shader_pipeline_glsl(
|
||||
def build_custom_shader_pipeline(
|
||||
pipeline: Pipeline, target: ShaderTarget, cache_dir: str = '', invocation_tracker: set[tuple[str, ...]] | None = None
|
||||
) -> tuple[str, str, dict[str, Any]]:
|
||||
) -> tuple[str | bytes, str | bytes, dict[str, Any]]:
|
||||
"""
|
||||
Build the program for a custom shader pipeline, for the specified shader
|
||||
target. Returns the vertex and fragment shaders, which are the same module
|
||||
for SPIR-V, and the program metadata.
|
||||
"""
|
||||
import kitty.constants as kc
|
||||
|
||||
cache_dir = os.path.join(cache_dir or kc.cache_dir(), 'shaders')
|
||||
|
|
@ -2321,63 +2377,66 @@ def build_custom_shader_pipeline_glsl(
|
|||
if invocation_tracker is None:
|
||||
invocation_tracker = set()
|
||||
|
||||
def run(cmds: Sequence[tuple[str, list[str]]], src: bytes) -> None:
|
||||
procs = []
|
||||
try:
|
||||
for name, cmd in cmds:
|
||||
invocation_tracker.add(tuple(cmd))
|
||||
p = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
|
||||
procs.append((name, cmd, p))
|
||||
assert p.stdin is not None
|
||||
p.stdin.write(src), p.stdin.close()
|
||||
for name, cmd, p in procs:
|
||||
assert p.stderr is not None
|
||||
if (rc := p.wait()) != 0:
|
||||
raise SlangFailed(name, subprocess.CompletedProcess(cmd, rc, stderr=p.stderr.read()))
|
||||
finally:
|
||||
for _, _, p in procs:
|
||||
if p.stderr is not None:
|
||||
p.stderr.close()
|
||||
|
||||
with lock_with_file(
|
||||
os.path.join(cache_dir, 'lock'),
|
||||
):
|
||||
pipeline_ir = build_custom_shader_pipeline_ir(pipeline, cache_dir, invocation_tracker)
|
||||
import_dirs, slang_module_path = pipeline_ir.import_dirs, pipeline_ir.module_path
|
||||
glsl_dir = os.path.join(os.path.dirname(os.path.dirname(slang_module_path)), target.id)
|
||||
os.makedirs(glsl_dir, exist_ok=True)
|
||||
os.makedirs(os.path.join(glsl_dir, target.id), exist_ok=True)
|
||||
out_dir = os.path.join(os.path.dirname(os.path.dirname(slang_module_path)), target.id)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
os.makedirs(os.path.join(out_dir, target.id), exist_ok=True)
|
||||
module_mtime = safe_mtime(slang_module_path)
|
||||
slot = pipeline['slot']
|
||||
vertex = os.path.join(glsl_dir, f'{slot}.vert.glsl')
|
||||
fragment = os.path.join(glsl_dir, f'{slot}.frag.glsl')
|
||||
metadata = os.path.join(glsl_dir, f'{slot}.json')
|
||||
if module_mtime > safe_mtime(metadata):
|
||||
inc = []
|
||||
for x in import_dirs:
|
||||
inc.extend(('-I', x))
|
||||
cmd = (
|
||||
list(slangc())
|
||||
+ inc
|
||||
+ [
|
||||
'-warnings-as-errors',
|
||||
'all',
|
||||
'-line-directive-mode',
|
||||
'none',
|
||||
'-lang',
|
||||
'slang',
|
||||
]
|
||||
+ target.slangc_args
|
||||
)
|
||||
vrefl, frefl = reflection_path(target, glsl_dir, vertex), reflection_path(target, glsl_dir, fragment)
|
||||
vcmd = cmd + ['-stage', 'vertex', '-entry', 'vmain_wrap', '-o', vertex, '-reflection-json', vrefl, '--', '-']
|
||||
fcmd = cmd + ['-stage', 'fragment', '-entry', 'fmain_wrap', '-o', fragment, '-reflection-json', frefl, '--', '-']
|
||||
src = module_wrapper_for_slot(slot)
|
||||
invocation_tracker.add(tuple(vcmd))
|
||||
invocation_tracker.add(tuple(fcmd))
|
||||
v = subprocess.Popen(vcmd, stderr=subprocess.PIPE, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
|
||||
f = subprocess.Popen(fcmd, stderr=subprocess.PIPE, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
|
||||
assert v.stdin is not None and f.stdin is not None
|
||||
assert v.stderr is not None and f.stderr is not None
|
||||
v.stdin.write(src), v.stdin.close()
|
||||
f.stdin.write(src), f.stdin.close()
|
||||
try:
|
||||
if (rc := v.wait()) != 0:
|
||||
raise SlangFailed(f'{slot}.vert.glsl', subprocess.CompletedProcess(vcmd, rc, stderr=v.stderr.read()))
|
||||
if (rc := f.wait()) != 0:
|
||||
raise SlangFailed(f'{slot}.frag.glsl', subprocess.CompletedProcess(fcmd, rc, stderr=f.stderr.read()))
|
||||
finally:
|
||||
v.stderr.close()
|
||||
f.stderr.close()
|
||||
fixup_opengl_files((fragment, vertex), target, glsl_dir)
|
||||
with open(vertex) as vf, open(fragment) as ff, open(metadata) as mf:
|
||||
metadata = os.path.join(out_dir, f'{slot}.json')
|
||||
inc = []
|
||||
for x in import_dirs:
|
||||
inc.extend(('-I', x))
|
||||
cmd = list(slangc()) + inc + ['-warnings-as-errors', 'all', '-line-directive-mode', 'none', '-lang', 'slang'] + target.slangc_args
|
||||
src = module_wrapper_for_slot(slot)
|
||||
if target.is_spirv:
|
||||
# A single module containing both entry points
|
||||
module = os.path.join(out_dir, f'{slot}.spv')
|
||||
if module_mtime > safe_mtime(metadata):
|
||||
run(((f'{slot}.spv', cmd + ['-o', module, '-reflection-json', reflection_path(target, out_dir, module), '--', '-']),), src)
|
||||
write_spirv_metadata((module,), target, out_dir)
|
||||
with open(module, 'rb') as sf:
|
||||
vcode: str | bytes = sf.read()
|
||||
fcode = vcode
|
||||
else:
|
||||
vertex = os.path.join(out_dir, f'{slot}.vert.glsl')
|
||||
fragment = os.path.join(out_dir, f'{slot}.frag.glsl')
|
||||
if module_mtime > safe_mtime(metadata):
|
||||
vrefl, frefl = reflection_path(target, out_dir, vertex), reflection_path(target, out_dir, fragment)
|
||||
vcmd = cmd + ['-stage', 'vertex', '-entry', 'vmain_wrap', '-o', vertex, '-reflection-json', vrefl, '--', '-']
|
||||
fcmd = cmd + ['-stage', 'fragment', '-entry', 'fmain_wrap', '-o', fragment, '-reflection-json', frefl, '--', '-']
|
||||
run(((f'{slot}.vert.glsl', vcmd), (f'{slot}.frag.glsl', fcmd)), src)
|
||||
fixup_opengl_files((fragment, vertex), target, out_dir)
|
||||
with open(vertex) as vf, open(fragment) as ff:
|
||||
vcode, fcode = vf.read(), ff.read()
|
||||
with open(metadata) as mf:
|
||||
m = json.load(mf)
|
||||
m['pipeline'] = pipeline
|
||||
m['has_pointer_map'] = bool(pipeline_ir.pointer_map_shaders)
|
||||
m['pointer_map_groups'] = sorted({g for g, _ in pipeline_ir.pointer_map_shaders})
|
||||
return vf.read(), ff.read(), m
|
||||
m['pipeline'] = pipeline
|
||||
m['has_pointer_map'] = bool(pipeline_ir.pointer_map_shaders)
|
||||
m['pointer_map_groups'] = sorted({g for g, _ in pipeline_ir.pointer_map_shaders})
|
||||
return vcode, fcode, m
|
||||
|
||||
|
||||
def clear_caches() -> None:
|
||||
|
|
@ -2391,9 +2450,10 @@ def test_slang_build() -> None:
|
|||
|
||||
if shutil.which(slangc()[0]) is None:
|
||||
raise AssertionError(f'The shader slang compiler ({slangc()[0]}) not in PATH: {os.environ.get("PATH")}')
|
||||
q = os.path.join(shaders_dir, 'graphics.spv')
|
||||
if not os.path.isfile(q):
|
||||
raise AssertionError(f'The compiled graphics shader {q} does not exist')
|
||||
# The built-in shaders are compiled for the shader target of every GPU driver in the build
|
||||
targets = [x for x in os.listdir(shaders_dir) if os.path.isfile(os.path.join(shaders_dir, x, 'graphics.json'))]
|
||||
if not targets:
|
||||
raise AssertionError(f'The compiled graphics shader does not exist for any shader target in {shaders_dir}')
|
||||
if not get_shader_src('graphics'):
|
||||
raise AssertionError('Could not load graphics.slang shader source')
|
||||
src = b"""
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from kitty.shaders.slang import (
|
|||
SlangShaderQuery,
|
||||
Stage,
|
||||
apply_pipeline_specializations,
|
||||
build_custom_shader_pipeline_glsl,
|
||||
build_custom_shader_pipeline,
|
||||
build_import_graph,
|
||||
cell_variant,
|
||||
cell_variations,
|
||||
|
|
@ -39,6 +39,7 @@ from kitty.shaders.slang import (
|
|||
pointer_map_shaders,
|
||||
program_metadata,
|
||||
slangc_version,
|
||||
spirv_validation_command,
|
||||
text_contrast_is_identity,
|
||||
topological_layers,
|
||||
topological_sort,
|
||||
|
|
@ -374,7 +375,7 @@ class TestSlang(BaseTest):
|
|||
self.ae(pipeline['groups'][0]['animation_step'], 0)
|
||||
with (
|
||||
patch('kitty.shaders.slang.parse_pipeline', return_value=pipeline),
|
||||
patch('kitty.shaders.slang.build_custom_shader_pipeline_glsl', return_value=('vertex', 'fragment', {})),
|
||||
patch('kitty.shaders.slang.build_custom_shader_pipeline', return_value=('vertex', 'fragment', {})),
|
||||
patch('kitty.shaders.slang.current_shader_target', return_value=driver_shader_target()),
|
||||
patch('kitty.shaders.slang.compile_program') as compile_program,
|
||||
):
|
||||
|
|
@ -851,7 +852,7 @@ float f() { static const float SCAN = 2; return SCAN; }
|
|||
self.assertEqual(p['groups'][0]['shaders'], ('sample',))
|
||||
self.assertEqual(p['groups'][1]['shaders'], ('sample',))
|
||||
|
||||
def test_build_custom_shader_pipeline_glsl(self):
|
||||
def test_build_custom_shader_pipeline(self):
|
||||
if not shutil.which(slangc()[0]):
|
||||
self.skipTest(f'slangc ({slangc()[0]}) not found in PATH')
|
||||
|
||||
|
|
@ -867,9 +868,9 @@ float f() { static const float SCAN = 2; return SCAN; }
|
|||
'test',
|
||||
)
|
||||
try:
|
||||
vert_src, frag_src, metadata = build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir)
|
||||
vert_src, frag_src, metadata = build_custom_shader_pipeline(p, driver_shader_target(), cache_dir=cache_dir)
|
||||
invocation_tracker = set()
|
||||
build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir, invocation_tracker=invocation_tracker)
|
||||
build_custom_shader_pipeline(p, driver_shader_target(), cache_dir=cache_dir, invocation_tracker=invocation_tracker)
|
||||
finally:
|
||||
clear_caches()
|
||||
|
||||
|
|
@ -900,7 +901,7 @@ float f() { static const float SCAN = 2; return SCAN; }
|
|||
'test',
|
||||
)
|
||||
try:
|
||||
_, pm_frag_src, pm_metadata = build_custom_shader_pipeline_glsl(p, driver_shader_target(), cache_dir=cache_dir)
|
||||
_, pm_frag_src, pm_metadata = build_custom_shader_pipeline(p, driver_shader_target(), cache_dir=cache_dir)
|
||||
finally:
|
||||
clear_caches()
|
||||
self.assertTrue(pm_metadata['has_pointer_map'])
|
||||
|
|
@ -965,34 +966,53 @@ float f() { static const float SCAN = 2; return SCAN; }
|
|||
if entry.name.endswith('.slang') and entry.name[: -len('.slang')] not in _SUPPORT_SHADER_NAMES
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as cache_dir:
|
||||
failures: list[str] = []
|
||||
clear_caches()
|
||||
for name in shader_names:
|
||||
pipeline = parse_pipeline_definition(
|
||||
['startgroup', f'shaders {name}', 'endgroup'],
|
||||
name,
|
||||
)
|
||||
try:
|
||||
vert_src, frag_src, _ = build_custom_shader_pipeline_glsl(pipeline, driver_shader_target(), cache_dir=cache_dir)
|
||||
except Exception as e:
|
||||
failures.append(f'{name}: {e}')
|
||||
continue
|
||||
if not vert_src:
|
||||
failures.append(f'{name}: empty vertex GLSL')
|
||||
if not frag_src:
|
||||
failures.append(f'{name}: empty fragment GLSL')
|
||||
clear_caches()
|
||||
# Builtin pipelines can override constants with var and chain
|
||||
# shaders, so compile them as well
|
||||
pipeline_names = sorted(entry.name[: -len('.pipeline')] for entry in pkg.iterdir() if entry.name.endswith('.pipeline'))
|
||||
for name in pipeline_names:
|
||||
try:
|
||||
pipeline = parse_pipeline_definition(get_custom_pipeline_src(name).decode().splitlines(), name)
|
||||
build_custom_shader_pipeline_glsl(pipeline, driver_shader_target(), cache_dir=cache_dir)
|
||||
except Exception as e:
|
||||
failures.append(f'{name}.pipeline: {e}')
|
||||
clear_caches()
|
||||
# Compile for the target of the loaded driver and for SPIR-V, used by
|
||||
# the Vulkan driver, which may not be loadable on the test machine
|
||||
targets = [driver_shader_target()]
|
||||
if not targets[0].is_spirv:
|
||||
targets.append(ShaderTarget('spirv', 16))
|
||||
failures: list[str] = []
|
||||
|
||||
def check(name: str, target: ShaderTarget, vert_src: str | bytes, frag_src: str | bytes, tdir: str) -> None:
|
||||
if not vert_src:
|
||||
failures.append(f'{name}: empty vertex shader for {target.id}')
|
||||
if not frag_src:
|
||||
failures.append(f'{name}: empty fragment shader for {target.id}')
|
||||
if target.is_spirv and isinstance(vert_src, bytes) and shutil.which('spirv-val'):
|
||||
path = os.path.join(tdir, 'check.spv')
|
||||
with open(path, 'wb') as f:
|
||||
f.write(vert_src)
|
||||
cp = subprocess.run(spirv_validation_command(target, path), capture_output=True)
|
||||
if cp.returncode != 0:
|
||||
failures.append(f'{name}: invalid SPIR-V: {cp.stderr.decode()}{cp.stdout.decode()}')
|
||||
|
||||
for target in targets:
|
||||
with tempfile.TemporaryDirectory() as cache_dir:
|
||||
clear_caches()
|
||||
for name in shader_names:
|
||||
pipeline = parse_pipeline_definition(
|
||||
['startgroup', f'shaders {name}', 'endgroup'],
|
||||
name,
|
||||
)
|
||||
try:
|
||||
vert_src, frag_src, _ = build_custom_shader_pipeline(pipeline, target, cache_dir=cache_dir)
|
||||
except Exception as e:
|
||||
failures.append(f'{name} for {target.id}: {e}')
|
||||
continue
|
||||
check(name, target, vert_src, frag_src, cache_dir)
|
||||
clear_caches()
|
||||
# Builtin pipelines can override constants with var and chain
|
||||
# shaders, so compile them as well
|
||||
pipeline_names = sorted(entry.name[: -len('.pipeline')] for entry in pkg.iterdir() if entry.name.endswith('.pipeline'))
|
||||
for name in pipeline_names:
|
||||
try:
|
||||
pipeline = parse_pipeline_definition(get_custom_pipeline_src(name).decode().splitlines(), name)
|
||||
vert_src, frag_src, _ = build_custom_shader_pipeline(pipeline, target, cache_dir=cache_dir)
|
||||
except Exception as e:
|
||||
failures.append(f'{name}.pipeline for {target.id}: {e}')
|
||||
continue
|
||||
check(f'{name}.pipeline', target, vert_src, frag_src, cache_dir)
|
||||
clear_caches()
|
||||
|
||||
if failures:
|
||||
self.fail('Custom shader compilation failures:\n' + '\n'.join(failures))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue