diff --git a/comparative_benchmark.py b/comparative_benchmark.py new file mode 100755 index 000000000..21c745ec2 --- /dev/null +++ b/comparative_benchmark.py @@ -0,0 +1,1031 @@ +#!/usr/bin/env python +# License: GPLv3 Copyright: 2026, Kovid Goyal + +""" +Run the kitty benchmark kitten in all installed terminals from a fixed set and +present the results for comparison. Linux only. + +Every terminal is run with its default configuration, except for the font +family, font size and window size, which are made the same for all terminals. +User configuration is ignored by pointing all the XDG base directories at +a temporary directory. The terminals are pinned to the fast cores on machines +with heterogeneous cores and the CPU frequency governor is set to performance +for the duration of the run, if possible. +""" + +import argparse +import json +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Iterator, Sequence +from contextlib import ExitStack, contextmanager, suppress +from dataclasses import dataclass, field +from typing import Any + +BASE = os.path.dirname(os.path.abspath(__file__)) +CPU_SYSFS = '/sys/devices/system/cpu' +READY_FILE, GO_FILE, DONE_FILE, PROGRESS_FILE = 'ready.json', 'go', 'done.json', 'progress' +ENV_PREFIXES_TO_REMOVE = ( + 'KITTY_', + 'TMUX', + 'WEZTERM_', + 'GHOSTTY_', + 'ALACRITTY_', + 'KONSOLE_', + 'VTE_', + 'TERM_PROGRAM', + 'TERMINFO', + 'COLORTERM', + 'WINDOWID', + 'XTERM_', + 'PTYXIS_', + 'SHELL_SESSION_ID', +) + + +@dataclass(frozen=True) +class Settings: + font_family: str + font_size: float + columns: int + lines: int + window_width: int + window_height: int + + +@dataclass +class Terminal: + name: str + executable: str + # Build the command line to run cmd in the terminal, given the per-run + # directory for config files and the settings + build_cmd: Callable[[str, list[str], Settings], list[str]] + version_cmd: list[str] = field(default_factory=list) + # Run after the terminal is done, to cleanup any left over processes + cleanup: Callable[[str], None] | None = None + # Needs an X11 display, a private rootful Xwayland is started if DISPLAY is not set + needs_x11: bool = False + # Needs a D-Bus session bus and a systemd user instance (to spawn processes in scopes) + needs_session: bool = False + + +# Terminal definitions {{{ + + +def write_file(path: str, text: str) -> str: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write(text) + return path + + +def fmt_size(x: float) -> str: + return str(int(x)) if x == int(x) else str(x) + + +def toml_str(x: str) -> str: + return json.dumps(x) + + +def xterm_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return ['xterm', '-fa', s.font_family, '-fs', fmt_size(s.font_size), '-geometry', f'{s.columns}x{s.lines}', '-e'] + cmd + + +def foot_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file(os.path.join(d, 'foot.ini'), f'font={s.font_family}:size={fmt_size(s.font_size)}\ninitial-window-size-chars={s.columns}x{s.lines}\n') + return ['foot', '--config', conf] + cmd + + +def kitty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file( + os.path.join(d, 'kitty.conf'), + '\n'.join( + ( + f'font_family family="{s.font_family}"', + f'font_size {fmt_size(s.font_size)}', + 'remember_window_size no', + f'initial_window_width {s.columns}c', + f'initial_window_height {s.lines}c', + '', + ) + ), + ) + return [kitty_exe(), '--config', conf] + cmd + + +def ghostty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file( + os.path.join(d, 'ghostty.conf'), + '\n'.join( + ( + f'font-family = {s.font_family}', + f'font-size = {fmt_size(s.font_size)}', + f'window-width = {s.columns}', + f'window-height = {s.lines}', + # Needed so that the window is created in a new process that exits when the command finishes + 'gtk-single-instance = false', + 'confirm-close-surface = false', + 'quit-after-last-window-closed = true', + '', + ) + ), + ) + return ['ghostty', '--config-default-files=false', f'--config-file={conf}', '-e'] + cmd + + +def alacritty_config(d: str, s: Settings) -> str: + return write_file( + os.path.join(d, 'alacritty.toml'), + '\n'.join( + ( + '[font]', + f'size = {float(s.font_size)}', + f'normal = {{ family = {toml_str(s.font_family)} }}', + '[window]', + f'dimensions = {{ columns = {s.columns}, lines = {s.lines} }}', + '', + ) + ), + ) + + +def alacritty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return ['alacritty', '--config-file', alacritty_config(d, s), '-e'] + cmd + + +def wezterm_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file( + os.path.join(d, 'wezterm.lua'), + '\n'.join( + ( + 'local wezterm = require "wezterm"', + 'local config = wezterm.config_builder()', + f'config.font = wezterm.font({json.dumps(s.font_family)})', + f'config.font_size = {float(s.font_size)}', + f'config.initial_cols = {s.columns}', + f'config.initial_rows = {s.lines}', + 'return config', + '', + ) + ), + ) + return ['wezterm', '--config-file', conf, 'start', '--always-new-process', '--'] + cmd + + +def konsole_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return [ + 'konsole', + '--separate', + '--builtin-profile', + '-p', + f'Font={s.font_family},{fmt_size(s.font_size)}', + '-p', + f'TerminalColumns={s.columns}', + '-p', + f'TerminalRows={s.lines}', + '-e', + ] + cmd + + +def ptyxis_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + # ptyxis uses GSettings, the run environment sets GSETTINGS_BACKEND=keyfile + # with XDG_CONFIG_HOME pointing into d, so this is where its settings are read from + write_file( + os.path.join(d, 'xdg', 'config', 'glib-2.0', 'settings', 'keyfile'), + '\n'.join( + ( + '[org/gnome/Ptyxis]', + 'use-system-font=false', + f"font-name='{s.font_family} {fmt_size(s.font_size)}'", + f'default-columns=uint32 {s.columns}', + f'default-rows=uint32 {s.lines}', + 'restore-window-size=false', + 'restore-session=false', + '', + ) + ), + ) + return ['ptyxis', '--standalone', '--'] + cmd + + +TMUX_SOCKET = 'kitty-comparative-benchmark' + + +def alacritty_tmux_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return ['alacritty', '--config-file', alacritty_config(d, s), '-e', 'tmux', '-L', TMUX_SOCKET, '-f', '/dev/null', 'new-session', '--'] + cmd + + +def tmux_cleanup(d: str) -> None: + subprocess.run(['tmux', '-L', TMUX_SOCKET, 'kill-server'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def kitty_exe() -> str: + q = os.path.join(BASE, 'kitty', 'launcher', 'kitty') + return q if os.access(q, os.X_OK) else 'kitty' + + +def kitten_exe() -> str: + q = os.path.join(BASE, 'kitty', 'launcher', 'kitten') + if os.access(q, os.X_OK): + return q + ans = shutil.which('kitten') + if not ans: + raise SystemExit('Could not find the kitten executable, build kitty first with: make debug') + return ans + + +def all_terminals() -> list[Terminal]: + return [ + Terminal('xterm', 'xterm', xterm_cmd, ['xterm', '-version'], needs_x11=True), + Terminal('foot', 'foot', foot_cmd, ['foot', '--version']), + Terminal('kitty', kitty_exe(), kitty_cmd, [kitty_exe(), '--version']), + Terminal('ghostty', 'ghostty', ghostty_cmd, ['ghostty', '--version']), + Terminal('alacritty', 'alacritty', alacritty_cmd, ['alacritty', '--version']), + Terminal('wezterm', 'wezterm', wezterm_cmd, ['wezterm', '--version']), + Terminal('konsole', 'konsole', konsole_cmd, ['konsole', '--version']), + Terminal('ptyxis', 'ptyxis', ptyxis_cmd, ['ptyxis', '--version'], needs_session=True), + Terminal('alacritty+tmux', 'alacritty', alacritty_tmux_cmd, ['tmux', '-V'], tmux_cleanup), + ] + + +def is_installed(t: Terminal) -> bool: + if t.name == 'alacritty+tmux' and not shutil.which('tmux'): + return False + return bool(shutil.which(t.executable)) + + +def terminal_version(t: Terminal) -> str: + if not t.version_cmd: + return '' + try: + cp = subprocess.run(t.version_cmd, capture_output=True, text=True, timeout=10) + except (OSError, subprocess.TimeoutExpired) as e: + return f'unknown ({e})' + lines = [x.strip() for x in (cp.stdout or cp.stderr).splitlines() if x.strip()] + ans = lines[0] if lines else 'unknown' + if t.name == 'alacritty+tmux': + a = Terminal('alacritty', 'alacritty', alacritty_cmd, ['alacritty', '--version']) + ans = f'{terminal_version(a)} + {ans}' + return ans + + +# }}} + + +# CPU setup {{{ + + +def read_sysfs(path: str) -> str: + try: + with open(path) as f: + return f.read().strip() + except OSError: + return '' + + +def online_cpus() -> list[int]: + return sorted(os.sched_getaffinity(0)) + + +# Per CPU sysfs files that indicate relative CPU performance: cpu_capacity (ARM +# big.LITTLE, constant on x86), CPPC highest perf (AMD and Intel) and max frequency +PERF_METRICS = ('cpu_capacity', 'cpufreq/amd_pstate_highest_perf', 'acpi_cppc/highest_perf', 'cpufreq/cpuinfo_max_freq') + + +def physical_core_key(cpu: int) -> tuple[str, str]: + base = f'{CPU_SYSFS}/cpu{cpu}/topology' + return read_sysfs(f'{base}/physical_package_id'), read_sysfs(f'{base}/core_id') + + +def detect_fast_cpus(one_thread_per_core: bool) -> tuple[list[int], str]: + cpus = online_cpus() + intel_core = read_sysfs('/sys/devices/cpu_core/cpus') + fast: list[int] + if intel_core: # Intel hybrid, P-cores + fast = [c for c in parse_cpu_list(intel_core) if c in cpus] + how = 'Intel hybrid P-cores' + else: + for q in PERF_METRICS: + vals = {c: read_sysfs(f'{CPU_SYSFS}/cpu{c}/{q}') for c in cpus} + if not all(v.isdigit() for v in vals.values()): + continue + metrics = {c: int(v) for c, v in vals.items()} + top = max(metrics.values()) + # Ignore small per-core variations from binning, e.g. AMD preferred cores + fast = [c for c in cpus if metrics[c] >= 0.9 * top] + if len(fast) < len(cpus): + how = f'heterogeneous cores, using the fastest as per {q}' + break + else: + return cpus, 'homogeneous cores, using all CPUs' + if one_thread_per_core: + seen: set[tuple[str, str]] = set() + ans = [] + for c in fast: + k = physical_core_key(c) + if k not in seen: + seen.add(k) + ans.append(c) + fast, how = ans, how + ', one thread per core' + return fast, how + + +def parse_cpu_list(spec: str) -> list[int]: + ans: list[int] = [] + for part in spec.split(','): + part = part.strip() + if not part: + continue + if '-' in part: + a, b = part.split('-', 1) + ans.extend(range(int(a), int(b) + 1)) + else: + ans.append(int(part)) + return ans + + +def write_sysfs_values(values: dict[str, str]) -> str: + """Write the values, using sudo if needed. Returns an error message on failure.""" + if not values: + return '' + try: + for path, val in values.items(): + with open(path, 'w') as f: + f.write(val) + return '' + except PermissionError: + pass + except OSError as e: + return str(e) + script = '\n'.join(f'echo {shlex.quote(v)} > {shlex.quote(p)}' for p, v in values.items()) + cp = subprocess.run(['sudo', '-n', 'sh', '-c', script], capture_output=True, text=True) + if cp.returncode == 0: + return '' + if sys.stdin.isatty(): + print('Root access is needed to change the CPU frequency governor, asking sudo for your password', file=sys.stderr) + cp = subprocess.run(['sudo', 'sh', '-c', script], stdout=subprocess.PIPE, text=True) + if cp.returncode == 0: + return '' + return 'permission denied: not root and passwordless sudo is not available' + + +@contextmanager +def performance_governor(enabled: bool) -> Iterator[str]: + """Yields a description of what was done""" + if not enabled: + yield 'not changed (disabled on command line)' + return + original: dict[str, str] = {} + for cpu in online_cpus(): + p = f'{CPU_SYSFS}/cpu{cpu}/cpufreq/scaling_governor' + if cur := read_sysfs(p): + avail = read_sysfs(f'{CPU_SYSFS}/cpu{cpu}/cpufreq/scaling_available_governors').split() + if 'performance' not in avail: + yield f'not changed: the performance governor is not available (available: {" ".join(avail)})' + return + original[p] = cur + if not original: + yield 'not changed: no cpufreq support' + return + to_change = {p: 'performance' for p, v in original.items() if v != 'performance'} + if not to_change: + yield 'performance (already set)' + return + if err := write_sysfs_values(to_change): + yield f'not changed ({", ".join(sorted(set(original.values())))}): {err}' + return + try: + yield f'performance (was {", ".join(sorted(set(original.values())))}, will be restored)' + finally: + if err := write_sysfs_values({p: original[p] for p in to_change}): + print(f'Failed to restore the CPU frequency governor: {err}', file=sys.stderr) + + +# }}} + + +# Sway support for making window sizes identical {{{ + + +def sway_available() -> bool: + return bool(os.environ.get('SWAYSOCK')) and bool(shutil.which('swaymsg')) + + +def sway_find_window(pid: int, marker: str) -> int | None: + try: + cp = subprocess.run(['swaymsg', '-t', 'get_tree', '-r'], capture_output=True, text=True, check=True, timeout=10) + except (OSError, subprocess.SubprocessError): + return None + tree: dict[str, Any] = json.loads(cp.stdout) + stack = [tree] + while stack: + n = stack.pop() + if n.get('type') in ('con', 'floating_con') and (n.get('pid') == pid or marker in (n.get('name') or '')): + con_id = n.get('id') + if isinstance(con_id, int): + return con_id + stack.extend(n.get('nodes', ())) + stack.extend(n.get('floating_nodes', ())) + return None + + +def sway_float(pid: int, marker: str, width: int, height: int, timeout: float = 15) -> str: + """Make the window floating, so that it gets its natural (requested) size, + unless width and height are non-zero in which case it is resized to them.""" + deadline = time.monotonic() + timeout + while (con_id := sway_find_window(pid, marker)) is None: + if time.monotonic() > deadline: + return 'could not find terminal window in the sway tree' + time.sleep(0.1) + resize = ['resize', 'set', 'width', f'{width}px', 'height', f'{height}px,'] if width and height else [] + cp = subprocess.run( + ['swaymsg', f'[con_id={con_id}]', 'floating', 'enable,'] + resize + ['move', 'position', '0', '0,', 'focus'], capture_output=True, text=True + ) + if cp.returncode != 0: + return f'swaymsg failed: {cp.stdout.strip()} {cp.stderr.strip()}' + return '' + + +# }}} + + +# Helpers for running terminals in minimal sessions {{{ + + +@contextmanager +def private_xwayland(width: int, height: int) -> Iterator[tuple[str, int]]: + """Start a rootful Xwayland, yields (DISPLAY, pid)""" + if not os.environ.get('WAYLAND_DISPLAY') or not shutil.which('Xwayland'): + raise RuntimeError('DISPLAY is not set and Xwayland is not available to create one') + r, w = os.pipe() + try: + proc = subprocess.Popen( + ['Xwayland', '-displayfd', str(w), '-geometry', f'{width}x{height}', '-nolisten', 'tcp'], + pass_fds=(w,), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + finally: + os.close(w) + try: + with open(r, 'rb') as f: + num = f.readline().strip().decode() + if not num.isdigit(): + raise RuntimeError(f'Xwayland failed to start, exit code: {proc.poll()}') + yield f':{num}', proc.pid + finally: + kill_process_group(proc) + + +def has_session_bus() -> bool: + rd = os.environ.get('XDG_RUNTIME_DIR', '') + return bool(os.environ.get('DBUS_SESSION_BUS_ADDRESS')) or (bool(rd) and os.path.exists(os.path.join(rd, 'bus'))) + + +def has_systemd_user_instance() -> bool: + if not shutil.which('systemctl'): + return False + return subprocess.run(['systemctl', '--user', 'show-environment'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 + + +SYSTEMD_RUN_SHIM = """#!/bin/sh +# Used when there is no systemd user instance: ignore all options and run the command directly +while [ $# -gt 0 ]; do case "$1" in --) shift; break;; -*) shift;; *) break;; esac; done +exec "$@" +""" + + +def setup_session(d: str, cmd: list[str], env: dict[str, str]) -> list[str]: + """Make up for a missing session bus and/or systemd user instance, as found in headless sessions""" + if not has_systemd_user_instance(): + shim = write_file(os.path.join(d, 'shim', 'systemd-run'), SYSTEMD_RUN_SHIM) + os.chmod(shim, 0o755) + env['PATH'] = os.path.dirname(shim) + os.pathsep + env.get('PATH', '') + if not has_session_bus(): + if not shutil.which('dbus-run-session'): + raise RuntimeError('there is no D-Bus session bus and dbus-run-session is not available to create one') + env['GTK_A11Y'] = 'none' # the accessibility bus is not available in a private session + cmd = ['dbus-run-session', '--'] + cmd + return cmd + + +# }}} + + +# Running a single terminal {{{ + + +@dataclass +class BenchmarkResult: + name: str + status: str = 'ok' # ok, timeout or failed + error: str = '' + description: str = '' + duration: float = 0 + data_size: int = 0 + repetitions: int = 0 + mb_per_second: float = 0 + cells_per_second: float = 0 + + +@dataclass +class TerminalRun: + terminal: str + version: str + results: list[BenchmarkResult] = field(default_factory=list) + error: str = '' + size: str = '' + size_warning: str = '' + wall_time: float = 0 + + +def clean_env(xdg_root: str) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith(ENV_PREFIXES_TO_REMOVE)} + env.pop('TERM', None) + for name in ('CONFIG', 'DATA', 'STATE', 'CACHE'): + p = os.path.join(xdg_root, name.lower()) + os.makedirs(p, exist_ok=True) + env[f'XDG_{name}_HOME'] = p + env['GSETTINGS_BACKEND'] = 'keyfile' + return env + + +def wait_for(predicate: Callable[[], bool], proc: subprocess.Popen[bytes], timeout: float, on_poll: Callable[[], None] | None = None) -> str: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + return f'timed out after {timeout:.0f} seconds' + if (rc := proc.poll()) is not None and not predicate(): + # some launchers fork and exit, so give the real process a chance + time.sleep(0.5) + if predicate(): + break + if rc != 0: + return f'terminal process exited with code {rc}' + if on_poll is not None: + on_poll() + time.sleep(0.05) + return '' + + +def kill_process_group(proc: subprocess.Popen[bytes]) -> None: + for sig in (signal.SIGTERM, signal.SIGKILL): + with suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, sig) + try: + proc.wait(timeout=3) + return + except subprocess.TimeoutExpired: + continue + + +def available_benchmarks() -> list[str]: + cp = subprocess.run([kitten_exe(), '__benchmark__', '--help'], stdin=subprocess.DEVNULL, capture_output=True, text=True) + ans = re.findall(r'^\s*\*\s+(\w+):', cp.stdout, flags=re.MULTILINE) + if not ans: + raise SystemExit(f'Failed to get the list of benchmarks from the benchmark kitten: {cp.stderr}') + return ans + + +def load_result(name: str, info: dict[str, Any], path: str) -> BenchmarkResult: + status = str(info.get('status', 'failed')) + if status != 'ok': + return BenchmarkResult(name, status=status, error=str(info.get('error', ''))) + with open(path) as f: + raw: list[dict[str, Any]] = json.load(f) + r = raw[0] + return BenchmarkResult( + name=str(r['name']), + description=str(r['description']), + duration=float(r['duration_seconds']), + data_size=int(r['data_size_bytes']), + repetitions=int(r['repetitions']), + mb_per_second=float(r['mb_per_second']), + cells_per_second=float(r.get('cells_per_second', 0)), + ) + + +def check_size(size: str, settings: Settings) -> str: + if settings.window_width or not size: + return '' # forced pixel size, so the number of cells is expected to differ + cols, lines = map(int, size.split('x')) + # allow for off by one because of padding, and the status line in tmux + if abs(cols - settings.columns) > 1 or abs(lines - settings.lines) > 1: + return f'window size is {size} cells instead of {settings.columns}x{settings.lines}' + return '' + + +def run_terminal( + t: Terminal, args: argparse.Namespace, settings: Settings, cpus: list[int], log: Callable[[str], None], progress: Callable[[str], None] +) -> TerminalRun: + ans = TerminalRun(t.name, terminal_version(t)) + with tempfile.TemporaryDirectory(prefix='kitty-cbench-') as d: + marker = f'cbench-{os.getpid()}-{t.name}' + inner = [sys.executable, os.path.abspath(__file__), '--inner', d, marker, kitten_exe(), str(args.repetitions), str(args.benchmark_timeout)] + list( + args.benchmarks + ) + cmd = t.build_cmd(d, inner, settings) + env = clean_env(os.path.join(d, 'xdg')) + stack = ExitStack() + window_pid = 0 + try: + if t.needs_x11 and not env.get('DISPLAY'): + env['DISPLAY'], window_pid = stack.enter_context(private_xwayland(settings.window_width or 1600, settings.window_height or 1000)) + if t.needs_session: + cmd = setup_session(d, cmd, env) + except RuntimeError as e: + stack.close() + ans.error = str(e) + return ans + log(f'Running: {shlex.join(cmd)}') + start = time.monotonic() + try: + proc = subprocess.Popen( + cmd, + env=env, + cwd=d, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + preexec_fn=lambda: os.sched_setaffinity(0, cpus), + ) + except OSError as e: + stack.close() + ans.error = f'failed to launch: {e}' + return ans + try: + ans.error = run_in_launched_terminal(proc, window_pid or proc.pid, d, marker, args, settings, progress) + if not ans.error: + with open(os.path.join(d, DONE_FILE)) as f: + done: dict[str, Any] = json.load(f) + ans.size = str(done.get('size', '')) + ans.size_warning = check_size(ans.size, settings) + statuses: dict[str, dict[str, Any]] = done['benchmarks'] + ans.results = [load_result(name, info, os.path.join(d, f'results-{name}.json')) for name, info in statuses.items()] + except (OSError, ValueError, KeyError, IndexError) as e: + ans.error = f'failed to read results: {e}' + finally: + ans.wall_time = time.monotonic() - start + kill_process_group(proc) + if t.cleanup is not None: + t.cleanup(d) + stack.close() + return ans + + +def run_in_launched_terminal( + proc: subprocess.Popen[bytes], + window_pid: int, + d: str, + marker: str, + args: argparse.Namespace, + settings: Settings, + progress: Callable[[str], None], +) -> str: + if err := wait_for(lambda: os.path.exists(os.path.join(d, READY_FILE)), proc, 60): + return f'terminal did not start: {err}' + if args.sway: + if err := sway_float(window_pid, marker, settings.window_width, settings.window_height): + return f'could not resize the window: {err}' + time.sleep(0.5) # let the terminal process the resize + with open(os.path.join(d, GO_FILE), 'w'): + pass + progress_file = os.path.join(d, PROGRESS_FILE) + last_progress = '' + + def report_progress() -> None: + nonlocal last_progress + try: + with open(progress_file) as f: + current = f.read() + except FileNotFoundError: + return + if current and current != last_progress: + last_progress = current + progress(current) + + # The inner process enforces the per benchmark timeout, allow some extra time for startup and cleanup + timeout = (args.benchmark_timeout + 30) * len(args.benchmarks) + 60 + if err := wait_for(lambda: os.path.exists(os.path.join(d, DONE_FILE)), proc, timeout, report_progress): + return f'benchmark did not complete: {err}' + return '' + + +def write_atomic(path: str, data: str) -> None: + tmp = path + '.tmp' + with open(tmp, 'w') as f: + f.write(data) + os.replace(tmp, path) + + +def recover_tty_after_kill(tty_fd: int) -> None: + import termios + + # Wait for the terminal to process whatever is left in the pty buffer, reset + # it and then discard any responses it sent to queries + os.write(tty_fd, b'\x1b[?2026l\x1b[m\x1bc') + time.sleep(1) + termios.tcflush(tty_fd, termios.TCIFLUSH) + + +def inner_main(argv: list[str]) -> None: + """Runs inside the terminal being benchmarked""" + d, marker, kitten, repetitions, timeout_str, *benchmarks = argv + timeout = float(timeout_str) + tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_CLOEXEC) + os.write(tty_fd, f'\x1b]2;{marker}\x1b\\Waiting to start benchmark...\r\n'.encode()) + write_atomic(os.path.join(d, READY_FILE), json.dumps({'pid': os.getpid()})) + go = os.path.join(d, GO_FILE) + while not os.path.exists(go): + time.sleep(0.05) + size = os.get_terminal_size(tty_fd) + statuses: dict[str, dict[str, str]] = {} + for name in benchmarks: + write_atomic(os.path.join(d, PROGRESS_FILE), name) + with open(os.path.join(d, f'log-{name}.txt'), 'w+') as log: + p = subprocess.Popen( + [kitten, '__benchmark__', '--repetitions', repetitions, '--output', os.path.join(d, f'results-{name}.json'), name], + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + rc = p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + p.wait() + recover_tty_after_kill(tty_fd) + statuses[name] = {'status': 'timeout', 'error': f'took longer than {timeout:.0f} seconds'} + continue + if rc == 0: + statuses[name] = {'status': 'ok'} + else: + log.seek(0) + statuses[name] = {'status': 'failed', 'error': f'exit code {rc}: {log.read()[-1000:].strip()}'} + write_atomic(os.path.join(d, DONE_FILE), json.dumps({'benchmarks': statuses, 'size': f'{size.columns}x{size.lines}'})) + + +# }}} + + +# Output {{{ + + +class Style: + def __init__(self, enabled: bool) -> None: + self.enabled = enabled + + def __call__(self, text: str, *codes: int) -> str: + if not self.enabled or not codes: + return text + return f'\x1b[{";".join(map(str, codes))}m{text}\x1b[m' + + def bold(self, text: str) -> str: + return self(text, 1) + + def dim(self, text: str) -> str: + return self(text, 2) + + def green(self, text: str) -> str: + return self(text, 32) + + def red(self, text: str) -> str: + return self(text, 31) + + def yellow(self, text: str) -> str: + return self(text, 33) + + def cyan(self, text: str) -> str: + return self(text, 36) + + +ANSI_PAT = re.compile(r'\x1b\[[0-9;]*m') + + +def visible_len(text: str) -> int: + return len(ANSI_PAT.sub('', text)) + + +def pad(text: str, width: int, right: bool = True) -> str: + p = ' ' * max(0, width - visible_len(text)) + return p + text if right else text + p + + +def table(header: Sequence[str], rows: Sequence[Sequence[str]], style: Style) -> str: + widths = [max(visible_len(r[i]) for r in [header, *rows]) for i in range(len(header))] + lines = [' '.join(pad(style.bold(h), w, i > 0) for i, (h, w) in enumerate(zip(header, widths)))] + lines.append(style.dim(' '.join('─' * w for w in widths))) + for r in rows: + lines.append(' '.join(pad(c, w, i > 0) for i, (c, w) in enumerate(zip(r, widths)))) + return '\n'.join(lines) + + +def fmt_cells(x: float) -> str: + return f'{x / 1e6:.1f}M' if x else '-' + + +def fmt_duration(x: float) -> str: + return f'{x:.2f}s' if x < 60 else f'{int(x // 60)}m{x % 60:04.1f}s' + + +def print_terminal_run(r: TerminalRun, style: Style) -> None: + title = style.bold(style.cyan(r.terminal)) + meta = ', '.join(x for x in (r.version, f'{r.size} cells' if r.size else '', f'took {fmt_duration(r.wall_time)}') if x) + print(f'{title} {style.dim(f"({meta})")}') + if r.size_warning: + print(style.yellow(f' WARNING: {r.size_warning}')) + if r.error: + print(style.red(f' FAILED: {r.error}')) + print() + return + rows = [] + for b in r.results: + if b.status == 'ok': + rows.append([b.name, f'{b.mb_per_second:.1f}', fmt_cells(b.cells_per_second), fmt_duration(b.duration), f'{b.data_size / (1024 * 1024):.0f} MB']) + else: + rows.append([b.name, style.red(b.status), '', '', '']) + print('\n'.join(' ' + x for x in table(['Benchmark', 'MB/s', 'Cells/s', 'Time', 'Data'], rows, style).splitlines())) + for b in r.results: + if b.status != 'ok': + print(style.red(f' {b.name} {b.status}: {b.error}')) + print() + + +def print_summary(runs: list[TerminalRun], style: Style) -> None: + ok = [r for r in runs if r.results and not r.error] + if not ok: + return + names: list[str] = [] + for r in ok: + for b in r.results: + if b.name not in names: + names.append(b.name) + print( + style.bold('Summary: throughput in MB/s, higher is better. ') + + style.dim('Fastest in green, slowest in red, (n×) is how many times slower than the fastest.') + ) + rows = [] + for name in names: + vals = {r.terminal: b.mb_per_second for r in ok for b in r.results if b.name == name and b.status == 'ok'} + statuses = {r.terminal: b.status for r in ok for b in r.results if b.name == name} + best, worst = max(vals.values(), default=0), min(vals.values(), default=0) + row = [name] + for r in ok: + v = vals.get(r.terminal) + if v is None: + row.append(style.red(statuses.get(r.terminal, '-'))) + continue + text = f'{v:.1f}' + if v == best: + text = style.bold(style.green(text)) + elif len(vals) > 2 and v == worst: + text = style.red(text) + if v != best and v > 0: + text += style.dim(f' ({best / v:.1f}×)') + row.append(text) + rows.append(row) + print(table(['Benchmark'] + [r.terminal for r in ok], rows, style)) + print() + + +# }}} + + +def parse_args() -> argparse.Namespace: + terminal_names = [t.name for t in all_terminals()] + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('benchmarks', nargs='*', help='Benchmarks to run, by default all are run. See kitten __benchmark__ --help for the list') + p.add_argument( + '--terminals', '-t', default=','.join(terminal_names), help=f'Comma separated list of terminals to benchmark, from: {", ".join(terminal_names)}' + ) + p.add_argument('--repetitions', '-r', type=int, default=500, help='The number of repetitions of each benchmark. Default: %(default)s') + p.add_argument('--font-family', default='DejaVu Sans Mono', help='Font family to use in all terminals. Default: %(default)s') + p.add_argument('--font-size', type=float, default=11, help='Font size in pts to use in all terminals. Default: %(default)s') + p.add_argument('--columns', type=int, default=120, help='Initial window width in cells. Default: %(default)s') + p.add_argument('--lines', type=int, default=40, help='Initial window height in cells. Default: %(default)s') + p.add_argument( + '--window-size', + default='', + help='Window size in pixels, used only when running under sway. Under sway, windows are made floating so that tiling does not' + ' override the initial size in cells. If this is specified they are additionally resized to it.', + ) + p.add_argument('--cpus', default='', help='Comma separated list of CPUs to pin the terminals to, overrides automatic fast core detection') + p.add_argument('--all-smt-threads', action='store_true', help='Use all SMT threads of the fast cores instead of one per core') + p.add_argument('--no-governor', action='store_true', help='Do not change the CPU frequency governor') + p.add_argument('--no-sway', action='store_true', help='Do not use sway to make window sizes identical, even if running under sway') + p.add_argument( + '--benchmark-timeout', + type=float, + default=180, + help='Max time in seconds for each benchmark in each terminal, after which it is aborted and reported as timed out. Default: %(default)s', + ) + p.add_argument('--output', '-o', default='', help='Also write all results as JSON to the specified file') + p.add_argument('--verbose', '-v', action='store_true', help='Print the command lines used to run the terminals') + args = p.parse_args() + args.sway = sway_available() and not args.no_sway + if args.benchmarks: + avail = available_benchmarks() + if unknown := [x for x in args.benchmarks if x not in avail]: + raise SystemExit(f'Unknown benchmarks: {", ".join(unknown)}. Choose from: {", ".join(avail)}') + else: + args.benchmarks = available_benchmarks() + return args + + +def main() -> None: + if len(sys.argv) > 1 and sys.argv[1] == '--inner': + inner_main(sys.argv[2:]) + return + if not sys.platform.startswith('linux'): + raise SystemExit('This script only works on Linux') + args = parse_args() + # Ensure cleanup code (killing terminals, restoring the governor) runs when terminated + signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(128 + signum)) + style = Style(sys.stdout.isatty()) + err_style = Style(sys.stderr.isatty()) + wanted = [x.strip() for x in args.terminals.split(',') if x.strip()] + known = {t.name: t for t in all_terminals()} + if unknown := [x for x in wanted if x not in known]: + raise SystemExit(f'Unknown terminals: {", ".join(unknown)}') + terminals = [known[x] for x in wanted] + missing = [t.name for t in terminals if not is_installed(t)] + terminals = [t for t in terminals if is_installed(t)] + if not terminals: + raise SystemExit('None of the terminals are installed') + ww = wh = 0 + if args.window_size: + try: + ww, wh = map(int, args.window_size.lower().split('x')) + except ValueError: + raise SystemExit(f'Invalid window size: {args.window_size}') + settings = Settings(args.font_family, args.font_size, args.columns, args.lines, ww, wh) + kitten_exe() # fail early if not found + if args.cpus: + cpus, cpu_how = parse_cpu_list(args.cpus), 'specified on command line' + else: + cpus, cpu_how = detect_fast_cpus(not args.all_smt_threads) + if shutil.which('fc-list'): + fc = subprocess.run(['fc-list', f'{settings.font_family}:spacing=mono', 'family'], capture_output=True, text=True) + if not fc.stdout.strip(): + print( + err_style.yellow(f'Warning: the font {settings.font_family!r} does not seem to be installed, terminals will use fallback fonts'), + file=sys.stderr, + ) + + def log(msg: str) -> None: + if args.verbose: + print(err_style.dim(msg), file=sys.stderr) + + runs: list[TerminalRun] = [] + with performance_governor(not args.no_governor) as governor: + print(style.bold('Comparative terminal benchmark')) + print(f' CPUs: {",".join(map(str, cpus))} ({cpu_how})') + print(f' Governor: {governor}') + print(f' Font: {settings.font_family} {fmt_size(settings.font_size)}pt') + win = f'{settings.columns}x{settings.lines} cells' + if args.sway: + win += f', resized to {ww}x{wh} pixels' if ww else ', floating sway window' + print(f' Window: {win}') + print(f' Reps: {args.repetitions} (timeout of {args.benchmark_timeout:.0f}s per benchmark)') + if missing: + print(f' Skipped: {", ".join(missing)} (not installed)') + print() + sys.stdout.flush() + for t in terminals: + print(err_style.dim(f'Benchmarking {t.name}...'), file=sys.stderr) + r = run_terminal(t, args, settings, cpus, log, lambda name: print(err_style.dim(f' {name}'), file=sys.stderr)) + runs.append(r) + print_terminal_run(r, style) + sys.stdout.flush() + print_summary(runs, style) + if args.output: + with open(args.output, 'w') as f: + json.dump( + [ + { + 'terminal': r.terminal, + 'version': r.version, + 'size': r.size, + 'size_warning': r.size_warning, + 'error': r.error, + 'wall_time': r.wall_time, + 'results': [b.__dict__ for b in r.results], + } + for r in runs + ], + f, + indent=2, + ) + + +if __name__ == '__main__': + main()