From 8a58e283f1b22cf08fa5a49c5ab461b480855d14 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 8 Nov 2026 15:21:41 +0530 Subject: [PATCH] Add a script to run the benchmark kitten in multiple terminals for comparison Runs the benchmark kitten in all installed terminals from xterm, foot, kitty, ghostty, alacritty, wezterm, konsole, ptyxis and alacritty+tmux, with default configuration except for font, font size and window size. Terminals are pinned to the fast cores on machines with heterogeneous cores and the performance governor is set if root access is available. --- comparative_benchmark.py | 1031 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1031 insertions(+) create mode 100755 comparative_benchmark.py diff --git a/comparative_benchmark.py b/comparative_benchmark.py new file mode 100755 index 000000000..21c745ec2 --- /dev/null +++ b/comparative_benchmark.py @@ -0,0 +1,1031 @@ +#!/usr/bin/env python +# License: GPLv3 Copyright: 2026, Kovid Goyal + +""" +Run the kitty benchmark kitten in all installed terminals from a fixed set and +present the results for comparison. Linux only. + +Every terminal is run with its default configuration, except for the font +family, font size and window size, which are made the same for all terminals. +User configuration is ignored by pointing all the XDG base directories at +a temporary directory. The terminals are pinned to the fast cores on machines +with heterogeneous cores and the CPU frequency governor is set to performance +for the duration of the run, if possible. +""" + +import argparse +import json +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Iterator, Sequence +from contextlib import ExitStack, contextmanager, suppress +from dataclasses import dataclass, field +from typing import Any + +BASE = os.path.dirname(os.path.abspath(__file__)) +CPU_SYSFS = '/sys/devices/system/cpu' +READY_FILE, GO_FILE, DONE_FILE, PROGRESS_FILE = 'ready.json', 'go', 'done.json', 'progress' +ENV_PREFIXES_TO_REMOVE = ( + 'KITTY_', + 'TMUX', + 'WEZTERM_', + 'GHOSTTY_', + 'ALACRITTY_', + 'KONSOLE_', + 'VTE_', + 'TERM_PROGRAM', + 'TERMINFO', + 'COLORTERM', + 'WINDOWID', + 'XTERM_', + 'PTYXIS_', + 'SHELL_SESSION_ID', +) + + +@dataclass(frozen=True) +class Settings: + font_family: str + font_size: float + columns: int + lines: int + window_width: int + window_height: int + + +@dataclass +class Terminal: + name: str + executable: str + # Build the command line to run cmd in the terminal, given the per-run + # directory for config files and the settings + build_cmd: Callable[[str, list[str], Settings], list[str]] + version_cmd: list[str] = field(default_factory=list) + # Run after the terminal is done, to cleanup any left over processes + cleanup: Callable[[str], None] | None = None + # Needs an X11 display, a private rootful Xwayland is started if DISPLAY is not set + needs_x11: bool = False + # Needs a D-Bus session bus and a systemd user instance (to spawn processes in scopes) + needs_session: bool = False + + +# Terminal definitions {{{ + + +def write_file(path: str, text: str) -> str: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write(text) + return path + + +def fmt_size(x: float) -> str: + return str(int(x)) if x == int(x) else str(x) + + +def toml_str(x: str) -> str: + return json.dumps(x) + + +def xterm_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return ['xterm', '-fa', s.font_family, '-fs', fmt_size(s.font_size), '-geometry', f'{s.columns}x{s.lines}', '-e'] + cmd + + +def foot_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file(os.path.join(d, 'foot.ini'), f'font={s.font_family}:size={fmt_size(s.font_size)}\ninitial-window-size-chars={s.columns}x{s.lines}\n') + return ['foot', '--config', conf] + cmd + + +def kitty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file( + os.path.join(d, 'kitty.conf'), + '\n'.join( + ( + f'font_family family="{s.font_family}"', + f'font_size {fmt_size(s.font_size)}', + 'remember_window_size no', + f'initial_window_width {s.columns}c', + f'initial_window_height {s.lines}c', + '', + ) + ), + ) + return [kitty_exe(), '--config', conf] + cmd + + +def ghostty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file( + os.path.join(d, 'ghostty.conf'), + '\n'.join( + ( + f'font-family = {s.font_family}', + f'font-size = {fmt_size(s.font_size)}', + f'window-width = {s.columns}', + f'window-height = {s.lines}', + # Needed so that the window is created in a new process that exits when the command finishes + 'gtk-single-instance = false', + 'confirm-close-surface = false', + 'quit-after-last-window-closed = true', + '', + ) + ), + ) + return ['ghostty', '--config-default-files=false', f'--config-file={conf}', '-e'] + cmd + + +def alacritty_config(d: str, s: Settings) -> str: + return write_file( + os.path.join(d, 'alacritty.toml'), + '\n'.join( + ( + '[font]', + f'size = {float(s.font_size)}', + f'normal = {{ family = {toml_str(s.font_family)} }}', + '[window]', + f'dimensions = {{ columns = {s.columns}, lines = {s.lines} }}', + '', + ) + ), + ) + + +def alacritty_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return ['alacritty', '--config-file', alacritty_config(d, s), '-e'] + cmd + + +def wezterm_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + conf = write_file( + os.path.join(d, 'wezterm.lua'), + '\n'.join( + ( + 'local wezterm = require "wezterm"', + 'local config = wezterm.config_builder()', + f'config.font = wezterm.font({json.dumps(s.font_family)})', + f'config.font_size = {float(s.font_size)}', + f'config.initial_cols = {s.columns}', + f'config.initial_rows = {s.lines}', + 'return config', + '', + ) + ), + ) + return ['wezterm', '--config-file', conf, 'start', '--always-new-process', '--'] + cmd + + +def konsole_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return [ + 'konsole', + '--separate', + '--builtin-profile', + '-p', + f'Font={s.font_family},{fmt_size(s.font_size)}', + '-p', + f'TerminalColumns={s.columns}', + '-p', + f'TerminalRows={s.lines}', + '-e', + ] + cmd + + +def ptyxis_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + # ptyxis uses GSettings, the run environment sets GSETTINGS_BACKEND=keyfile + # with XDG_CONFIG_HOME pointing into d, so this is where its settings are read from + write_file( + os.path.join(d, 'xdg', 'config', 'glib-2.0', 'settings', 'keyfile'), + '\n'.join( + ( + '[org/gnome/Ptyxis]', + 'use-system-font=false', + f"font-name='{s.font_family} {fmt_size(s.font_size)}'", + f'default-columns=uint32 {s.columns}', + f'default-rows=uint32 {s.lines}', + 'restore-window-size=false', + 'restore-session=false', + '', + ) + ), + ) + return ['ptyxis', '--standalone', '--'] + cmd + + +TMUX_SOCKET = 'kitty-comparative-benchmark' + + +def alacritty_tmux_cmd(d: str, cmd: list[str], s: Settings) -> list[str]: + return ['alacritty', '--config-file', alacritty_config(d, s), '-e', 'tmux', '-L', TMUX_SOCKET, '-f', '/dev/null', 'new-session', '--'] + cmd + + +def tmux_cleanup(d: str) -> None: + subprocess.run(['tmux', '-L', TMUX_SOCKET, 'kill-server'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def kitty_exe() -> str: + q = os.path.join(BASE, 'kitty', 'launcher', 'kitty') + return q if os.access(q, os.X_OK) else 'kitty' + + +def kitten_exe() -> str: + q = os.path.join(BASE, 'kitty', 'launcher', 'kitten') + if os.access(q, os.X_OK): + return q + ans = shutil.which('kitten') + if not ans: + raise SystemExit('Could not find the kitten executable, build kitty first with: make debug') + return ans + + +def all_terminals() -> list[Terminal]: + return [ + Terminal('xterm', 'xterm', xterm_cmd, ['xterm', '-version'], needs_x11=True), + Terminal('foot', 'foot', foot_cmd, ['foot', '--version']), + Terminal('kitty', kitty_exe(), kitty_cmd, [kitty_exe(), '--version']), + Terminal('ghostty', 'ghostty', ghostty_cmd, ['ghostty', '--version']), + Terminal('alacritty', 'alacritty', alacritty_cmd, ['alacritty', '--version']), + Terminal('wezterm', 'wezterm', wezterm_cmd, ['wezterm', '--version']), + Terminal('konsole', 'konsole', konsole_cmd, ['konsole', '--version']), + Terminal('ptyxis', 'ptyxis', ptyxis_cmd, ['ptyxis', '--version'], needs_session=True), + Terminal('alacritty+tmux', 'alacritty', alacritty_tmux_cmd, ['tmux', '-V'], tmux_cleanup), + ] + + +def is_installed(t: Terminal) -> bool: + if t.name == 'alacritty+tmux' and not shutil.which('tmux'): + return False + return bool(shutil.which(t.executable)) + + +def terminal_version(t: Terminal) -> str: + if not t.version_cmd: + return '' + try: + cp = subprocess.run(t.version_cmd, capture_output=True, text=True, timeout=10) + except (OSError, subprocess.TimeoutExpired) as e: + return f'unknown ({e})' + lines = [x.strip() for x in (cp.stdout or cp.stderr).splitlines() if x.strip()] + ans = lines[0] if lines else 'unknown' + if t.name == 'alacritty+tmux': + a = Terminal('alacritty', 'alacritty', alacritty_cmd, ['alacritty', '--version']) + ans = f'{terminal_version(a)} + {ans}' + return ans + + +# }}} + + +# CPU setup {{{ + + +def read_sysfs(path: str) -> str: + try: + with open(path) as f: + return f.read().strip() + except OSError: + return '' + + +def online_cpus() -> list[int]: + return sorted(os.sched_getaffinity(0)) + + +# Per CPU sysfs files that indicate relative CPU performance: cpu_capacity (ARM +# big.LITTLE, constant on x86), CPPC highest perf (AMD and Intel) and max frequency +PERF_METRICS = ('cpu_capacity', 'cpufreq/amd_pstate_highest_perf', 'acpi_cppc/highest_perf', 'cpufreq/cpuinfo_max_freq') + + +def physical_core_key(cpu: int) -> tuple[str, str]: + base = f'{CPU_SYSFS}/cpu{cpu}/topology' + return read_sysfs(f'{base}/physical_package_id'), read_sysfs(f'{base}/core_id') + + +def detect_fast_cpus(one_thread_per_core: bool) -> tuple[list[int], str]: + cpus = online_cpus() + intel_core = read_sysfs('/sys/devices/cpu_core/cpus') + fast: list[int] + if intel_core: # Intel hybrid, P-cores + fast = [c for c in parse_cpu_list(intel_core) if c in cpus] + how = 'Intel hybrid P-cores' + else: + for q in PERF_METRICS: + vals = {c: read_sysfs(f'{CPU_SYSFS}/cpu{c}/{q}') for c in cpus} + if not all(v.isdigit() for v in vals.values()): + continue + metrics = {c: int(v) for c, v in vals.items()} + top = max(metrics.values()) + # Ignore small per-core variations from binning, e.g. AMD preferred cores + fast = [c for c in cpus if metrics[c] >= 0.9 * top] + if len(fast) < len(cpus): + how = f'heterogeneous cores, using the fastest as per {q}' + break + else: + return cpus, 'homogeneous cores, using all CPUs' + if one_thread_per_core: + seen: set[tuple[str, str]] = set() + ans = [] + for c in fast: + k = physical_core_key(c) + if k not in seen: + seen.add(k) + ans.append(c) + fast, how = ans, how + ', one thread per core' + return fast, how + + +def parse_cpu_list(spec: str) -> list[int]: + ans: list[int] = [] + for part in spec.split(','): + part = part.strip() + if not part: + continue + if '-' in part: + a, b = part.split('-', 1) + ans.extend(range(int(a), int(b) + 1)) + else: + ans.append(int(part)) + return ans + + +def write_sysfs_values(values: dict[str, str]) -> str: + """Write the values, using sudo if needed. Returns an error message on failure.""" + if not values: + return '' + try: + for path, val in values.items(): + with open(path, 'w') as f: + f.write(val) + return '' + except PermissionError: + pass + except OSError as e: + return str(e) + script = '\n'.join(f'echo {shlex.quote(v)} > {shlex.quote(p)}' for p, v in values.items()) + cp = subprocess.run(['sudo', '-n', 'sh', '-c', script], capture_output=True, text=True) + if cp.returncode == 0: + return '' + if sys.stdin.isatty(): + print('Root access is needed to change the CPU frequency governor, asking sudo for your password', file=sys.stderr) + cp = subprocess.run(['sudo', 'sh', '-c', script], stdout=subprocess.PIPE, text=True) + if cp.returncode == 0: + return '' + return 'permission denied: not root and passwordless sudo is not available' + + +@contextmanager +def performance_governor(enabled: bool) -> Iterator[str]: + """Yields a description of what was done""" + if not enabled: + yield 'not changed (disabled on command line)' + return + original: dict[str, str] = {} + for cpu in online_cpus(): + p = f'{CPU_SYSFS}/cpu{cpu}/cpufreq/scaling_governor' + if cur := read_sysfs(p): + avail = read_sysfs(f'{CPU_SYSFS}/cpu{cpu}/cpufreq/scaling_available_governors').split() + if 'performance' not in avail: + yield f'not changed: the performance governor is not available (available: {" ".join(avail)})' + return + original[p] = cur + if not original: + yield 'not changed: no cpufreq support' + return + to_change = {p: 'performance' for p, v in original.items() if v != 'performance'} + if not to_change: + yield 'performance (already set)' + return + if err := write_sysfs_values(to_change): + yield f'not changed ({", ".join(sorted(set(original.values())))}): {err}' + return + try: + yield f'performance (was {", ".join(sorted(set(original.values())))}, will be restored)' + finally: + if err := write_sysfs_values({p: original[p] for p in to_change}): + print(f'Failed to restore the CPU frequency governor: {err}', file=sys.stderr) + + +# }}} + + +# Sway support for making window sizes identical {{{ + + +def sway_available() -> bool: + return bool(os.environ.get('SWAYSOCK')) and bool(shutil.which('swaymsg')) + + +def sway_find_window(pid: int, marker: str) -> int | None: + try: + cp = subprocess.run(['swaymsg', '-t', 'get_tree', '-r'], capture_output=True, text=True, check=True, timeout=10) + except (OSError, subprocess.SubprocessError): + return None + tree: dict[str, Any] = json.loads(cp.stdout) + stack = [tree] + while stack: + n = stack.pop() + if n.get('type') in ('con', 'floating_con') and (n.get('pid') == pid or marker in (n.get('name') or '')): + con_id = n.get('id') + if isinstance(con_id, int): + return con_id + stack.extend(n.get('nodes', ())) + stack.extend(n.get('floating_nodes', ())) + return None + + +def sway_float(pid: int, marker: str, width: int, height: int, timeout: float = 15) -> str: + """Make the window floating, so that it gets its natural (requested) size, + unless width and height are non-zero in which case it is resized to them.""" + deadline = time.monotonic() + timeout + while (con_id := sway_find_window(pid, marker)) is None: + if time.monotonic() > deadline: + return 'could not find terminal window in the sway tree' + time.sleep(0.1) + resize = ['resize', 'set', 'width', f'{width}px', 'height', f'{height}px,'] if width and height else [] + cp = subprocess.run( + ['swaymsg', f'[con_id={con_id}]', 'floating', 'enable,'] + resize + ['move', 'position', '0', '0,', 'focus'], capture_output=True, text=True + ) + if cp.returncode != 0: + return f'swaymsg failed: {cp.stdout.strip()} {cp.stderr.strip()}' + return '' + + +# }}} + + +# Helpers for running terminals in minimal sessions {{{ + + +@contextmanager +def private_xwayland(width: int, height: int) -> Iterator[tuple[str, int]]: + """Start a rootful Xwayland, yields (DISPLAY, pid)""" + if not os.environ.get('WAYLAND_DISPLAY') or not shutil.which('Xwayland'): + raise RuntimeError('DISPLAY is not set and Xwayland is not available to create one') + r, w = os.pipe() + try: + proc = subprocess.Popen( + ['Xwayland', '-displayfd', str(w), '-geometry', f'{width}x{height}', '-nolisten', 'tcp'], + pass_fds=(w,), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + finally: + os.close(w) + try: + with open(r, 'rb') as f: + num = f.readline().strip().decode() + if not num.isdigit(): + raise RuntimeError(f'Xwayland failed to start, exit code: {proc.poll()}') + yield f':{num}', proc.pid + finally: + kill_process_group(proc) + + +def has_session_bus() -> bool: + rd = os.environ.get('XDG_RUNTIME_DIR', '') + return bool(os.environ.get('DBUS_SESSION_BUS_ADDRESS')) or (bool(rd) and os.path.exists(os.path.join(rd, 'bus'))) + + +def has_systemd_user_instance() -> bool: + if not shutil.which('systemctl'): + return False + return subprocess.run(['systemctl', '--user', 'show-environment'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 + + +SYSTEMD_RUN_SHIM = """#!/bin/sh +# Used when there is no systemd user instance: ignore all options and run the command directly +while [ $# -gt 0 ]; do case "$1" in --) shift; break;; -*) shift;; *) break;; esac; done +exec "$@" +""" + + +def setup_session(d: str, cmd: list[str], env: dict[str, str]) -> list[str]: + """Make up for a missing session bus and/or systemd user instance, as found in headless sessions""" + if not has_systemd_user_instance(): + shim = write_file(os.path.join(d, 'shim', 'systemd-run'), SYSTEMD_RUN_SHIM) + os.chmod(shim, 0o755) + env['PATH'] = os.path.dirname(shim) + os.pathsep + env.get('PATH', '') + if not has_session_bus(): + if not shutil.which('dbus-run-session'): + raise RuntimeError('there is no D-Bus session bus and dbus-run-session is not available to create one') + env['GTK_A11Y'] = 'none' # the accessibility bus is not available in a private session + cmd = ['dbus-run-session', '--'] + cmd + return cmd + + +# }}} + + +# Running a single terminal {{{ + + +@dataclass +class BenchmarkResult: + name: str + status: str = 'ok' # ok, timeout or failed + error: str = '' + description: str = '' + duration: float = 0 + data_size: int = 0 + repetitions: int = 0 + mb_per_second: float = 0 + cells_per_second: float = 0 + + +@dataclass +class TerminalRun: + terminal: str + version: str + results: list[BenchmarkResult] = field(default_factory=list) + error: str = '' + size: str = '' + size_warning: str = '' + wall_time: float = 0 + + +def clean_env(xdg_root: str) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith(ENV_PREFIXES_TO_REMOVE)} + env.pop('TERM', None) + for name in ('CONFIG', 'DATA', 'STATE', 'CACHE'): + p = os.path.join(xdg_root, name.lower()) + os.makedirs(p, exist_ok=True) + env[f'XDG_{name}_HOME'] = p + env['GSETTINGS_BACKEND'] = 'keyfile' + return env + + +def wait_for(predicate: Callable[[], bool], proc: subprocess.Popen[bytes], timeout: float, on_poll: Callable[[], None] | None = None) -> str: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + return f'timed out after {timeout:.0f} seconds' + if (rc := proc.poll()) is not None and not predicate(): + # some launchers fork and exit, so give the real process a chance + time.sleep(0.5) + if predicate(): + break + if rc != 0: + return f'terminal process exited with code {rc}' + if on_poll is not None: + on_poll() + time.sleep(0.05) + return '' + + +def kill_process_group(proc: subprocess.Popen[bytes]) -> None: + for sig in (signal.SIGTERM, signal.SIGKILL): + with suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, sig) + try: + proc.wait(timeout=3) + return + except subprocess.TimeoutExpired: + continue + + +def available_benchmarks() -> list[str]: + cp = subprocess.run([kitten_exe(), '__benchmark__', '--help'], stdin=subprocess.DEVNULL, capture_output=True, text=True) + ans = re.findall(r'^\s*\*\s+(\w+):', cp.stdout, flags=re.MULTILINE) + if not ans: + raise SystemExit(f'Failed to get the list of benchmarks from the benchmark kitten: {cp.stderr}') + return ans + + +def load_result(name: str, info: dict[str, Any], path: str) -> BenchmarkResult: + status = str(info.get('status', 'failed')) + if status != 'ok': + return BenchmarkResult(name, status=status, error=str(info.get('error', ''))) + with open(path) as f: + raw: list[dict[str, Any]] = json.load(f) + r = raw[0] + return BenchmarkResult( + name=str(r['name']), + description=str(r['description']), + duration=float(r['duration_seconds']), + data_size=int(r['data_size_bytes']), + repetitions=int(r['repetitions']), + mb_per_second=float(r['mb_per_second']), + cells_per_second=float(r.get('cells_per_second', 0)), + ) + + +def check_size(size: str, settings: Settings) -> str: + if settings.window_width or not size: + return '' # forced pixel size, so the number of cells is expected to differ + cols, lines = map(int, size.split('x')) + # allow for off by one because of padding, and the status line in tmux + if abs(cols - settings.columns) > 1 or abs(lines - settings.lines) > 1: + return f'window size is {size} cells instead of {settings.columns}x{settings.lines}' + return '' + + +def run_terminal( + t: Terminal, args: argparse.Namespace, settings: Settings, cpus: list[int], log: Callable[[str], None], progress: Callable[[str], None] +) -> TerminalRun: + ans = TerminalRun(t.name, terminal_version(t)) + with tempfile.TemporaryDirectory(prefix='kitty-cbench-') as d: + marker = f'cbench-{os.getpid()}-{t.name}' + inner = [sys.executable, os.path.abspath(__file__), '--inner', d, marker, kitten_exe(), str(args.repetitions), str(args.benchmark_timeout)] + list( + args.benchmarks + ) + cmd = t.build_cmd(d, inner, settings) + env = clean_env(os.path.join(d, 'xdg')) + stack = ExitStack() + window_pid = 0 + try: + if t.needs_x11 and not env.get('DISPLAY'): + env['DISPLAY'], window_pid = stack.enter_context(private_xwayland(settings.window_width or 1600, settings.window_height or 1000)) + if t.needs_session: + cmd = setup_session(d, cmd, env) + except RuntimeError as e: + stack.close() + ans.error = str(e) + return ans + log(f'Running: {shlex.join(cmd)}') + start = time.monotonic() + try: + proc = subprocess.Popen( + cmd, + env=env, + cwd=d, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + preexec_fn=lambda: os.sched_setaffinity(0, cpus), + ) + except OSError as e: + stack.close() + ans.error = f'failed to launch: {e}' + return ans + try: + ans.error = run_in_launched_terminal(proc, window_pid or proc.pid, d, marker, args, settings, progress) + if not ans.error: + with open(os.path.join(d, DONE_FILE)) as f: + done: dict[str, Any] = json.load(f) + ans.size = str(done.get('size', '')) + ans.size_warning = check_size(ans.size, settings) + statuses: dict[str, dict[str, Any]] = done['benchmarks'] + ans.results = [load_result(name, info, os.path.join(d, f'results-{name}.json')) for name, info in statuses.items()] + except (OSError, ValueError, KeyError, IndexError) as e: + ans.error = f'failed to read results: {e}' + finally: + ans.wall_time = time.monotonic() - start + kill_process_group(proc) + if t.cleanup is not None: + t.cleanup(d) + stack.close() + return ans + + +def run_in_launched_terminal( + proc: subprocess.Popen[bytes], + window_pid: int, + d: str, + marker: str, + args: argparse.Namespace, + settings: Settings, + progress: Callable[[str], None], +) -> str: + if err := wait_for(lambda: os.path.exists(os.path.join(d, READY_FILE)), proc, 60): + return f'terminal did not start: {err}' + if args.sway: + if err := sway_float(window_pid, marker, settings.window_width, settings.window_height): + return f'could not resize the window: {err}' + time.sleep(0.5) # let the terminal process the resize + with open(os.path.join(d, GO_FILE), 'w'): + pass + progress_file = os.path.join(d, PROGRESS_FILE) + last_progress = '' + + def report_progress() -> None: + nonlocal last_progress + try: + with open(progress_file) as f: + current = f.read() + except FileNotFoundError: + return + if current and current != last_progress: + last_progress = current + progress(current) + + # The inner process enforces the per benchmark timeout, allow some extra time for startup and cleanup + timeout = (args.benchmark_timeout + 30) * len(args.benchmarks) + 60 + if err := wait_for(lambda: os.path.exists(os.path.join(d, DONE_FILE)), proc, timeout, report_progress): + return f'benchmark did not complete: {err}' + return '' + + +def write_atomic(path: str, data: str) -> None: + tmp = path + '.tmp' + with open(tmp, 'w') as f: + f.write(data) + os.replace(tmp, path) + + +def recover_tty_after_kill(tty_fd: int) -> None: + import termios + + # Wait for the terminal to process whatever is left in the pty buffer, reset + # it and then discard any responses it sent to queries + os.write(tty_fd, b'\x1b[?2026l\x1b[m\x1bc') + time.sleep(1) + termios.tcflush(tty_fd, termios.TCIFLUSH) + + +def inner_main(argv: list[str]) -> None: + """Runs inside the terminal being benchmarked""" + d, marker, kitten, repetitions, timeout_str, *benchmarks = argv + timeout = float(timeout_str) + tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_CLOEXEC) + os.write(tty_fd, f'\x1b]2;{marker}\x1b\\Waiting to start benchmark...\r\n'.encode()) + write_atomic(os.path.join(d, READY_FILE), json.dumps({'pid': os.getpid()})) + go = os.path.join(d, GO_FILE) + while not os.path.exists(go): + time.sleep(0.05) + size = os.get_terminal_size(tty_fd) + statuses: dict[str, dict[str, str]] = {} + for name in benchmarks: + write_atomic(os.path.join(d, PROGRESS_FILE), name) + with open(os.path.join(d, f'log-{name}.txt'), 'w+') as log: + p = subprocess.Popen( + [kitten, '__benchmark__', '--repetitions', repetitions, '--output', os.path.join(d, f'results-{name}.json'), name], + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + rc = p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + p.wait() + recover_tty_after_kill(tty_fd) + statuses[name] = {'status': 'timeout', 'error': f'took longer than {timeout:.0f} seconds'} + continue + if rc == 0: + statuses[name] = {'status': 'ok'} + else: + log.seek(0) + statuses[name] = {'status': 'failed', 'error': f'exit code {rc}: {log.read()[-1000:].strip()}'} + write_atomic(os.path.join(d, DONE_FILE), json.dumps({'benchmarks': statuses, 'size': f'{size.columns}x{size.lines}'})) + + +# }}} + + +# Output {{{ + + +class Style: + def __init__(self, enabled: bool) -> None: + self.enabled = enabled + + def __call__(self, text: str, *codes: int) -> str: + if not self.enabled or not codes: + return text + return f'\x1b[{";".join(map(str, codes))}m{text}\x1b[m' + + def bold(self, text: str) -> str: + return self(text, 1) + + def dim(self, text: str) -> str: + return self(text, 2) + + def green(self, text: str) -> str: + return self(text, 32) + + def red(self, text: str) -> str: + return self(text, 31) + + def yellow(self, text: str) -> str: + return self(text, 33) + + def cyan(self, text: str) -> str: + return self(text, 36) + + +ANSI_PAT = re.compile(r'\x1b\[[0-9;]*m') + + +def visible_len(text: str) -> int: + return len(ANSI_PAT.sub('', text)) + + +def pad(text: str, width: int, right: bool = True) -> str: + p = ' ' * max(0, width - visible_len(text)) + return p + text if right else text + p + + +def table(header: Sequence[str], rows: Sequence[Sequence[str]], style: Style) -> str: + widths = [max(visible_len(r[i]) for r in [header, *rows]) for i in range(len(header))] + lines = [' '.join(pad(style.bold(h), w, i > 0) for i, (h, w) in enumerate(zip(header, widths)))] + lines.append(style.dim(' '.join('─' * w for w in widths))) + for r in rows: + lines.append(' '.join(pad(c, w, i > 0) for i, (c, w) in enumerate(zip(r, widths)))) + return '\n'.join(lines) + + +def fmt_cells(x: float) -> str: + return f'{x / 1e6:.1f}M' if x else '-' + + +def fmt_duration(x: float) -> str: + return f'{x:.2f}s' if x < 60 else f'{int(x // 60)}m{x % 60:04.1f}s' + + +def print_terminal_run(r: TerminalRun, style: Style) -> None: + title = style.bold(style.cyan(r.terminal)) + meta = ', '.join(x for x in (r.version, f'{r.size} cells' if r.size else '', f'took {fmt_duration(r.wall_time)}') if x) + print(f'{title} {style.dim(f"({meta})")}') + if r.size_warning: + print(style.yellow(f' WARNING: {r.size_warning}')) + if r.error: + print(style.red(f' FAILED: {r.error}')) + print() + return + rows = [] + for b in r.results: + if b.status == 'ok': + rows.append([b.name, f'{b.mb_per_second:.1f}', fmt_cells(b.cells_per_second), fmt_duration(b.duration), f'{b.data_size / (1024 * 1024):.0f} MB']) + else: + rows.append([b.name, style.red(b.status), '', '', '']) + print('\n'.join(' ' + x for x in table(['Benchmark', 'MB/s', 'Cells/s', 'Time', 'Data'], rows, style).splitlines())) + for b in r.results: + if b.status != 'ok': + print(style.red(f' {b.name} {b.status}: {b.error}')) + print() + + +def print_summary(runs: list[TerminalRun], style: Style) -> None: + ok = [r for r in runs if r.results and not r.error] + if not ok: + return + names: list[str] = [] + for r in ok: + for b in r.results: + if b.name not in names: + names.append(b.name) + print( + style.bold('Summary: throughput in MB/s, higher is better. ') + + style.dim('Fastest in green, slowest in red, (n×) is how many times slower than the fastest.') + ) + rows = [] + for name in names: + vals = {r.terminal: b.mb_per_second for r in ok for b in r.results if b.name == name and b.status == 'ok'} + statuses = {r.terminal: b.status for r in ok for b in r.results if b.name == name} + best, worst = max(vals.values(), default=0), min(vals.values(), default=0) + row = [name] + for r in ok: + v = vals.get(r.terminal) + if v is None: + row.append(style.red(statuses.get(r.terminal, '-'))) + continue + text = f'{v:.1f}' + if v == best: + text = style.bold(style.green(text)) + elif len(vals) > 2 and v == worst: + text = style.red(text) + if v != best and v > 0: + text += style.dim(f' ({best / v:.1f}×)') + row.append(text) + rows.append(row) + print(table(['Benchmark'] + [r.terminal for r in ok], rows, style)) + print() + + +# }}} + + +def parse_args() -> argparse.Namespace: + terminal_names = [t.name for t in all_terminals()] + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('benchmarks', nargs='*', help='Benchmarks to run, by default all are run. See kitten __benchmark__ --help for the list') + p.add_argument( + '--terminals', '-t', default=','.join(terminal_names), help=f'Comma separated list of terminals to benchmark, from: {", ".join(terminal_names)}' + ) + p.add_argument('--repetitions', '-r', type=int, default=500, help='The number of repetitions of each benchmark. Default: %(default)s') + p.add_argument('--font-family', default='DejaVu Sans Mono', help='Font family to use in all terminals. Default: %(default)s') + p.add_argument('--font-size', type=float, default=11, help='Font size in pts to use in all terminals. Default: %(default)s') + p.add_argument('--columns', type=int, default=120, help='Initial window width in cells. Default: %(default)s') + p.add_argument('--lines', type=int, default=40, help='Initial window height in cells. Default: %(default)s') + p.add_argument( + '--window-size', + default='', + help='Window size in pixels, used only when running under sway. Under sway, windows are made floating so that tiling does not' + ' override the initial size in cells. If this is specified they are additionally resized to it.', + ) + p.add_argument('--cpus', default='', help='Comma separated list of CPUs to pin the terminals to, overrides automatic fast core detection') + p.add_argument('--all-smt-threads', action='store_true', help='Use all SMT threads of the fast cores instead of one per core') + p.add_argument('--no-governor', action='store_true', help='Do not change the CPU frequency governor') + p.add_argument('--no-sway', action='store_true', help='Do not use sway to make window sizes identical, even if running under sway') + p.add_argument( + '--benchmark-timeout', + type=float, + default=180, + help='Max time in seconds for each benchmark in each terminal, after which it is aborted and reported as timed out. Default: %(default)s', + ) + p.add_argument('--output', '-o', default='', help='Also write all results as JSON to the specified file') + p.add_argument('--verbose', '-v', action='store_true', help='Print the command lines used to run the terminals') + args = p.parse_args() + args.sway = sway_available() and not args.no_sway + if args.benchmarks: + avail = available_benchmarks() + if unknown := [x for x in args.benchmarks if x not in avail]: + raise SystemExit(f'Unknown benchmarks: {", ".join(unknown)}. Choose from: {", ".join(avail)}') + else: + args.benchmarks = available_benchmarks() + return args + + +def main() -> None: + if len(sys.argv) > 1 and sys.argv[1] == '--inner': + inner_main(sys.argv[2:]) + return + if not sys.platform.startswith('linux'): + raise SystemExit('This script only works on Linux') + args = parse_args() + # Ensure cleanup code (killing terminals, restoring the governor) runs when terminated + signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(128 + signum)) + style = Style(sys.stdout.isatty()) + err_style = Style(sys.stderr.isatty()) + wanted = [x.strip() for x in args.terminals.split(',') if x.strip()] + known = {t.name: t for t in all_terminals()} + if unknown := [x for x in wanted if x not in known]: + raise SystemExit(f'Unknown terminals: {", ".join(unknown)}') + terminals = [known[x] for x in wanted] + missing = [t.name for t in terminals if not is_installed(t)] + terminals = [t for t in terminals if is_installed(t)] + if not terminals: + raise SystemExit('None of the terminals are installed') + ww = wh = 0 + if args.window_size: + try: + ww, wh = map(int, args.window_size.lower().split('x')) + except ValueError: + raise SystemExit(f'Invalid window size: {args.window_size}') + settings = Settings(args.font_family, args.font_size, args.columns, args.lines, ww, wh) + kitten_exe() # fail early if not found + if args.cpus: + cpus, cpu_how = parse_cpu_list(args.cpus), 'specified on command line' + else: + cpus, cpu_how = detect_fast_cpus(not args.all_smt_threads) + if shutil.which('fc-list'): + fc = subprocess.run(['fc-list', f'{settings.font_family}:spacing=mono', 'family'], capture_output=True, text=True) + if not fc.stdout.strip(): + print( + err_style.yellow(f'Warning: the font {settings.font_family!r} does not seem to be installed, terminals will use fallback fonts'), + file=sys.stderr, + ) + + def log(msg: str) -> None: + if args.verbose: + print(err_style.dim(msg), file=sys.stderr) + + runs: list[TerminalRun] = [] + with performance_governor(not args.no_governor) as governor: + print(style.bold('Comparative terminal benchmark')) + print(f' CPUs: {",".join(map(str, cpus))} ({cpu_how})') + print(f' Governor: {governor}') + print(f' Font: {settings.font_family} {fmt_size(settings.font_size)}pt') + win = f'{settings.columns}x{settings.lines} cells' + if args.sway: + win += f', resized to {ww}x{wh} pixels' if ww else ', floating sway window' + print(f' Window: {win}') + print(f' Reps: {args.repetitions} (timeout of {args.benchmark_timeout:.0f}s per benchmark)') + if missing: + print(f' Skipped: {", ".join(missing)} (not installed)') + print() + sys.stdout.flush() + for t in terminals: + print(err_style.dim(f'Benchmarking {t.name}...'), file=sys.stderr) + r = run_terminal(t, args, settings, cpus, log, lambda name: print(err_style.dim(f' {name}'), file=sys.stderr)) + runs.append(r) + print_terminal_run(r, style) + sys.stdout.flush() + print_summary(runs, style) + if args.output: + with open(args.output, 'w') as f: + json.dump( + [ + { + 'terminal': r.terminal, + 'version': r.version, + 'size': r.size, + 'size_warning': r.size_warning, + 'error': r.error, + 'wall_time': r.wall_time, + 'results': [b.__dict__ for b in r.results], + } + for r in runs + ], + f, + indent=2, + ) + + +if __name__ == '__main__': + main()