// License: GPLv3 Copyright: 2023, Kovid Goyal, package benchmark import ( "bytes" "encoding/json" "errors" "fmt" "math/rand/v2" "os" "slices" "strings" "time" "github.com/kovidgoyal/kitty/tools/cli" "github.com/kovidgoyal/kitty/tools/tty" "github.com/kovidgoyal/kitty/tools/tui/graphics" "github.com/kovidgoyal/kitty/tools/tui/loop" "github.com/kovidgoyal/kitty/tools/utils" "github.com/kovidgoyal/kitty/tools/wcswidth" "golang.org/x/sys/unix" ) var _ = fmt.Print type Options struct { Repetitions int Output string } // How a benchmark is run type run_settings struct { repetitions int // Use the main screen instead of the alternate screen, so that lines // scrolled off the screen go into the scrollback main_screen bool // Allow the terminal to render while the data is being sent render bool } func default_settings() run_settings { return run_settings{repetitions: opts.Repetitions} } const reset = "\x1b]\x1b\\\x1bc" const ascii_printable = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ `~!@#$%^&*()_+-=[]{}\\|;:'\",<.>/?" const control_chars = "\n\t" const chinese_lorem_ipsum = "旦海司有幼雞讀松鼻種比門真目怪少:扒裝虎怕您跑綠蝶黃,位香法士錯乙音造活羽詞坡村目園尺封鳥朋;法松夕點我冬停雪因科對只貓息加黃住蝶,明鴨乾春呢風乙時昔孝助?小紅女父故去。" + "飯躲裝個哥害共買去隻把氣年,己你校跟飛百拉!快石牙飽知唱想土人吹象毛吉每浪四又連見、欠耍外豆雞秋鼻。住步帶。" + "打六申幾麼:或皮又荷隻乙犬孝習秋還何氣;幾裏活打能花是入海乙山節會。種第共後陽沒喜姐三拍弟海肖,行知走亮包,他字幾,的木卜流旦乙左杯根毛。" + "您皮買身苦八手牛目地止哥彩第合麻讀午。原朋河乾種果「才波久住這香松」兄主衣快他玉坐要羽和亭但小山吉也吃耳怕,也爪斗斥可害朋許波怎祖葉卜。" + "行花兩耍許車丟學「示想百吃門高事」不耳見室九星枝買裝,枝十新央發旁品丁青給,科房火;事出出孝肉古:北裝愛升幸百東鼻到從會故北「可休笑物勿三游細斗」娘蛋占犬。我羊波雨跳風。" + "牛大燈兆新七馬,叫這牙後戶耳、荷北吃穿停植身玩間告或西丟再呢,他禾七愛干寺服石安:他次唱息它坐屋父見這衣發現來,苗會開條弓世者吃英定豆哭;跳風掃叫美神。" + "寸再了耍休壯植己,燈錯和,蝶幾欠雞定和愛,司紅後弓第樹會金拉快喝夕見往,半瓜日邊出讀雞苦歌許開;發火院爸乙;四帶亮錯鳥洋個讀。" const misc_unicode = "‘’“”‹›«»‚„ 😀😛😇😈😉😍😎😮👍👎 —–§¶†‡©®™ →⇒•·°±−×÷¼½½¾" + "…µ¢£€¿¡¨´¸ˆ˜ ÀÁÂÃÄÅÆÇÈÉÊË ÌÍÎÏÐÑÒÓÔÕÖØ ŒŠÙÚÛÜÝŸÞßàá âãäåæçèéêëìí" + "îïðñòóôõöøœš ùúûüýÿþªºαΩ∞ ū̀n̂o᷵H̨a̠b̡͓̐c̡͓̐X̡͓̐" var opts Options func benchmark_data(description string, data string, settings run_settings) (duration time.Duration, sent_data_size int, reps int, err error) { term, err := tty.OpenControllingTerm(tty.SetRaw) if err != nil { return 0, 0, 0, err } defer term.RestoreAndClose() write_with_retry := func(data string) (err error) { return term.WriteAllString(data) } state := loop.TerminalStateOptions{Alternate_screen: !settings.main_screen} if err = write_with_retry(state.SetStateEscapeCodes() + loop.DECTCEM.EscapeCodeToReset()); err != nil { return } defer func() { _ = write_with_retry(state.ResetStateEscapeCodes() + loop.DECTCEM.EscapeCodeToSet() + reset) }() const count = 3 const clear_screen = "\x1b[m\x1b[H\x1b[2J" desc := clear_screen + "Running: " + description + "\r\n" const pause_rendering = "\x1b[?2026h" const resume_rendering = "\x1b[?2026l" if !settings.render { if err = write_with_retry(desc + pause_rendering); err != nil { return } } start := time.Now() end_of_loop_reset := desc if !settings.render { end_of_loop_reset += resume_rendering + pause_rendering } for reps < settings.repetitions { if err = write_with_retry(data); err != nil { return } sent_data_size += len(data) reps += 1 if err = write_with_retry(end_of_loop_reset); err != nil { return } } finalize := clear_screen + "Waiting for response indicating parsing finished\r\n" if !settings.render { finalize += resume_rendering } finalize += "\x1b[6n" + strings.Repeat("\x1b[5n", count) if err = write_with_retry(finalize); err != nil { return } q := []byte(strings.Repeat("\x1b[0n", count)) var read_data []byte buf := make([]byte, 8192) for !bytes.Contains(read_data, q) { n, err := term.Read(buf) if err != nil { if (errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EINTR)) && n == 0 { continue } break } read_data = append(read_data, buf[:n]...) } duration = time.Since(start) return } func random_string_of_bytes(n int, alphabet string) string { b := make([]byte, n) al := len(alphabet) for i := range n { b[i] = alphabet[rand.IntN(al)] } return utils.UnsafeBytesToString(b) } type result struct { name, desc string data_sz int duration time.Duration repetitions int // The number of cells drawn per repetition, for text benchmarks, zero otherwise cells int } // The ASCII and Unicode text benchmarks use text with the same structure: the // same lines, of the same widths in cells, ending in CRLF and with tabs at the // same positions. They differ only in the characters used to fill the lines, // so that they are comparable. Since non-ASCII characters are encoded in more // bytes, compare them by the rate at which cells are drawn rather than bytes. type text_line struct { width, tab_at int // tab_at is -1 for lines without a tab } const num_text_lines = 16000 func text_layout() []text_line { r := rand.New(rand.NewPCG(1, 2)) // the same layout every time lines := make([]text_line, num_text_lines) for i := range lines { // Lines wider than the screen wrap lines[i] = text_line{width: r.IntN(101), tab_at: -1} if lines[i].width > 8 && r.IntN(4) == 0 { lines[i].tab_at = r.IntN(lines[i].width) } } return lines } // next returns a character, which may be a multi-codepoint grapheme, and its // width in cells, which must not be more than remaining func text_with_layout(next func(remaining int) (string, int)) (text string, cells int) { var b strings.Builder b.Grow(num_text_lines * 128) for _, line := range text_layout() { tab_done := line.tab_at < 0 for filled := 0; filled < line.width; { if !tab_done && filled >= line.tab_at { b.WriteByte('\t') tab_done = true } limit := line.width if !tab_done { limit = line.tab_at // do not let a wide char step over the tab position } ch, w := next(limit - filled) b.WriteString(ch) filled += w } b.WriteString("\r\n") cells += line.width } return b.String(), cells } func text_benchmark(desc string, data string, cells int, settings run_settings) (r result, err error) { duration, data_sz, reps, err := benchmark_data(desc, data, settings) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, cells}, nil } func ascii_text() (string, int) { return text_with_layout(func(int) (string, int) { i := rand.IntN(len(ascii_printable)) return ascii_printable[i : i+1], 1 }) } func simple_ascii() (r result, err error) { data, cells := ascii_text() return text_benchmark("ASCII text", data, cells, default_settings()) } func ascii_with_scrollback() (r result, err error) { data, cells := ascii_text() s := default_settings() s.main_screen = true return text_benchmark("ASCII text with scrollback", data, cells, s) } func ascii_with_rendering() (r result, err error) { data, cells := ascii_text() s := default_settings() s.render = true return text_benchmark("ASCII text with rendering", data, cells, s) } func unicode_text() (string, int) { // Half the characters are wide: CJK and emoji, half narrow: accented // letters, symbols and letters with combining marks var narrow, wide []string for _, g := range wcswidth.SplitIntoGraphemes(chinese_lorem_ipsum + misc_unicode) { switch wcswidth.Stringwidth(g) { case 1: if g != " " { narrow = append(narrow, g) } case 2: wide = append(wide, g) } } return text_with_layout(func(remaining int) (string, int) { if remaining > 1 && rand.IntN(2) == 0 { return wide[rand.IntN(len(wide))], 2 } if rand.IntN(16) == 0 { return " ", 1 } return narrow[rand.IntN(len(narrow))], 1 }) } func unicode() (r result, err error) { data, cells := unicode_text() return text_benchmark("Unicode text", data, cells, default_settings()) } func unique_unicode() (r result, err error) { const cell_count = 256 * 1024 const combining_count = 0x70 var data strings.Builder data.Grow(cell_count * 10) for i := range cell_count { q := i data.WriteByte('a') for range 3 { data.WriteRune(rune(0x300 + q%combining_count)) q /= combining_count } } const desc = "Unique multi-codepoint Unicode cells" duration, data_sz, reps, err := benchmark_data(desc, data.String(), default_settings()) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, 0}, nil } func ascii_with_csi() (r result, err error) { const sz = 1024*1024 + 17 out := make([]byte, 0, sz+48) chunk := "" for len(out) < sz { q := rand.IntN(100) switch { case (q < 10): chunk = random_string_of_bytes(rand.IntN(72)+1, ascii_printable+control_chars) case (10 <= q && q < 30): chunk = "\x1b[m\x1b[?1h\x1b[H" case (30 <= q && q < 40): chunk = "\x1b[1;2;3;4:3;31m" case (40 <= q && q < 50): chunk = "\x1b[38:5:24;48:2:125:136:147m" case (50 <= q && q < 60): chunk = "\x1b[58;5;44;2m" case (60 <= q && q < 80): chunk = "\x1b[m\x1b[10A\x1b[3E\x1b[2K" case (80 <= q && q < 100): chunk = "\x1b[39m\x1b[10`a\x1b[100b\x1b[?1l" } out = append(out, utils.UnsafeStringToBytes(chunk)...) } out = append(out, "\x1b[m"...) const desc = "CSI codes with few chars" duration, data_sz, reps, err := benchmark_data(desc, utils.UnsafeBytesToString(out), default_settings()) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, 0}, nil } type rgb struct{ r, g, b int } func random_color() rgb { // Components in [100, 255] so that every color has the same encoded length return rgb{100 + rand.IntN(156), 100 + rand.IntN(156), 100 + rand.IntN(156)} } func colored_text_benchmark(desc string, next_color func() rgb) (r result, err error) { alphabet := ascii_printable + control_chars var b strings.Builder b.Grow(2 * 1024 * 1024) for b.Len() < 2*1024*1024 { c := next_color() fmt.Fprintf(&b, "\x1b[38;2;%d;%d;%dm%c", c.r, c.g, c.b, alphabet[rand.IntN(len(alphabet))]) } b.WriteString("\x1b[m") duration, data_sz, reps, err := benchmark_data(desc, b.String(), default_settings()) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, 0}, nil } func truecolor_text() (r result, err error) { // Every char has a random 24-bit foreground, like lolcat output, so nearly every // cell has a distinct style and terminals that intern styles cannot reuse them. return colored_text_benchmark("Truecolor text", random_color) } func palette16_text() (r result, err error) { // The same as truecolor_text except that only 16 distinct colors are used, so // styles can be reused. Comparing the two isolates the cost of many distinct styles. colors := make([]rgb, 16) for i := range colors { colors[i] = random_color() } return colored_text_benchmark("16 color text", func() rgb { return colors[rand.IntN(len(colors))] }) } func halfblocks() (r result, err error) { // Rows of upper half blocks each with random 24-bit foreground and background, // as written by terminal image viewers that use half blocks. var b strings.Builder b.Grow(2 * 1024 * 1024) for b.Len() < 2*1024*1024 { for range 200 { fg, bg := random_color(), random_color() fmt.Fprintf(&b, "\x1b[38;2;%d;%d;%d;48;2;%d;%d;%dm\u2580", fg.r, fg.g, fg.b, bg.r, bg.g, bg.b) } b.WriteString("\x1b[m\r\n") } const desc = "Truecolor half blocks" duration, data_sz, reps, err := benchmark_data(desc, b.String(), default_settings()) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, 0}, nil } func images() (r result, err error) { g := graphics.GraphicsCommand{} g.SetImageId(12345) g.SetQuiet(graphics.GRT_quiet_silent) g.SetAction(graphics.GRT_action_transmit) g.SetFormat(graphics.GRT_format_rgba) const dim = 1024 g.SetDataWidth(dim) g.SetDataHeight(dim) g.DisableCompression = true // dont want to measure the speed of zlib b := strings.Builder{} b.Grow(8 * dim * dim) _ = g.WriteWithPayloadTo(&b, make([]byte, 4*dim*dim)) g.SetAction(graphics.GRT_action_delete) g.SetDelete(graphics.GRT_free_by_id) _ = g.WriteWithPayloadTo(&b, nil) data := b.String() const desc = "Images" duration, data_sz, reps, err := benchmark_data(desc, data, default_settings()) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, 0}, nil } func long_escape_codes() (r result, err error) { data := random_string_of_bytes(8024, ascii_printable) // OSC 6 is document reporting or XTerm special color which kitty ignores after parsing data = strings.Repeat("\x1b]6;"+data+"\x07", 1024) const desc = "Long escape codes" duration, data_sz, reps, err := benchmark_data(desc, data, default_settings()) if err != nil { return result{}, err } return result{"", desc, data_sz, duration, reps, 0}, nil } var divs = []time.Duration{ time.Duration(1), time.Duration(10), time.Duration(100), time.Duration(1000)} func round(d time.Duration, digits int) time.Duration { switch { case d > time.Second: d = d.Round(time.Second / divs[digits]) case d > time.Millisecond: d = d.Round(time.Millisecond / divs[digits]) case d > time.Microsecond: d = d.Round(time.Microsecond / divs[digits]) } return d } func present_result(r result, col_width int) { rate := float64(r.data_sz) / r.duration.Seconds() rate /= 1024. * 1024. f := fmt.Sprintf("%%-%ds", col_width) cells := "" if r.cells > 0 { cells = fmt.Sprintf(" @ \x1b[32m%.1f\x1b[m Mcells/s", float64(r.cells*r.repetitions)/r.duration.Seconds()/1e6) } fmt.Printf(" "+f+" : %-10v @ \x1b[32m%-7.1f\x1b[m MB/s%s\n", r.desc, round(r.duration, 2), rate, cells) } type benchmark struct { name, help string run func() (result, error) } func all_benchmarks() []benchmark { return []benchmark{ {"ascii", "lines of ASCII text", simple_ascii}, {"unicode", "lines of non-ASCII text, the same lines, in cells, as ascii", unicode}, {"scrollback", "the ascii text, on the main screen so lines scrolled off the screen go into the scrollback", ascii_with_scrollback}, {"render", "the ascii text, with rendering allowed while it is processed", ascii_with_rendering}, {"unique_unicode", "cells with many distinct multi-codepoint graphemes", unique_unicode}, {"csi", "ASCII text mixed with many CSI escape codes", ascii_with_csi}, {"long_escape_codes", "long OSC escape codes", long_escape_codes}, {"truecolor_text", "text with a random 24-bit color for every char", truecolor_text}, {"palette16_text", "text with one of 16 colors for every char", palette16_text}, {"halfblocks", "half block characters with random 24-bit foreground and background colors", halfblocks}, {"images", "images sent using the kitty graphics protocol", images}, } } func benchmark_names() (ans []string) { for _, b := range all_benchmarks() { ans = append(ans, b.name) } return } func warmup_data() string { num_colors := 1000 + rand.IntN(1001) type rgb struct{ r, g, b uint8 } colors := make([]rgb, num_colors) for i := range colors { colors[i] = rgb{uint8(rand.IntN(256)), uint8(rand.IntN(256)), uint8(rand.IntN(256))} } colored_text := ascii_printable + control_chars + chinese_lorem_ipsum var b strings.Builder b.Grow(len(colored_text)*20 + len(misc_unicode) + 8) color_idx := 0 for _, ch := range colored_text { c := colors[color_idx%num_colors] fmt.Fprintf(&b, "\x1b[38;2;%d;%d;%dm%c", c.r, c.g, c.b, ch) color_idx++ } b.WriteString("\x1b[m") b.WriteString(misc_unicode) return b.String() } func main(args []string) (err error) { if len(args) == 0 { args = benchmark_names() } for _, a := range args { if slices.Index(benchmark_names(), a) < 0 { return fmt.Errorf("Unknown benchmark: %s, choose from: %s", a, strings.Join(benchmark_names(), ", ")) } } var results []result // First warm up the terminal by getting it to render all chars so that font rendering // time is not polluting the benchmarks. if _, _, _, err = benchmark_data("Warmup", warmup_data(), run_settings{repetitions: 1}); err != nil { return err } time.Sleep(time.Second / 2) for _, b := range all_benchmarks() { if slices.Index(args, b.name) < 0 { continue } r, err := b.run() if err != nil { return err } r.name = b.name results = append(results, r) } fmt.Print(reset) fmt.Println( "These results measure the time it takes the terminal to fully parse all the data sent to it.") fmt.Println("Note that \x1b[31mrendering is suppressed\x1b[m (if the terminal supports the synchronized output escape code) to better benchmark parser performance, except in the render benchmark.") fmt.Println() fmt.Println("Results:") mlen := 10 for _, r := range results { mlen = max(mlen, len(r.desc)) } for _, r := range results { present_result(r, mlen) } if opts.Output != "" { if err = write_results(opts.Output, results); err != nil { return fmt.Errorf("Failed to write results to %s with error: %w", opts.Output, err) } } return } type json_result struct { Name string `json:"name"` Description string `json:"description"` Duration float64 `json:"duration_seconds"` DataSize int `json:"data_size_bytes"` Repetitions int `json:"repetitions"` MBPerSecond float64 `json:"mb_per_second"` CellsPerSecond float64 `json:"cells_per_second,omitempty"` } func write_results(path string, results []result) error { ans := make([]json_result, len(results)) for i, r := range results { ans[i] = json_result{ Name: r.name, Description: r.desc, Duration: r.duration.Seconds(), DataSize: r.data_sz, Repetitions: r.repetitions, MBPerSecond: float64(r.data_sz) / r.duration.Seconds() / (1024. * 1024.), } if r.cells > 0 { ans[i].CellsPerSecond = float64(r.cells*r.repetitions) / r.duration.Seconds() } } data, err := json.MarshalIndent(ans, "", " ") if err != nil { return err } return os.WriteFile(path, append(data, '\n'), 0o644) } func help_text() string { var b strings.Builder b.WriteString("Benchmarking works by sending large amounts of data to the TTY device and waiting for the terminal to process the data and respond to queries sent to it in the data. ") b.WriteString("Rendering is suppressed during benchmarking to focus on parser performance, using the synchronized output escape code, except in the render benchmark. ") b.WriteString("Note that rendering in modern terminals is typically asynchronous so it wont be properly benchmarked by this kitten. ") b.WriteString("All benchmarks use the alternate screen, except the scrollback benchmark.\n\n") b.WriteString("To run only particular benchmarks, specify them on the command line. The available benchmarks are:\n\n") for _, bm := range all_benchmarks() { fmt.Fprintf(&b, "* :code:`%s`: %s\n\n", bm.name, bm.help) } return b.String() } func EntryPoint(root *cli.Command) { sc := root.AddSubCommand(&cli.Command{ Name: "__benchmark__", ShortDescription: "Run various benchmarks", HelpText: help_text(), Usage: "[options] [optional benchmark to run ...]", Hidden: true, Run: func(cmd *cli.Command, args []string) (ret int, err error) { if err = cmd.GetOptionValues(&opts); err != nil { return 1, err } opts.Repetitions = max(1, opts.Repetitions) if err = main(args); err != nil { ret = 1 } return }, }) sc.Add(cli.OptionSpec{ Name: "--repetitions", Default: "500", Type: "int", Help: "The number of repetitions of each benchmark", }) sc.Add(cli.OptionSpec{ Name: "--output -o", Help: "Also write the results to the specified file, as JSON. For every benchmark, the file has its name, description, duration, the amount of data sent, the number of repetitions, the throughput in MB per second and, for the text benchmarks, the number of cells drawn per second.", }) }