From 0762c21c440289d25ff5ce10ca6cbc4ea170a589 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 28 Sep 2026 06:53:52 +0530 Subject: [PATCH] Cleanup previous PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use Unicode word boundaries with the stdlib regexp engine Go's regexp only treats ASCII characters as word characters for \b, while regexp2 uses Unicode ones, so switching the builtin path pattern to the stdlib engine changed which paths were marked next to non-ASCII text, for example config.yaml文件 marked config.yaml. Match against a proxy of the text in which every non-ASCII rune is replaced by a single ASCII byte that is a word character iff the rune is one, then map the offsets back. --- kittens/hints/marks.go | 39 ++++++++++++++++++++++++++++++++++--- kittens/hints/marks_test.go | 4 ++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/kittens/hints/marks.go b/kittens/hints/marks.go index 5a685990d..737be04b9 100644 --- a/kittens/hints/marks.go +++ b/kittens/hints/marks.go @@ -17,6 +17,7 @@ import ( "unicode/utf8" "github.com/dlclark/regexp2" + "github.com/dlclark/regexp2/syntax" "github.com/seancfoley/ipaddress-go/ipaddr" "github.com/kovidgoyal/kitty" @@ -490,13 +491,44 @@ func find_all_matches(re *regexp2.Regexp, text string) (ans []Match, err error) return } -func find_all_matches_stdlib(re *regexp.Regexp, text string) (ans []Match) { +// Go's regexp only considers ASCII characters when matching \b whereas regexp2 +// uses Unicode word characters. So replace every non-ASCII rune with a single +// ASCII byte that is a word character iff the rune is one, and return the byte +// offsets of the runes in text, to map match offsets back. +func ascii_word_proxy(text string) (proxy string, byte_offsets []int) { + if !slices.ContainsFunc(utils.UnsafeStringToBytes(text), func(b byte) bool { return b >= utf8.RuneSelf }) { + return text, nil + } + buf := make([]byte, 0, len(text)) + byte_offsets = make([]int, 0, len(text)+1) + for i, r := range text { + byte_offsets = append(byte_offsets, i) + switch { + case r < utf8.RuneSelf: + buf = append(buf, byte(r)) + case syntax.IsWordChar(r): + buf = append(buf, '_') + default: + buf = append(buf, 0x7f) + } + } + return string(buf), append(byte_offsets, len(text)) +} + +func find_all_matches_stdlib(re *regexp.Regexp, text string, unicode_word_boundaries bool) (ans []Match) { + haystack, byte_offsets := text, []int(nil) + if unicode_word_boundaries { + haystack, byte_offsets = ascii_word_proxy(text) + } names := re.SubexpNames() - for _, loc := range re.FindAllStringSubmatchIndex(text, -1) { + for _, loc := range re.FindAllStringSubmatchIndex(haystack, -1) { match := Match{Groups: make([]Group, len(names))} for i, name := range names { match.Groups[i] = Group{Name: name, IsNamed: name != ""} if s, e := loc[2*i], loc[2*i+1]; s > -1 { + if byte_offsets != nil { + s, e = byte_offsets[s], byte_offsets[e] + } c := Capture{Text: text[s:e]} c.Byte_Offsets.Start, c.Byte_Offsets.End = s, e match.Groups[i].Captures = []Capture{c} @@ -677,7 +709,8 @@ func find_marks(text string, opts *Options, cli_args ...string) (sanitized_text if err != nil { return fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err) } - matches = find_all_matches_stdlib(r, sanitized_text) + // the proxy text is only valid for patterns that dont reference specific non-ASCII characters + matches = find_all_matches_stdlib(r, sanitized_text, opts.Type == "path" || opts.Type == "linenum") } else { r, err := regexp2.Compile(pattern, regexp2.RE2) if err != nil { diff --git a/kittens/hints/marks_test.go b/kittens/hints/marks_test.go index 059084196..0639140ee 100644 --- a/kittens/hints/marks_test.go +++ b/kittens/hints/marks_test.go @@ -102,6 +102,10 @@ func TestHintMarking(t *testing.T) { long_line := strings.Repeat("1,", 20000) r(long_line) r("漢字 "+long_line+" some/path", "some/path") + r("漢字"+long_line+"a.txt", "漢字"+long_line+"a.txt") + // \b must use Unicode word characters, like regexp2 + r("config.yaml文件") + r("a.txt文 é/b.c", "a.txt文", "é/b.c") opts.Type = "linenum" r(long_line) m(long_line+" a/file.c:23", "a/file.c", 23)