Cleanup previous PR

Use Unicode word boundaries with the stdlib regexp engine

Go's regexp only treats ASCII characters as word characters for \b, while
regexp2 uses Unicode ones, so switching the builtin path pattern to the
stdlib engine changed which paths were marked next to non-ASCII text, for
example config.yaml文件 marked config.yaml. Match against a proxy of the
text in which every non-ASCII rune is replaced by a single ASCII byte that
is a word character iff the rune is one, then map the offsets back.
This commit is contained in:
Kovid Goyal 2026-09-28 06:53:52 +05:30
parent a6afadb365
commit 0762c21c44
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
2 changed files with 40 additions and 3 deletions

View file

@ -17,6 +17,7 @@ import (
"unicode/utf8"
"github.com/dlclark/regexp2"
"github.com/dlclark/regexp2/syntax"
"github.com/seancfoley/ipaddress-go/ipaddr"
"github.com/kovidgoyal/kitty"
@ -490,13 +491,44 @@ func find_all_matches(re *regexp2.Regexp, text string) (ans []Match, err error)
return
}
func find_all_matches_stdlib(re *regexp.Regexp, text string) (ans []Match) {
// Go's regexp only considers ASCII characters when matching \b whereas regexp2
// uses Unicode word characters. So replace every non-ASCII rune with a single
// ASCII byte that is a word character iff the rune is one, and return the byte
// offsets of the runes in text, to map match offsets back.
func ascii_word_proxy(text string) (proxy string, byte_offsets []int) {
if !slices.ContainsFunc(utils.UnsafeStringToBytes(text), func(b byte) bool { return b >= utf8.RuneSelf }) {
return text, nil
}
buf := make([]byte, 0, len(text))
byte_offsets = make([]int, 0, len(text)+1)
for i, r := range text {
byte_offsets = append(byte_offsets, i)
switch {
case r < utf8.RuneSelf:
buf = append(buf, byte(r))
case syntax.IsWordChar(r):
buf = append(buf, '_')
default:
buf = append(buf, 0x7f)
}
}
return string(buf), append(byte_offsets, len(text))
}
func find_all_matches_stdlib(re *regexp.Regexp, text string, unicode_word_boundaries bool) (ans []Match) {
haystack, byte_offsets := text, []int(nil)
if unicode_word_boundaries {
haystack, byte_offsets = ascii_word_proxy(text)
}
names := re.SubexpNames()
for _, loc := range re.FindAllStringSubmatchIndex(text, -1) {
for _, loc := range re.FindAllStringSubmatchIndex(haystack, -1) {
match := Match{Groups: make([]Group, len(names))}
for i, name := range names {
match.Groups[i] = Group{Name: name, IsNamed: name != ""}
if s, e := loc[2*i], loc[2*i+1]; s > -1 {
if byte_offsets != nil {
s, e = byte_offsets[s], byte_offsets[e]
}
c := Capture{Text: text[s:e]}
c.Byte_Offsets.Start, c.Byte_Offsets.End = s, e
match.Groups[i].Captures = []Capture{c}
@ -677,7 +709,8 @@ func find_marks(text string, opts *Options, cli_args ...string) (sanitized_text
if err != nil {
return fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
}
matches = find_all_matches_stdlib(r, sanitized_text)
// the proxy text is only valid for patterns that dont reference specific non-ASCII characters
matches = find_all_matches_stdlib(r, sanitized_text, opts.Type == "path" || opts.Type == "linenum")
} else {
r, err := regexp2.Compile(pattern, regexp2.RE2)
if err != nil {

View file

@ -102,6 +102,10 @@ func TestHintMarking(t *testing.T) {
long_line := strings.Repeat("1,", 20000)
r(long_line)
r("漢字 "+long_line+" some/path", "some/path")
r("漢字"+long_line+"a.txt", "漢字"+long_line+"a.txt")
// \b must use Unicode word characters, like regexp2
r("config.yaml文件")
r("a.txt文 é/b.c", "a.txt文", "é/b.c")
opts.Type = "linenum"
r(long_line)
m(long_line+" a/file.c:23", "a/file.c", 23)