mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-21 23:54:57 +00:00
Cleanup previous PR
Use Unicode word boundaries with the stdlib regexp engine Go's regexp only treats ASCII characters as word characters for \b, while regexp2 uses Unicode ones, so switching the builtin path pattern to the stdlib engine changed which paths were marked next to non-ASCII text, for example config.yaml文件 marked config.yaml. Match against a proxy of the text in which every non-ASCII rune is replaced by a single ASCII byte that is a word character iff the rune is one, then map the offsets back.
This commit is contained in:
parent
a6afadb365
commit
0762c21c44
2 changed files with 40 additions and 3 deletions
|
|
@ -17,6 +17,7 @@ import (
|
|||
"unicode/utf8"
|
||||
|
||||
"github.com/dlclark/regexp2"
|
||||
"github.com/dlclark/regexp2/syntax"
|
||||
"github.com/seancfoley/ipaddress-go/ipaddr"
|
||||
|
||||
"github.com/kovidgoyal/kitty"
|
||||
|
|
@ -490,13 +491,44 @@ func find_all_matches(re *regexp2.Regexp, text string) (ans []Match, err error)
|
|||
return
|
||||
}
|
||||
|
||||
func find_all_matches_stdlib(re *regexp.Regexp, text string) (ans []Match) {
|
||||
// Go's regexp only considers ASCII characters when matching \b whereas regexp2
|
||||
// uses Unicode word characters. So replace every non-ASCII rune with a single
|
||||
// ASCII byte that is a word character iff the rune is one, and return the byte
|
||||
// offsets of the runes in text, to map match offsets back.
|
||||
func ascii_word_proxy(text string) (proxy string, byte_offsets []int) {
|
||||
if !slices.ContainsFunc(utils.UnsafeStringToBytes(text), func(b byte) bool { return b >= utf8.RuneSelf }) {
|
||||
return text, nil
|
||||
}
|
||||
buf := make([]byte, 0, len(text))
|
||||
byte_offsets = make([]int, 0, len(text)+1)
|
||||
for i, r := range text {
|
||||
byte_offsets = append(byte_offsets, i)
|
||||
switch {
|
||||
case r < utf8.RuneSelf:
|
||||
buf = append(buf, byte(r))
|
||||
case syntax.IsWordChar(r):
|
||||
buf = append(buf, '_')
|
||||
default:
|
||||
buf = append(buf, 0x7f)
|
||||
}
|
||||
}
|
||||
return string(buf), append(byte_offsets, len(text))
|
||||
}
|
||||
|
||||
func find_all_matches_stdlib(re *regexp.Regexp, text string, unicode_word_boundaries bool) (ans []Match) {
|
||||
haystack, byte_offsets := text, []int(nil)
|
||||
if unicode_word_boundaries {
|
||||
haystack, byte_offsets = ascii_word_proxy(text)
|
||||
}
|
||||
names := re.SubexpNames()
|
||||
for _, loc := range re.FindAllStringSubmatchIndex(text, -1) {
|
||||
for _, loc := range re.FindAllStringSubmatchIndex(haystack, -1) {
|
||||
match := Match{Groups: make([]Group, len(names))}
|
||||
for i, name := range names {
|
||||
match.Groups[i] = Group{Name: name, IsNamed: name != ""}
|
||||
if s, e := loc[2*i], loc[2*i+1]; s > -1 {
|
||||
if byte_offsets != nil {
|
||||
s, e = byte_offsets[s], byte_offsets[e]
|
||||
}
|
||||
c := Capture{Text: text[s:e]}
|
||||
c.Byte_Offsets.Start, c.Byte_Offsets.End = s, e
|
||||
match.Groups[i].Captures = []Capture{c}
|
||||
|
|
@ -677,7 +709,8 @@ func find_marks(text string, opts *Options, cli_args ...string) (sanitized_text
|
|||
if err != nil {
|
||||
return fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
|
||||
}
|
||||
matches = find_all_matches_stdlib(r, sanitized_text)
|
||||
// the proxy text is only valid for patterns that dont reference specific non-ASCII characters
|
||||
matches = find_all_matches_stdlib(r, sanitized_text, opts.Type == "path" || opts.Type == "linenum")
|
||||
} else {
|
||||
r, err := regexp2.Compile(pattern, regexp2.RE2)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -102,6 +102,10 @@ func TestHintMarking(t *testing.T) {
|
|||
long_line := strings.Repeat("1,", 20000)
|
||||
r(long_line)
|
||||
r("漢字 "+long_line+" some/path", "some/path")
|
||||
r("漢字"+long_line+"a.txt", "漢字"+long_line+"a.txt")
|
||||
// \b must use Unicode word characters, like regexp2
|
||||
r("config.yaml文件")
|
||||
r("a.txt文 é/b.c", "a.txt文", "é/b.c")
|
||||
opts.Type = "linenum"
|
||||
r(long_line)
|
||||
m(long_line+" a/file.c:23", "a/file.c", 23)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue