hints: Migrate from regexp2 v1 to v2
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run

Use Capture.ByteRange() from v2 instead of a hand-rolled rune to byte
offset mapping. The old mapping required capture offsets to be monotonic
across matches, so a capture inside a lookaround overlapping a later match
in non-ASCII text caused all matches to be silently dropped.

Also propagate match errors instead of discarding them. Drops the v1
dependency, v2 was already pulled in by chroma.
This commit is contained in:
Kovid Goyal 2026-11-07 05:54:16 +05:30
parent 01036b6caa
commit e4988b522d
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
5 changed files with 19 additions and 46 deletions

View file

@ -125,6 +125,10 @@ Detailed list of changes
0.50.1 [unreleased]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- :doc:`Hints kitten </kittens/hints>`: Fix custom regex patterns with capture
groups inside lookarounds that overlap later matches finding no matches at all
when the text contains non-ASCII characters
- Vertical tab bar: Fix clicking the blank line below a tab not activating the tab (:iss:`10810`)
- :doc:`Custom shaders <custom-shaders>`: Add ``screen-shake`` and

3
go.mod
View file

@ -8,7 +8,7 @@ require (
github.com/ALTree/bigfloat v0.3.0
github.com/alecthomas/chroma/v2 v2.27.0
github.com/bmatcuk/doublestar/v4 v4.10.4
github.com/dlclark/regexp2 v1.12.0
github.com/dlclark/regexp2/v2 v2.2.1
github.com/ebitengine/purego v0.11.2
github.com/emmansun/base64 v0.11.0
github.com/google/go-cmp v0.7.0
@ -39,7 +39,6 @@ require (
// replace github.com/kovidgoyal/imaging => ../imaging
require (
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/kovidgoyal/go-shm v1.0.0 // indirect

2
go.sum
View file

@ -8,8 +8,6 @@ github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/bmatcuk/doublestar/v4 v4.10.4 h1:oTKkRmRuOsKMqFe1AQazd9gGBFNEpOmgKp/xzB97dnA=
github.com/bmatcuk/doublestar/v4 v4.10.4/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8=
github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU=

View file

@ -16,8 +16,8 @@ import (
"unicode"
"unicode/utf8"
"github.com/dlclark/regexp2"
"github.com/dlclark/regexp2/syntax"
"github.com/dlclark/regexp2/v2"
"github.com/dlclark/regexp2/v2/syntax"
"github.com/seancfoley/ipaddress-go/ipaddr"
"github.com/kovidgoyal/kitty"
@ -447,55 +447,22 @@ func (self Match) HasNamedGroups() bool {
func find_all_matches(re *regexp2.Regexp, text string) (ans []Match, err error) {
m, err := re.FindStringMatch(text)
if err != nil {
return
}
rune_to_bytes := utils.RuneOffsetsToByteOffsets(text)
get_byte_offset_map := func(groups []regexp2.Group) (ans map[int]int, err error) {
ans = make(map[int]int, len(groups)*2)
rune_offsets := make([]int, 0, len(groups)*2)
for _, g := range groups {
for _, c := range g.Captures {
if _, found := ans[c.Index]; !found {
rune_offsets = append(rune_offsets, c.Index)
ans[c.Index] = -1
}
end := c.Index + c.Length
if _, found := ans[end]; !found {
rune_offsets = append(rune_offsets, end)
ans[end] = -1
}
}
}
slices.Sort(rune_offsets)
for _, pos := range rune_offsets {
if ans[pos] = rune_to_bytes(pos); ans[pos] < 0 {
return nil, fmt.Errorf("Matches are not monotonic cannot map rune offsets to byte offsets")
}
}
return
}
for m != nil {
for m != nil && err == nil {
groups := m.Groups()
bom, err := get_byte_offset_map(groups)
if err != nil {
return nil, err
}
match := Match{Groups: make([]Group, len(groups))}
for i, g := range m.Groups() {
for i, g := range groups {
match.Groups[i].Name = g.Name
match.Groups[i].IsNamed = g.Name != "" && g.Name != strconv.Itoa(i)
for _, c := range g.Captures {
cn := Capture{Text: c.String(), Text_as_runes: c.Runes()}
cn.Rune_Offsets.End = c.Index + c.Length
cn.Rune_Offsets.Start = c.Index
cn.Byte_Offsets.Start, cn.Byte_Offsets.End = bom[c.Index], bom[cn.Rune_Offsets.End]
cn.Rune_Offsets.Start, cn.Rune_Offsets.End = c.RuneIndex, c.RuneIndex+c.RuneLength
start, length := c.ByteRange()
cn.Byte_Offsets.Start, cn.Byte_Offsets.End = start, start+length
match.Groups[i].Captures = append(match.Groups[i].Captures, cn)
}
}
ans = append(ans, match)
m, _ = re.FindNextMatch(m)
m, err = re.FindNextMatch(m)
}
return
}
@ -725,7 +692,9 @@ func find_marks(text string, opts *Options, cli_args ...string) (sanitized_text
if err != nil {
return fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
}
matches, _ = find_all_matches(r, sanitized_text)
if matches, err = find_all_matches(r, sanitized_text); err != nil {
return fmt.Errorf("Failed to match the regex pattern: %#v with error: %w", pattern, err)
}
}
ans = mark(matches, post_processors, group_processors, sanitized_text, opts)
used_pattern = pattern

View file

@ -142,6 +142,9 @@ func TestHintMarking(t *testing.T) {
r(`* 2b687c2 - test1`, `2b687c2`)
opts.Regex = `(?<=got: )sha256.{4}`
r(`got: sha256-L8=`, `sha256-L8=`)
// captures inside lookarounds can overlap subsequent matches
opts.Regex = `([αβγ])(?=([αβγ]+))`
r(`αβγ`, `α`, `β`)
reset()
opts.Type = "word"