kitty/tools/disk_cache/implementation.go
Kovid Goyal 6d8f7ac14f
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run
Cleanup previous PR
Let preview renderers supply extra cache key data instead of checking for
the cmd_renderer type. Include the command line in the key for custom
previews and the video_preview settings in the key for video thumbnails,
so that changing them no longer shows stale previews. GetPath and AddPath
now take this extra key data, replacing GetPathWithAlias.

Track the entry each path uses per extra key, so that previews of one file
with different settings do not evict each other. Do not remove an entry
while another path, such as another symlink to the same file, still uses
it. Cache hits are now recorded so that such shared entries are known.
Failing to remove an orphaned entry on a cache hit no longer turns the hit
into an error.

Fix ensure_entries() discarding unwritten changes when the entries file
does not exist yet, which lost the first path recorded in a new cache.
Also reset the path map when clearing the cache.
2026-11-11 21:26:01 +05:30

572 lines
14 KiB
Go

package disk_cache
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io/fs"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"syscall"
"time"
"github.com/kovidgoyal/kitty/tools/utils"
)
var _ = fmt.Print
type file_state struct {
Size int64
ModTime time.Time
Inode uint64
}
func (s file_state) String() string {
return fmt.Sprintf("fs{Size: %d, Inode: %d, ModTime: %s}", s.Size, s.Inode, s.ModTime)
}
func (s *file_state) equal(o *file_state) bool {
return o != nil && s.Size == o.Size && s.ModTime.Equal(o.ModTime) && s.Inode == o.Inode
}
func get_file_state(fi fs.FileInfo) *file_state {
// The Sys() method returns the underlying data source (can be nil).
// For Unix-like systems, it's a *syscall.Stat_t.
stat, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
// For non-Unix systems, you might not have an inode.
// In that case, you can fall back to using only size and mod time.
return &file_state{
Size: fi.Size(),
ModTime: fi.ModTime(),
Inode: 0, // Inode not available
}
}
return &file_state{
Size: fi.Size(),
ModTime: fi.ModTime(),
Inode: stat.Ino,
}
}
func get_file_state_from_path(path string) (*file_state, error) {
if s, err := os.Stat(path); err != nil {
return nil, err
} else {
return get_file_state(s), nil
}
}
const GET_DIR_PREFIX = "getdir-"
func new_disk_cache(path string, max_size int64) (dc *DiskCache, err error) {
if path, err = filepath.Abs(path); err != nil {
return
}
if err = os.MkdirAll(path, 0o700); err != nil {
return
}
ans := &DiskCache{Path: path, MaxSize: max_size}
if err = ans.lock(); err != nil {
return
}
defer ans.unlock_and_set_err(&err)
if err = ans.ensure_entries(); err != nil {
return
}
defer func() {
if we := ans.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
if _, err := ans.prune(); err != nil {
return nil, err
}
if ans.get_dir, err = os.MkdirTemp(ans.Path, GET_DIR_PREFIX+"*"); err != nil {
return
}
if err = utils.AtExitRmtree(ans.get_dir); err != nil {
return
}
return ans, nil
}
func key_for_path(path, extra_key string) (key string, err error) {
resolved_path, err := filepath.EvalSymlinks(path)
if err != nil {
return
}
if resolved_path, err = filepath.Abs(resolved_path); err != nil {
return
}
s, err := os.Stat(resolved_path)
if err != nil {
return
}
data := fmt.Sprintf("%s\x00%d\x00%d", resolved_path, s.Size(), s.ModTime().UnixNano())
if extra_key != "" {
data += "\x00" + extra_key
}
sum := sha256.Sum256(utils.UnsafeStringToBytes(data))
return hex.EncodeToString(sum[:]), nil
}
// lock() and unlock() must be paired: unlock() must only be called after a
// successful lock(). lock() is not re-entrant, so it must not be called while
// the lock is already held, that will deadlock.
func (dc *DiskCache) lock() (err error) {
dc.lock_mutex.Lock()
defer func() {
if err != nil {
_ = dc.unlock()
}
}()
if dc.lock_file, err = os.OpenFile(filepath.Join(dc.Path, "lockfile"), os.O_RDWR|os.O_CREATE, 0o600); err != nil {
return
}
return utils.LockFileExclusive(dc.lock_file)
}
func (dc *DiskCache) unlock() (err error) {
defer dc.lock_mutex.Unlock()
if dc.lock_file != nil {
err = utils.UnlockFile(dc.lock_file)
if ce := dc.lock_file.Close(); ce != nil && err == nil {
err = ce
}
dc.lock_file = nil
}
return
}
const ENTRIES_NAME = "entries.json"
func (dc *DiskCache) entries_path() string { return filepath.Join(dc.Path, ENTRIES_NAME) }
func (dc *DiskCache) write_entries_if_dirty() (err error) {
if !dc.entries_dirty {
return
}
path := dc.entries_path()
defer func() {
if err == nil {
dc.entries_dirty = false
if s, serr := get_file_state_from_path(path); serr == nil {
dc.entries_last_read_state = s
}
}
}()
if d, err := json.Marshal(dc.entries); err != nil {
return err
} else {
// use a rename so that the inode number changes
// dont bother with full utils.AtomicWriteFile() as it is slower
temp := path + ".temp"
removed := false
defer func() {
if !removed {
_ = os.Remove(temp)
removed = true
}
}()
if err = os.WriteFile(temp, d, 0o600); err == nil {
if err = os.Rename(temp, path); err == nil {
removed = true
}
}
return err
}
}
func (e Entry) String() string {
return fmt.Sprintf("Entry{Key: %s, Size: %d, LastUsed: %s}", e.Key, e.Size, e.LastUsed)
}
func (dc *DiskCache) entries_from_folders() (total_size int64, ans map[string]*Entry, sorted []*Entry, err error) {
if entries, err := os.ReadDir(dc.Path); err != nil {
return 0, nil, nil, err
} else {
ans = make(map[string]*Entry)
var total int64
for _, x := range entries {
if x.IsDir() {
if sub_entries, err := os.ReadDir(filepath.Join(dc.Path, x.Name())); err == nil && len(sub_entries) == 1 {
key := sub_entries[0].Name()
path := dc.folder_for_key(key)
if file_entries, err := os.ReadDir(path); err == nil {
e := Entry{Key: key}
for _, f := range file_entries {
if fi, err := f.Info(); err == nil {
e.Size += fi.Size()
if fi.ModTime().After(e.LastUsed) {
e.LastUsed = fi.ModTime()
}
}
}
ans[key] = &e
total += e.Size
}
}
}
}
sorted = utils.Values(ans)
slices.SortFunc(sorted, func(a, b *Entry) int {
return a.LastUsed.Compare(b.LastUsed)
})
return total, ans, sorted, nil
}
}
func (dc *DiskCache) rebuild_entries() error {
total, ans, sorted, err := dc.entries_from_folders()
if err != nil {
return err
}
dc.entries = Metadata{TotalSize: total, SortedEntries: sorted, PathMap: make(map[string]string)}
dc.entry_map = ans
dc.entries_dirty = true
return dc.write_entries_if_dirty()
}
func (dc *DiskCache) ensure_entries() error {
needed := dc.entry_map == nil || dc.entries_last_read_state == nil
path := dc.entries_path()
var fstate *file_state
if !needed {
if s, err := get_file_state_from_path(path); err == nil {
fstate = s
if !s.equal(dc.entries_last_read_state) {
needed = true
}
}
}
if needed {
if data, err := os.ReadFile(path); err != nil {
if os.IsNotExist(err) {
dc.entry_map = make(map[string]*Entry)
dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
// Write the file so that its state is recorded. Otherwise every
// later call reloads and discards in-memory changes not yet written.
dc.entries_dirty = true
return dc.write_entries_if_dirty()
} else {
return err
}
} else {
dc.read_count += 1
dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
if err := json.Unmarshal(data, &dc.entries); err != nil {
// corrupted data
dc.rebuild_entries()
} else {
if fstate == nil {
if s, err := get_file_state_from_path(path); err == nil {
fstate = s
}
}
dc.entries_last_read_state = fstate
}
dc.entry_map = make(map[string]*Entry)
for _, e := range dc.entries.SortedEntries {
dc.entry_map[e.Key] = e
}
}
}
return nil
}
func (dc *DiskCache) folder_for_key(key string) (ans string) {
if len(key) < 5 {
ans = filepath.Join(key, key)
} else {
ans = filepath.Join(key[:4], key)
}
return filepath.Join(dc.Path, ans)
}
func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) {
dest := filepath.Join(dc.get_dir, key+"-"+filepath.Base(path))
if err := os.Link(path, dest); err == nil {
return dest, nil
} else if !os.IsExist(err) {
return "", err
}
src_info, err := os.Stat(path)
if err != nil {
return "", err
}
if dest_info, err := os.Stat(dest); err == nil && os.SameFile(src_info, dest_info) {
return dest, nil
}
// Replace dest atomically so that a path previously returned to a
// caller is never missing, even momentarily.
temp := dest + ".temp"
_ = os.Remove(temp)
if err := os.Link(path, temp); err != nil {
return "", err
}
if err := os.Rename(temp, dest); err != nil {
_ = os.Remove(temp)
return "", err
}
return dest, nil
}
func (dc *DiskCache) get(key string, items []string) (map[string]string, error) {
if err := dc.ensure_entries(); err != nil {
return nil, err
}
base := dc.folder_for_key(key)
if len(items) == 0 {
if entries, err := os.ReadDir(base); err != nil {
if os.IsNotExist(err) {
err = nil
}
return nil, err
} else {
for _, e := range entries {
items = append(items, e.Name())
}
}
} else {
if s, err := os.Stat(base); err != nil || !s.IsDir() {
if os.IsNotExist(err) {
err = nil
}
return nil, err
}
}
ans := make(map[string]string, len(items))
for _, x := range items {
p := filepath.Join(base, x)
if s, err := os.Stat(p); err != nil || s.IsDir() {
continue
}
dest, _ := dc.export_to_get_dir(key, p)
if dest != "" {
ans[x] = dest
}
}
if len(items) > 0 {
dc.update_timestamp(key)
}
return ans, dc.write_entries_if_dirty()
}
func (dc *DiskCache) clear() (err error) {
if entries, err := os.ReadDir(dc.Path); err != nil {
return err
} else {
defer func() {
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
for _, x := range entries {
if x.IsDir() && !strings.HasPrefix(x.Name(), GET_DIR_PREFIX) {
_ = os.RemoveAll(filepath.Join(dc.Path, x.Name()))
}
}
dc.entries_dirty = true
dc.entry_map = make(map[string]*Entry)
dc.entries.SortedEntries = make([]*Entry, 0)
dc.entries.PathMap = make(map[string]string)
dc.entries.TotalSize = 0
}
return
}
func (dc *DiskCache) remove(key string) (err error) {
if err = dc.ensure_entries(); err != nil {
return
}
defer func() {
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
base := dc.folder_for_key(key)
if err = os.RemoveAll(base); err == nil {
t := dc.entry_map[key]
if t != nil {
delete(dc.entry_map, key)
dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
dc.entries.SortedEntries = utils.Filter(dc.entries.SortedEntries, func(x *Entry) bool { return x.Key != key })
dc.entries_dirty = true
}
}
return
}
func (dc *DiskCache) prune() (bool, error) {
if dc.MaxSize < 1 || dc.entries.TotalSize <= dc.MaxSize {
return false, nil
}
for dc.entries.TotalSize > dc.MaxSize && len(dc.entries.SortedEntries) > 0 {
base := dc.folder_for_key(dc.entries.SortedEntries[0].Key)
if err := os.RemoveAll(base); err == nil {
t := dc.entries.SortedEntries[0]
delete(dc.entry_map, t.Key)
maps.DeleteFunc(dc.entries.PathMap, func(path, key string) bool { return key == t.Key })
dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
dc.entries.SortedEntries = dc.entries.SortedEntries[1:]
dc.entries_dirty = true
} else {
return false, err
}
}
return true, nil
}
func (dc *DiskCache) update_timestamp(key string) {
t := dc.entry_map[key]
if t == nil {
// the folder for key exists but is not in the metadata, can happen
// if entries.json was written by an older, racy version
return
}
t.LastUsed = time.Now()
if idx := slices.Index(dc.entries.SortedEntries, t); idx < 0 {
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
} else {
copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:])
dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t
}
dc.entries_dirty = true
}
func (dc *DiskCache) update_accounting(key string, changed int64) (err error) {
t := dc.entry_map[key]
if t == nil {
t = &Entry{Key: key}
dc.entry_map[key] = t
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
}
old_size := t.Size
t.Size += changed
t.Size = max(0, t.Size)
dc.entries.TotalSize += t.Size - old_size
dc.entries_dirty = true
dc.update_timestamp(key)
dc.prune()
return dc.write_entries_if_dirty()
}
func (dc *DiskCache) keys() (ans []string, err error) {
if err = dc.ensure_entries(); err != nil {
return
}
ans = make([]string, len(dc.entries.SortedEntries))
for i, e := range dc.entries.SortedEntries {
ans[i] = e.Key
}
return
}
func (dc *DiskCache) add_path(path, key string, items map[string][]byte) (ans map[string]string, err error) {
if err = dc.ensure_entries(); err != nil {
return
}
defer func() {
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
if orphan := dc.set_key_for_path(path, key); orphan != "" {
if err = dc.remove(orphan); err != nil {
return
}
}
return dc.add(key, items)
}
// path_map_key returns the PathMap key for path. It includes extra_key so
// that, for example, previews of one file by different commands do not evict
// each other. An empty extra_key gives the same PathMap key as older versions.
func path_map_key(path, extra_key string) string {
if extra_key == "" {
return path
}
return path + "\x00" + extra_key
}
// set_key_for_path records that path now uses the entry for key. It returns
// the entry path used previously if no other path, such as another symlink to
// the same file, still uses it, so that the caller can remove it.
func (dc *DiskCache) set_key_for_path(path, key string) (orphan string) {
existing := dc.entries.PathMap[path]
if existing == key {
return ""
}
dc.entries.PathMap[path] = key
dc.entries_dirty = true
if existing == "" {
return ""
}
for _, k := range dc.entries.PathMap {
if k == existing {
return ""
}
}
return existing
}
func (dc *DiskCache) add(key string, items map[string][]byte) (ans map[string]string, err error) {
if err = dc.ensure_entries(); err != nil {
return
}
base := dc.folder_for_key(key)
if err = os.MkdirAll(base, 0o700); err != nil {
return
}
var changed int64
defer func() {
e := dc.update_accounting(key, changed)
if err == nil {
err = e
}
}()
ans = make(map[string]string, len(items))
for x, data := range items {
p := filepath.Join(base, x)
var before int64
exists := false
if s, err := os.Stat(p); err == nil {
before = s.Size()
exists = true
}
if len(data) == 0 {
if exists {
if err = os.Remove(p); err != nil {
if !os.IsNotExist(err) {
return
}
err = nil
}
changed -= before
}
} else {
// unlink the file so that writing to it does not change a
// previously linked copy created by get()
if exists {
_ = os.Remove(p)
}
if err = os.WriteFile(p, data, 0o700); err != nil {
return
}
changed += int64(len(data)) - before
if dest, err := dc.export_to_get_dir(key, p); err != nil {
return ans, err
} else {
ans[x] = dest
}
}
}
return
}