kitty/tools/disk_cache/implementation.go
Kovid Goyal 517f418f9c
Cleanup previous PR
Drop the redundant v2 prefix from the disk cache key, as the key format
already changed when the inode was added.

Render the file the cache key was computed from, so that retargeting a
symlink while rendering, even if it is restored before AddPath() checks
the key again, cannot store the preview of one file under the key of
another. Renderers that need the selected path, custom preview commands
and calibre, opt out via NeedsSelectedPath(). The selected path is also
used when the target has a different extension, as renderers may detect
the format from it.
2026-11-17 19:11:40 +05:30

574 lines
14 KiB
Go

package disk_cache
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io/fs"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"syscall"
"time"
"github.com/kovidgoyal/kitty/tools/utils"
)
var _ = fmt.Print
type file_state struct {
Size int64
ModTime time.Time
Inode uint64
}
func (s file_state) String() string {
return fmt.Sprintf("fs{Size: %d, Inode: %d, ModTime: %s}", s.Size, s.Inode, s.ModTime)
}
func (s *file_state) equal(o *file_state) bool {
return o != nil && s.Size == o.Size && s.ModTime.Equal(o.ModTime) && s.Inode == o.Inode
}
func get_file_state(fi fs.FileInfo) *file_state {
// The Sys() method returns the underlying data source (can be nil).
// For Unix-like systems, it's a *syscall.Stat_t.
stat, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
// For non-Unix systems, you might not have an inode.
// In that case, you can fall back to using only size and mod time.
return &file_state{
Size: fi.Size(),
ModTime: fi.ModTime(),
Inode: 0, // Inode not available
}
}
return &file_state{
Size: fi.Size(),
ModTime: fi.ModTime(),
Inode: stat.Ino,
}
}
func get_file_state_from_path(path string) (*file_state, error) {
if s, err := os.Stat(path); err != nil {
return nil, err
} else {
return get_file_state(s), nil
}
}
const GET_DIR_PREFIX = "getdir-"
func new_disk_cache(path string, max_size int64) (dc *DiskCache, err error) {
if path, err = filepath.Abs(path); err != nil {
return
}
if err = os.MkdirAll(path, 0o700); err != nil {
return
}
ans := &DiskCache{Path: path, MaxSize: max_size}
if err = ans.lock(); err != nil {
return
}
defer ans.unlock_and_set_err(&err)
if err = ans.ensure_entries(); err != nil {
return
}
defer func() {
if we := ans.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
if _, err := ans.prune(); err != nil {
return nil, err
}
if ans.get_dir, err = os.MkdirTemp(ans.Path, GET_DIR_PREFIX+"*"); err != nil {
return
}
if err = utils.AtExitRmtree(ans.get_dir); err != nil {
return
}
return ans, nil
}
// key_for_path returns the cache key for path and the absolute path of the
// file it resolves to, whose state the key describes.
func key_for_path(path, extra_key string) (key, resolved_path string, err error) {
if resolved_path, err = filepath.EvalSymlinks(path); err != nil {
return
}
if resolved_path, err = filepath.Abs(resolved_path); err != nil {
return
}
info, err := os.Stat(resolved_path)
if err != nil {
return
}
s := get_file_state(info)
data := fmt.Sprintf("%s\x00%d\x00%d\x00%d", resolved_path, s.Inode, s.Size, s.ModTime.UnixNano())
if extra_key != "" {
data += "\x00" + extra_key
}
sum := sha256.Sum256(utils.UnsafeStringToBytes(data))
return hex.EncodeToString(sum[:]), resolved_path, nil
}
// lock() and unlock() must be paired: unlock() must only be called after a
// successful lock(). lock() is not re-entrant, so it must not be called while
// the lock is already held, that will deadlock.
func (dc *DiskCache) lock() (err error) {
dc.lock_mutex.Lock()
defer func() {
if err != nil {
_ = dc.unlock()
}
}()
if dc.lock_file, err = os.OpenFile(filepath.Join(dc.Path, "lockfile"), os.O_RDWR|os.O_CREATE, 0o600); err != nil {
return
}
return utils.LockFileExclusive(dc.lock_file)
}
func (dc *DiskCache) unlock() (err error) {
defer dc.lock_mutex.Unlock()
if dc.lock_file != nil {
err = utils.UnlockFile(dc.lock_file)
if ce := dc.lock_file.Close(); ce != nil && err == nil {
err = ce
}
dc.lock_file = nil
}
return
}
const ENTRIES_NAME = "entries.json"
func (dc *DiskCache) entries_path() string { return filepath.Join(dc.Path, ENTRIES_NAME) }
func (dc *DiskCache) write_entries_if_dirty() (err error) {
if !dc.entries_dirty {
return
}
path := dc.entries_path()
defer func() {
if err == nil {
dc.entries_dirty = false
if s, serr := get_file_state_from_path(path); serr == nil {
dc.entries_last_read_state = s
}
}
}()
if d, err := json.Marshal(dc.entries); err != nil {
return err
} else {
// use a rename so that the inode number changes
// dont bother with full utils.AtomicWriteFile() as it is slower
temp := path + ".temp"
removed := false
defer func() {
if !removed {
_ = os.Remove(temp)
removed = true
}
}()
if err = os.WriteFile(temp, d, 0o600); err == nil {
if err = os.Rename(temp, path); err == nil {
removed = true
}
}
return err
}
}
func (e Entry) String() string {
return fmt.Sprintf("Entry{Key: %s, Size: %d, LastUsed: %s}", e.Key, e.Size, e.LastUsed)
}
func (dc *DiskCache) entries_from_folders() (total_size int64, ans map[string]*Entry, sorted []*Entry, err error) {
if entries, err := os.ReadDir(dc.Path); err != nil {
return 0, nil, nil, err
} else {
ans = make(map[string]*Entry)
var total int64
for _, x := range entries {
if x.IsDir() {
if sub_entries, err := os.ReadDir(filepath.Join(dc.Path, x.Name())); err == nil && len(sub_entries) == 1 {
key := sub_entries[0].Name()
path := dc.folder_for_key(key)
if file_entries, err := os.ReadDir(path); err == nil {
e := Entry{Key: key}
for _, f := range file_entries {
if fi, err := f.Info(); err == nil {
e.Size += fi.Size()
if fi.ModTime().After(e.LastUsed) {
e.LastUsed = fi.ModTime()
}
}
}
ans[key] = &e
total += e.Size
}
}
}
}
sorted = utils.Values(ans)
slices.SortFunc(sorted, func(a, b *Entry) int {
return a.LastUsed.Compare(b.LastUsed)
})
return total, ans, sorted, nil
}
}
func (dc *DiskCache) rebuild_entries() error {
total, ans, sorted, err := dc.entries_from_folders()
if err != nil {
return err
}
dc.entries = Metadata{TotalSize: total, SortedEntries: sorted, PathMap: make(map[string]string)}
dc.entry_map = ans
dc.entries_dirty = true
return dc.write_entries_if_dirty()
}
func (dc *DiskCache) ensure_entries() error {
needed := dc.entry_map == nil || dc.entries_last_read_state == nil
path := dc.entries_path()
var fstate *file_state
if !needed {
if s, err := get_file_state_from_path(path); err == nil {
fstate = s
if !s.equal(dc.entries_last_read_state) {
needed = true
}
}
}
if needed {
if data, err := os.ReadFile(path); err != nil {
if os.IsNotExist(err) {
dc.entry_map = make(map[string]*Entry)
dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
// Write the file so that its state is recorded. Otherwise every
// later call reloads and discards in-memory changes not yet written.
dc.entries_dirty = true
return dc.write_entries_if_dirty()
} else {
return err
}
} else {
dc.read_count += 1
dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
if err := json.Unmarshal(data, &dc.entries); err != nil {
// corrupted data
dc.rebuild_entries()
} else {
if fstate == nil {
if s, err := get_file_state_from_path(path); err == nil {
fstate = s
}
}
dc.entries_last_read_state = fstate
}
dc.entry_map = make(map[string]*Entry)
for _, e := range dc.entries.SortedEntries {
dc.entry_map[e.Key] = e
}
}
}
return nil
}
func (dc *DiskCache) folder_for_key(key string) (ans string) {
if len(key) < 5 {
ans = filepath.Join(key, key)
} else {
ans = filepath.Join(key[:4], key)
}
return filepath.Join(dc.Path, ans)
}
func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) {
dest := filepath.Join(dc.get_dir, key+"-"+filepath.Base(path))
if err := os.Link(path, dest); err == nil {
return dest, nil
} else if !os.IsExist(err) {
return "", err
}
src_info, err := os.Stat(path)
if err != nil {
return "", err
}
if dest_info, err := os.Stat(dest); err == nil && os.SameFile(src_info, dest_info) {
return dest, nil
}
// Replace dest atomically so that a path previously returned to a
// caller is never missing, even momentarily.
temp := dest + ".temp"
_ = os.Remove(temp)
if err := os.Link(path, temp); err != nil {
return "", err
}
if err := os.Rename(temp, dest); err != nil {
_ = os.Remove(temp)
return "", err
}
return dest, nil
}
func (dc *DiskCache) get(key string, items []string) (map[string]string, error) {
if err := dc.ensure_entries(); err != nil {
return nil, err
}
base := dc.folder_for_key(key)
if len(items) == 0 {
if entries, err := os.ReadDir(base); err != nil {
if os.IsNotExist(err) {
err = nil
}
return nil, err
} else {
for _, e := range entries {
items = append(items, e.Name())
}
}
} else {
if s, err := os.Stat(base); err != nil || !s.IsDir() {
if os.IsNotExist(err) {
err = nil
}
return nil, err
}
}
ans := make(map[string]string, len(items))
for _, x := range items {
p := filepath.Join(base, x)
if s, err := os.Stat(p); err != nil || s.IsDir() {
continue
}
dest, _ := dc.export_to_get_dir(key, p)
if dest != "" {
ans[x] = dest
}
}
if len(items) > 0 {
dc.update_timestamp(key)
}
return ans, dc.write_entries_if_dirty()
}
func (dc *DiskCache) clear() (err error) {
if entries, err := os.ReadDir(dc.Path); err != nil {
return err
} else {
defer func() {
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
for _, x := range entries {
if x.IsDir() && !strings.HasPrefix(x.Name(), GET_DIR_PREFIX) {
_ = os.RemoveAll(filepath.Join(dc.Path, x.Name()))
}
}
dc.entries_dirty = true
dc.entry_map = make(map[string]*Entry)
dc.entries.SortedEntries = make([]*Entry, 0)
dc.entries.PathMap = make(map[string]string)
dc.entries.TotalSize = 0
}
return
}
func (dc *DiskCache) remove(key string) (err error) {
if err = dc.ensure_entries(); err != nil {
return
}
defer func() {
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
base := dc.folder_for_key(key)
if err = os.RemoveAll(base); err == nil {
t := dc.entry_map[key]
if t != nil {
delete(dc.entry_map, key)
dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
dc.entries.SortedEntries = utils.Filter(dc.entries.SortedEntries, func(x *Entry) bool { return x.Key != key })
dc.entries_dirty = true
}
}
return
}
func (dc *DiskCache) prune() (bool, error) {
if dc.MaxSize < 1 || dc.entries.TotalSize <= dc.MaxSize {
return false, nil
}
for dc.entries.TotalSize > dc.MaxSize && len(dc.entries.SortedEntries) > 0 {
base := dc.folder_for_key(dc.entries.SortedEntries[0].Key)
if err := os.RemoveAll(base); err == nil {
t := dc.entries.SortedEntries[0]
delete(dc.entry_map, t.Key)
maps.DeleteFunc(dc.entries.PathMap, func(path, key string) bool { return key == t.Key })
dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
dc.entries.SortedEntries = dc.entries.SortedEntries[1:]
dc.entries_dirty = true
} else {
return false, err
}
}
return true, nil
}
func (dc *DiskCache) update_timestamp(key string) {
t := dc.entry_map[key]
if t == nil {
// the folder for key exists but is not in the metadata, can happen
// if entries.json was written by an older, racy version
return
}
t.LastUsed = time.Now()
if idx := slices.Index(dc.entries.SortedEntries, t); idx < 0 {
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
} else {
copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:])
dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t
}
dc.entries_dirty = true
}
func (dc *DiskCache) update_accounting(key string, changed int64) (err error) {
t := dc.entry_map[key]
if t == nil {
t = &Entry{Key: key}
dc.entry_map[key] = t
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
}
old_size := t.Size
t.Size += changed
t.Size = max(0, t.Size)
dc.entries.TotalSize += t.Size - old_size
dc.entries_dirty = true
dc.update_timestamp(key)
dc.prune()
return dc.write_entries_if_dirty()
}
func (dc *DiskCache) keys() (ans []string, err error) {
if err = dc.ensure_entries(); err != nil {
return
}
ans = make([]string, len(dc.entries.SortedEntries))
for i, e := range dc.entries.SortedEntries {
ans[i] = e.Key
}
return
}
func (dc *DiskCache) add_path(path, key string, items map[string][]byte) (ans map[string]string, err error) {
if err = dc.ensure_entries(); err != nil {
return
}
defer func() {
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
err = we
}
}()
if orphan := dc.set_key_for_path(path, key); orphan != "" {
if err = dc.remove(orphan); err != nil {
return
}
}
return dc.add(key, items)
}
// path_map_key returns the PathMap key for path. It includes extra_key so
// that, for example, previews of one file by different commands do not evict
// each other. An empty extra_key gives the same PathMap key as older versions.
func path_map_key(path, extra_key string) string {
if extra_key == "" {
return path
}
return path + "\x00" + extra_key
}
// set_key_for_path records that path now uses the entry for key. It returns
// the entry path used previously if no other path, such as another symlink to
// the same file, still uses it, so that the caller can remove it.
func (dc *DiskCache) set_key_for_path(path, key string) (orphan string) {
existing := dc.entries.PathMap[path]
if existing == key {
return ""
}
dc.entries.PathMap[path] = key
dc.entries_dirty = true
if existing == "" {
return ""
}
for _, k := range dc.entries.PathMap {
if k == existing {
return ""
}
}
return existing
}
func (dc *DiskCache) add(key string, items map[string][]byte) (ans map[string]string, err error) {
if err = dc.ensure_entries(); err != nil {
return
}
base := dc.folder_for_key(key)
if err = os.MkdirAll(base, 0o700); err != nil {
return
}
var changed int64
defer func() {
e := dc.update_accounting(key, changed)
if err == nil {
err = e
}
}()
ans = make(map[string]string, len(items))
for x, data := range items {
p := filepath.Join(base, x)
var before int64
exists := false
if s, err := os.Stat(p); err == nil {
before = s.Size()
exists = true
}
if len(data) == 0 {
if exists {
if err = os.Remove(p); err != nil {
if !os.IsNotExist(err) {
return
}
err = nil
}
changed -= before
}
} else {
// unlink the file so that writing to it does not change a
// previously linked copy created by get()
if exists {
_ = os.Remove(p)
}
if err = os.WriteFile(p, data, 0o700); err != nil {
return
}
changed += int64(len(data)) - before
if dest, err := dc.export_to_get_dir(key, p); err != nil {
return ans, err
} else {
ans[x] = dest
}
}
}
return
}