mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-16 13:08:25 +00:00
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run
Let preview renderers supply extra cache key data instead of checking for the cmd_renderer type. Include the command line in the key for custom previews and the video_preview settings in the key for video thumbnails, so that changing them no longer shows stale previews. GetPath and AddPath now take this extra key data, replacing GetPathWithAlias. Track the entry each path uses per extra key, so that previews of one file with different settings do not evict each other. Do not remove an entry while another path, such as another symlink to the same file, still uses it. Cache hits are now recorded so that such shared entries are known. Failing to remove an orphaned entry on a cache hit no longer turns the hit into an error. Fix ensure_entries() discarding unwritten changes when the entries file does not exist yet, which lost the first path recorded in a new cache. Also reset the path map when clearing the cache.
572 lines
14 KiB
Go
572 lines
14 KiB
Go
package disk_cache
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/kovidgoyal/kitty/tools/utils"
|
|
)
|
|
|
|
var _ = fmt.Print
|
|
|
|
type file_state struct {
|
|
Size int64
|
|
ModTime time.Time
|
|
Inode uint64
|
|
}
|
|
|
|
func (s file_state) String() string {
|
|
return fmt.Sprintf("fs{Size: %d, Inode: %d, ModTime: %s}", s.Size, s.Inode, s.ModTime)
|
|
}
|
|
|
|
func (s *file_state) equal(o *file_state) bool {
|
|
return o != nil && s.Size == o.Size && s.ModTime.Equal(o.ModTime) && s.Inode == o.Inode
|
|
}
|
|
|
|
func get_file_state(fi fs.FileInfo) *file_state {
|
|
// The Sys() method returns the underlying data source (can be nil).
|
|
// For Unix-like systems, it's a *syscall.Stat_t.
|
|
stat, ok := fi.Sys().(*syscall.Stat_t)
|
|
if !ok {
|
|
// For non-Unix systems, you might not have an inode.
|
|
// In that case, you can fall back to using only size and mod time.
|
|
return &file_state{
|
|
Size: fi.Size(),
|
|
ModTime: fi.ModTime(),
|
|
Inode: 0, // Inode not available
|
|
}
|
|
}
|
|
return &file_state{
|
|
Size: fi.Size(),
|
|
ModTime: fi.ModTime(),
|
|
Inode: stat.Ino,
|
|
}
|
|
}
|
|
|
|
func get_file_state_from_path(path string) (*file_state, error) {
|
|
if s, err := os.Stat(path); err != nil {
|
|
return nil, err
|
|
} else {
|
|
return get_file_state(s), nil
|
|
}
|
|
}
|
|
|
|
const GET_DIR_PREFIX = "getdir-"
|
|
|
|
func new_disk_cache(path string, max_size int64) (dc *DiskCache, err error) {
|
|
if path, err = filepath.Abs(path); err != nil {
|
|
return
|
|
}
|
|
if err = os.MkdirAll(path, 0o700); err != nil {
|
|
return
|
|
}
|
|
ans := &DiskCache{Path: path, MaxSize: max_size}
|
|
if err = ans.lock(); err != nil {
|
|
return
|
|
}
|
|
defer ans.unlock_and_set_err(&err)
|
|
if err = ans.ensure_entries(); err != nil {
|
|
return
|
|
}
|
|
defer func() {
|
|
if we := ans.write_entries_if_dirty(); we != nil && err == nil {
|
|
err = we
|
|
}
|
|
}()
|
|
if _, err := ans.prune(); err != nil {
|
|
return nil, err
|
|
}
|
|
if ans.get_dir, err = os.MkdirTemp(ans.Path, GET_DIR_PREFIX+"*"); err != nil {
|
|
return
|
|
}
|
|
if err = utils.AtExitRmtree(ans.get_dir); err != nil {
|
|
return
|
|
}
|
|
return ans, nil
|
|
}
|
|
|
|
func key_for_path(path, extra_key string) (key string, err error) {
|
|
resolved_path, err := filepath.EvalSymlinks(path)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if resolved_path, err = filepath.Abs(resolved_path); err != nil {
|
|
return
|
|
}
|
|
|
|
s, err := os.Stat(resolved_path)
|
|
if err != nil {
|
|
return
|
|
}
|
|
data := fmt.Sprintf("%s\x00%d\x00%d", resolved_path, s.Size(), s.ModTime().UnixNano())
|
|
if extra_key != "" {
|
|
data += "\x00" + extra_key
|
|
}
|
|
sum := sha256.Sum256(utils.UnsafeStringToBytes(data))
|
|
return hex.EncodeToString(sum[:]), nil
|
|
}
|
|
|
|
// lock() and unlock() must be paired: unlock() must only be called after a
|
|
// successful lock(). lock() is not re-entrant, so it must not be called while
|
|
// the lock is already held, that will deadlock.
|
|
func (dc *DiskCache) lock() (err error) {
|
|
dc.lock_mutex.Lock()
|
|
defer func() {
|
|
if err != nil {
|
|
_ = dc.unlock()
|
|
}
|
|
}()
|
|
if dc.lock_file, err = os.OpenFile(filepath.Join(dc.Path, "lockfile"), os.O_RDWR|os.O_CREATE, 0o600); err != nil {
|
|
return
|
|
}
|
|
return utils.LockFileExclusive(dc.lock_file)
|
|
}
|
|
|
|
func (dc *DiskCache) unlock() (err error) {
|
|
defer dc.lock_mutex.Unlock()
|
|
if dc.lock_file != nil {
|
|
err = utils.UnlockFile(dc.lock_file)
|
|
if ce := dc.lock_file.Close(); ce != nil && err == nil {
|
|
err = ce
|
|
}
|
|
dc.lock_file = nil
|
|
}
|
|
return
|
|
}
|
|
|
|
const ENTRIES_NAME = "entries.json"
|
|
|
|
func (dc *DiskCache) entries_path() string { return filepath.Join(dc.Path, ENTRIES_NAME) }
|
|
|
|
func (dc *DiskCache) write_entries_if_dirty() (err error) {
|
|
if !dc.entries_dirty {
|
|
return
|
|
}
|
|
path := dc.entries_path()
|
|
defer func() {
|
|
if err == nil {
|
|
dc.entries_dirty = false
|
|
if s, serr := get_file_state_from_path(path); serr == nil {
|
|
dc.entries_last_read_state = s
|
|
}
|
|
}
|
|
}()
|
|
if d, err := json.Marshal(dc.entries); err != nil {
|
|
return err
|
|
} else {
|
|
// use a rename so that the inode number changes
|
|
// dont bother with full utils.AtomicWriteFile() as it is slower
|
|
temp := path + ".temp"
|
|
removed := false
|
|
defer func() {
|
|
if !removed {
|
|
_ = os.Remove(temp)
|
|
removed = true
|
|
}
|
|
}()
|
|
if err = os.WriteFile(temp, d, 0o600); err == nil {
|
|
if err = os.Rename(temp, path); err == nil {
|
|
removed = true
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
func (e Entry) String() string {
|
|
return fmt.Sprintf("Entry{Key: %s, Size: %d, LastUsed: %s}", e.Key, e.Size, e.LastUsed)
|
|
}
|
|
|
|
func (dc *DiskCache) entries_from_folders() (total_size int64, ans map[string]*Entry, sorted []*Entry, err error) {
|
|
if entries, err := os.ReadDir(dc.Path); err != nil {
|
|
return 0, nil, nil, err
|
|
} else {
|
|
ans = make(map[string]*Entry)
|
|
var total int64
|
|
for _, x := range entries {
|
|
if x.IsDir() {
|
|
if sub_entries, err := os.ReadDir(filepath.Join(dc.Path, x.Name())); err == nil && len(sub_entries) == 1 {
|
|
key := sub_entries[0].Name()
|
|
path := dc.folder_for_key(key)
|
|
if file_entries, err := os.ReadDir(path); err == nil {
|
|
e := Entry{Key: key}
|
|
for _, f := range file_entries {
|
|
if fi, err := f.Info(); err == nil {
|
|
e.Size += fi.Size()
|
|
if fi.ModTime().After(e.LastUsed) {
|
|
e.LastUsed = fi.ModTime()
|
|
}
|
|
}
|
|
}
|
|
ans[key] = &e
|
|
total += e.Size
|
|
}
|
|
}
|
|
}
|
|
}
|
|
sorted = utils.Values(ans)
|
|
slices.SortFunc(sorted, func(a, b *Entry) int {
|
|
return a.LastUsed.Compare(b.LastUsed)
|
|
})
|
|
return total, ans, sorted, nil
|
|
}
|
|
}
|
|
|
|
func (dc *DiskCache) rebuild_entries() error {
|
|
total, ans, sorted, err := dc.entries_from_folders()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dc.entries = Metadata{TotalSize: total, SortedEntries: sorted, PathMap: make(map[string]string)}
|
|
dc.entry_map = ans
|
|
dc.entries_dirty = true
|
|
return dc.write_entries_if_dirty()
|
|
}
|
|
|
|
func (dc *DiskCache) ensure_entries() error {
|
|
needed := dc.entry_map == nil || dc.entries_last_read_state == nil
|
|
path := dc.entries_path()
|
|
var fstate *file_state
|
|
if !needed {
|
|
if s, err := get_file_state_from_path(path); err == nil {
|
|
fstate = s
|
|
if !s.equal(dc.entries_last_read_state) {
|
|
needed = true
|
|
}
|
|
}
|
|
}
|
|
if needed {
|
|
if data, err := os.ReadFile(path); err != nil {
|
|
if os.IsNotExist(err) {
|
|
dc.entry_map = make(map[string]*Entry)
|
|
dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
|
|
// Write the file so that its state is recorded. Otherwise every
|
|
// later call reloads and discards in-memory changes not yet written.
|
|
dc.entries_dirty = true
|
|
return dc.write_entries_if_dirty()
|
|
} else {
|
|
return err
|
|
}
|
|
} else {
|
|
dc.read_count += 1
|
|
dc.entries = Metadata{SortedEntries: make([]*Entry, 0), PathMap: make(map[string]string)}
|
|
if err := json.Unmarshal(data, &dc.entries); err != nil {
|
|
// corrupted data
|
|
dc.rebuild_entries()
|
|
} else {
|
|
if fstate == nil {
|
|
if s, err := get_file_state_from_path(path); err == nil {
|
|
fstate = s
|
|
}
|
|
}
|
|
dc.entries_last_read_state = fstate
|
|
}
|
|
dc.entry_map = make(map[string]*Entry)
|
|
for _, e := range dc.entries.SortedEntries {
|
|
dc.entry_map[e.Key] = e
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (dc *DiskCache) folder_for_key(key string) (ans string) {
|
|
if len(key) < 5 {
|
|
ans = filepath.Join(key, key)
|
|
} else {
|
|
ans = filepath.Join(key[:4], key)
|
|
}
|
|
return filepath.Join(dc.Path, ans)
|
|
}
|
|
|
|
func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) {
|
|
dest := filepath.Join(dc.get_dir, key+"-"+filepath.Base(path))
|
|
if err := os.Link(path, dest); err == nil {
|
|
return dest, nil
|
|
} else if !os.IsExist(err) {
|
|
return "", err
|
|
}
|
|
src_info, err := os.Stat(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if dest_info, err := os.Stat(dest); err == nil && os.SameFile(src_info, dest_info) {
|
|
return dest, nil
|
|
}
|
|
// Replace dest atomically so that a path previously returned to a
|
|
// caller is never missing, even momentarily.
|
|
temp := dest + ".temp"
|
|
_ = os.Remove(temp)
|
|
if err := os.Link(path, temp); err != nil {
|
|
return "", err
|
|
}
|
|
if err := os.Rename(temp, dest); err != nil {
|
|
_ = os.Remove(temp)
|
|
return "", err
|
|
}
|
|
return dest, nil
|
|
}
|
|
|
|
func (dc *DiskCache) get(key string, items []string) (map[string]string, error) {
|
|
if err := dc.ensure_entries(); err != nil {
|
|
return nil, err
|
|
}
|
|
base := dc.folder_for_key(key)
|
|
if len(items) == 0 {
|
|
if entries, err := os.ReadDir(base); err != nil {
|
|
if os.IsNotExist(err) {
|
|
err = nil
|
|
}
|
|
return nil, err
|
|
} else {
|
|
for _, e := range entries {
|
|
items = append(items, e.Name())
|
|
}
|
|
}
|
|
} else {
|
|
if s, err := os.Stat(base); err != nil || !s.IsDir() {
|
|
if os.IsNotExist(err) {
|
|
err = nil
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
ans := make(map[string]string, len(items))
|
|
for _, x := range items {
|
|
p := filepath.Join(base, x)
|
|
if s, err := os.Stat(p); err != nil || s.IsDir() {
|
|
continue
|
|
}
|
|
dest, _ := dc.export_to_get_dir(key, p)
|
|
if dest != "" {
|
|
ans[x] = dest
|
|
}
|
|
}
|
|
if len(items) > 0 {
|
|
dc.update_timestamp(key)
|
|
}
|
|
return ans, dc.write_entries_if_dirty()
|
|
}
|
|
|
|
func (dc *DiskCache) clear() (err error) {
|
|
if entries, err := os.ReadDir(dc.Path); err != nil {
|
|
return err
|
|
} else {
|
|
defer func() {
|
|
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
|
|
err = we
|
|
}
|
|
}()
|
|
for _, x := range entries {
|
|
if x.IsDir() && !strings.HasPrefix(x.Name(), GET_DIR_PREFIX) {
|
|
_ = os.RemoveAll(filepath.Join(dc.Path, x.Name()))
|
|
}
|
|
}
|
|
dc.entries_dirty = true
|
|
dc.entry_map = make(map[string]*Entry)
|
|
dc.entries.SortedEntries = make([]*Entry, 0)
|
|
dc.entries.PathMap = make(map[string]string)
|
|
dc.entries.TotalSize = 0
|
|
}
|
|
return
|
|
}
|
|
|
|
func (dc *DiskCache) remove(key string) (err error) {
|
|
if err = dc.ensure_entries(); err != nil {
|
|
return
|
|
}
|
|
defer func() {
|
|
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
|
|
err = we
|
|
}
|
|
}()
|
|
base := dc.folder_for_key(key)
|
|
if err = os.RemoveAll(base); err == nil {
|
|
t := dc.entry_map[key]
|
|
if t != nil {
|
|
delete(dc.entry_map, key)
|
|
dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
|
|
dc.entries.SortedEntries = utils.Filter(dc.entries.SortedEntries, func(x *Entry) bool { return x.Key != key })
|
|
dc.entries_dirty = true
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (dc *DiskCache) prune() (bool, error) {
|
|
if dc.MaxSize < 1 || dc.entries.TotalSize <= dc.MaxSize {
|
|
return false, nil
|
|
}
|
|
for dc.entries.TotalSize > dc.MaxSize && len(dc.entries.SortedEntries) > 0 {
|
|
base := dc.folder_for_key(dc.entries.SortedEntries[0].Key)
|
|
if err := os.RemoveAll(base); err == nil {
|
|
t := dc.entries.SortedEntries[0]
|
|
delete(dc.entry_map, t.Key)
|
|
maps.DeleteFunc(dc.entries.PathMap, func(path, key string) bool { return key == t.Key })
|
|
dc.entries.TotalSize = max(0, dc.entries.TotalSize-t.Size)
|
|
dc.entries.SortedEntries = dc.entries.SortedEntries[1:]
|
|
dc.entries_dirty = true
|
|
} else {
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (dc *DiskCache) update_timestamp(key string) {
|
|
t := dc.entry_map[key]
|
|
if t == nil {
|
|
// the folder for key exists but is not in the metadata, can happen
|
|
// if entries.json was written by an older, racy version
|
|
return
|
|
}
|
|
t.LastUsed = time.Now()
|
|
if idx := slices.Index(dc.entries.SortedEntries, t); idx < 0 {
|
|
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
|
|
} else {
|
|
copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:])
|
|
dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t
|
|
}
|
|
dc.entries_dirty = true
|
|
}
|
|
|
|
func (dc *DiskCache) update_accounting(key string, changed int64) (err error) {
|
|
t := dc.entry_map[key]
|
|
if t == nil {
|
|
t = &Entry{Key: key}
|
|
dc.entry_map[key] = t
|
|
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
|
|
}
|
|
old_size := t.Size
|
|
t.Size += changed
|
|
t.Size = max(0, t.Size)
|
|
dc.entries.TotalSize += t.Size - old_size
|
|
dc.entries_dirty = true
|
|
dc.update_timestamp(key)
|
|
dc.prune()
|
|
return dc.write_entries_if_dirty()
|
|
}
|
|
|
|
func (dc *DiskCache) keys() (ans []string, err error) {
|
|
if err = dc.ensure_entries(); err != nil {
|
|
return
|
|
}
|
|
ans = make([]string, len(dc.entries.SortedEntries))
|
|
for i, e := range dc.entries.SortedEntries {
|
|
ans[i] = e.Key
|
|
}
|
|
return
|
|
}
|
|
|
|
func (dc *DiskCache) add_path(path, key string, items map[string][]byte) (ans map[string]string, err error) {
|
|
if err = dc.ensure_entries(); err != nil {
|
|
return
|
|
}
|
|
defer func() {
|
|
if we := dc.write_entries_if_dirty(); we != nil && err == nil {
|
|
err = we
|
|
}
|
|
}()
|
|
|
|
if orphan := dc.set_key_for_path(path, key); orphan != "" {
|
|
if err = dc.remove(orphan); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return dc.add(key, items)
|
|
}
|
|
|
|
// path_map_key returns the PathMap key for path. It includes extra_key so
|
|
// that, for example, previews of one file by different commands do not evict
|
|
// each other. An empty extra_key gives the same PathMap key as older versions.
|
|
func path_map_key(path, extra_key string) string {
|
|
if extra_key == "" {
|
|
return path
|
|
}
|
|
return path + "\x00" + extra_key
|
|
}
|
|
|
|
// set_key_for_path records that path now uses the entry for key. It returns
|
|
// the entry path used previously if no other path, such as another symlink to
|
|
// the same file, still uses it, so that the caller can remove it.
|
|
func (dc *DiskCache) set_key_for_path(path, key string) (orphan string) {
|
|
existing := dc.entries.PathMap[path]
|
|
if existing == key {
|
|
return ""
|
|
}
|
|
dc.entries.PathMap[path] = key
|
|
dc.entries_dirty = true
|
|
if existing == "" {
|
|
return ""
|
|
}
|
|
for _, k := range dc.entries.PathMap {
|
|
if k == existing {
|
|
return ""
|
|
}
|
|
}
|
|
return existing
|
|
}
|
|
|
|
func (dc *DiskCache) add(key string, items map[string][]byte) (ans map[string]string, err error) {
|
|
if err = dc.ensure_entries(); err != nil {
|
|
return
|
|
}
|
|
base := dc.folder_for_key(key)
|
|
if err = os.MkdirAll(base, 0o700); err != nil {
|
|
return
|
|
}
|
|
var changed int64
|
|
defer func() {
|
|
e := dc.update_accounting(key, changed)
|
|
if err == nil {
|
|
err = e
|
|
}
|
|
}()
|
|
ans = make(map[string]string, len(items))
|
|
for x, data := range items {
|
|
p := filepath.Join(base, x)
|
|
var before int64
|
|
exists := false
|
|
if s, err := os.Stat(p); err == nil {
|
|
before = s.Size()
|
|
exists = true
|
|
}
|
|
if len(data) == 0 {
|
|
if exists {
|
|
if err = os.Remove(p); err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return
|
|
}
|
|
err = nil
|
|
}
|
|
changed -= before
|
|
}
|
|
} else {
|
|
// unlink the file so that writing to it does not change a
|
|
// previously linked copy created by get()
|
|
if exists {
|
|
_ = os.Remove(p)
|
|
}
|
|
if err = os.WriteFile(p, data, 0o700); err != nil {
|
|
return
|
|
}
|
|
changed += int64(len(data)) - before
|
|
if dest, err := dc.export_to_get_dir(key, p); err != nil {
|
|
return ans, err
|
|
} else {
|
|
ans[x] = dest
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|