mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-19 22:55:29 +00:00
Cleanup previous PR
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run
Replace exported get_dir files atomically so paths returned by Get/Add never briefly vanish while another worker re-exports the same key. Don't panic in Get for a key folder missing from entries.json. Propagate errors from unlocking, document the lock()/unlock() contract, and fix the NewDiskCache lock error test which checked the wrong cache
This commit is contained in:
parent
faa50dfdcb
commit
7a751292bb
3 changed files with 147 additions and 43 deletions
|
|
@ -43,32 +43,37 @@ func (dc *DiskCache) ResultsDir() string {
|
|||
return dc.get_dir
|
||||
}
|
||||
|
||||
func (dc *DiskCache) Get(key string, items ...string) (map[string]string, error) {
|
||||
if err := dc.lock(); err != nil {
|
||||
return nil, err
|
||||
func (dc *DiskCache) unlock_and_set_err(err *error) {
|
||||
if ue := dc.unlock(); ue != nil && *err == nil {
|
||||
*err = ue
|
||||
}
|
||||
defer dc.unlock()
|
||||
}
|
||||
|
||||
func (dc *DiskCache) Get(key string, items ...string) (ans map[string]string, err error) {
|
||||
if err = dc.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer dc.unlock_and_set_err(&err)
|
||||
return dc.get(key, items)
|
||||
}
|
||||
|
||||
func (dc *DiskCache) GetPath(path string, items ...string) (string, map[string]string, error) {
|
||||
if err := dc.lock(); err != nil {
|
||||
func (dc *DiskCache) GetPath(path string, items ...string) (key string, ans map[string]string, err error) {
|
||||
if err = dc.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer dc.unlock_and_set_err(&err)
|
||||
if key, err = key_for_path(path); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
defer dc.unlock()
|
||||
key, err := key_for_path(path)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
ans, err := dc.get(key, items)
|
||||
return key, ans, err
|
||||
ans, err = dc.get(key, items)
|
||||
return
|
||||
}
|
||||
|
||||
func (dc *DiskCache) Remove(key string) (err error) {
|
||||
if err = dc.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer dc.unlock()
|
||||
defer dc.unlock_and_set_err(&err)
|
||||
return dc.remove(key)
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +81,7 @@ func (dc *DiskCache) Clear() (err error) {
|
|||
if err = dc.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer dc.unlock()
|
||||
defer dc.unlock_and_set_err(&err)
|
||||
return dc.clear()
|
||||
}
|
||||
|
||||
|
|
@ -84,15 +89,14 @@ func (dc *DiskCache) AddPath(path, key string, items map[string][]byte) (ans map
|
|||
if err = dc.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer dc.unlock()
|
||||
ans, err = dc.add_path(path, key, items)
|
||||
return
|
||||
defer dc.unlock_and_set_err(&err)
|
||||
return dc.add_path(path, key, items)
|
||||
}
|
||||
|
||||
func (dc *DiskCache) Add(key string, items map[string][]byte) (ans map[string]string, err error) {
|
||||
if err = dc.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer dc.unlock()
|
||||
defer dc.unlock_and_set_err(&err)
|
||||
return dc.add(key, items)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ func new_disk_cache(path string, max_size int64) (dc *DiskCache, err error) {
|
|||
if err = ans.lock(); err != nil {
|
||||
return
|
||||
}
|
||||
defer ans.unlock()
|
||||
defer ans.unlock_and_set_err(&err)
|
||||
if err = ans.ensure_entries(); err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -112,11 +112,14 @@ func key_for_path(path string) (key string, err error) {
|
|||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// lock() and unlock() must be paired: unlock() must only be called after a
|
||||
// successful lock(). lock() is not re-entrant, so it must not be called while
|
||||
// the lock is already held, that will deadlock.
|
||||
func (dc *DiskCache) lock() (err error) {
|
||||
dc.lock_mutex.Lock()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
dc.unlock()
|
||||
_ = dc.unlock()
|
||||
}
|
||||
}()
|
||||
if dc.lock_file, err = os.OpenFile(filepath.Join(dc.Path, "lockfile"), os.O_RDWR|os.O_CREATE, 0o600); err != nil {
|
||||
|
|
@ -125,13 +128,16 @@ func (dc *DiskCache) lock() (err error) {
|
|||
return utils.LockFileExclusive(dc.lock_file)
|
||||
}
|
||||
|
||||
func (dc *DiskCache) unlock() {
|
||||
func (dc *DiskCache) unlock() (err error) {
|
||||
defer dc.lock_mutex.Unlock()
|
||||
if dc.lock_file != nil {
|
||||
utils.UnlockFile(dc.lock_file)
|
||||
dc.lock_file.Close()
|
||||
err = utils.UnlockFile(dc.lock_file)
|
||||
if ce := dc.lock_file.Close(); ce != nil && err == nil {
|
||||
err = ce
|
||||
}
|
||||
dc.lock_file = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const ENTRIES_NAME = "entries.json"
|
||||
|
|
@ -277,14 +283,30 @@ func (dc *DiskCache) folder_for_key(key string) (ans string) {
|
|||
|
||||
func (dc *DiskCache) export_to_get_dir(key, path string) (string, error) {
|
||||
dest := filepath.Join(dc.get_dir, key+"-"+filepath.Base(path))
|
||||
if err := os.Link(path, dest); err != nil {
|
||||
os.Remove(dest)
|
||||
if err := os.Link(path, dest); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Link(path, dest); err == nil {
|
||||
return dest, nil
|
||||
} else if !os.IsExist(err) {
|
||||
return "", err
|
||||
}
|
||||
src_info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dest_info, err := os.Stat(dest); err == nil && os.SameFile(src_info, dest_info) {
|
||||
return dest, nil
|
||||
}
|
||||
// Replace dest atomically so that a path previously returned to a
|
||||
// caller is never missing, even momentarily.
|
||||
temp := dest + ".temp"
|
||||
_ = os.Remove(temp)
|
||||
if err := os.Link(path, temp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(temp, dest); err != nil {
|
||||
_ = os.Remove(temp)
|
||||
return "", err
|
||||
}
|
||||
return dest, nil
|
||||
|
||||
}
|
||||
|
||||
func (dc *DiskCache) get(key string, items []string) (map[string]string, error) {
|
||||
|
|
@ -394,10 +416,18 @@ func (dc *DiskCache) prune() (bool, error) {
|
|||
|
||||
func (dc *DiskCache) update_timestamp(key string) {
|
||||
t := dc.entry_map[key]
|
||||
if t == nil {
|
||||
// the folder for key exists but is not in the metadata, can happen
|
||||
// if entries.json was written by an older, racy version
|
||||
return
|
||||
}
|
||||
t.LastUsed = time.Now()
|
||||
idx := slices.Index(dc.entries.SortedEntries, t)
|
||||
copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:])
|
||||
dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t
|
||||
if idx := slices.Index(dc.entries.SortedEntries, t); idx < 0 {
|
||||
dc.entries.SortedEntries = append(dc.entries.SortedEntries, t)
|
||||
} else {
|
||||
copy(dc.entries.SortedEntries[idx:], dc.entries.SortedEntries[idx+1:])
|
||||
dc.entries.SortedEntries[len(dc.entries.SortedEntries)-1] = t
|
||||
}
|
||||
dc.entries_dirty = true
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -189,6 +189,10 @@ func TestDiskCacheConcurrentGet(t *testing.T) {
|
|||
t.Error("concurrent Get lost cached data")
|
||||
return
|
||||
}
|
||||
if data, err := os.ReadFile(paths["data"]); err != nil || string(data) != "cached" {
|
||||
t.Errorf("path returned by concurrent Get is not readable: %q, %v", data, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -196,6 +200,65 @@ func TestDiskCacheConcurrentGet(t *testing.T) {
|
|||
workers.Wait()
|
||||
}
|
||||
|
||||
func TestDiskCacheExportReplacesAtomically(t *testing.T) {
|
||||
dc, err := NewDiskCache(t.TempDir(), 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths, err := dc.Add("key", map[string][]byte{"data": []byte("old")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported := paths["data"]
|
||||
// a Get of unchanged data must not touch the previously exported path
|
||||
before, err := os.Stat(exported)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if paths, err = dc.Get("key", "data"); err != nil || paths["data"] != exported {
|
||||
t.Fatalf("Get returned unexpected paths: %v, %v", paths, err)
|
||||
}
|
||||
if after, err := os.Stat(exported); err != nil || !os.SameFile(before, after) {
|
||||
t.Fatalf("Get replaced an already exported file: %v", err)
|
||||
}
|
||||
// updated data must replace the exported path, leaving no temp files
|
||||
if paths, err = dc.Add("key", map[string][]byte{"data": []byte("new")}); err != nil || paths["data"] != exported {
|
||||
t.Fatalf("Add returned unexpected paths: %v, %v", paths, err)
|
||||
}
|
||||
if data, err := os.ReadFile(exported); err != nil || string(data) != "new" {
|
||||
t.Fatalf("exported file not updated: %q, %v", data, err)
|
||||
}
|
||||
entries, err := os.ReadDir(dc.ResultsDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("unexpected files in results dir: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheGetUntrackedKey(t *testing.T) {
|
||||
dc, err := NewDiskCache(t.TempDir(), 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// simulate a key folder that is missing from entries.json
|
||||
base := dc.folder_for_key("untracked")
|
||||
if err = os.MkdirAll(base, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(base, "data"), []byte("orphan"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths, err := dc.Get("untracked", "data")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data, err := os.ReadFile(paths["data"]); err != nil || string(data) != "orphan" {
|
||||
t.Fatalf("could not read untracked key: %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiskCacheLockErrors(t *testing.T) {
|
||||
source := filepath.Join(t.TempDir(), "source")
|
||||
if err := os.WriteFile(source, []byte("source"), 0o600); err != nil {
|
||||
|
|
@ -204,29 +267,31 @@ func TestDiskCacheLockErrors(t *testing.T) {
|
|||
for _, tc := range []struct {
|
||||
name string
|
||||
run func(*DiskCache) error
|
||||
// whether run() locks dc itself, rather than some other DiskCache
|
||||
uses_dc bool
|
||||
}{
|
||||
{"Get", func(dc *DiskCache) error {
|
||||
_, err := dc.Get("key", "data")
|
||||
return err
|
||||
}},
|
||||
}, true},
|
||||
{"GetPath", func(dc *DiskCache) error {
|
||||
_, _, err := dc.GetPath(source, "data")
|
||||
return err
|
||||
}},
|
||||
{"Remove", func(dc *DiskCache) error { return dc.Remove("key") }},
|
||||
{"Clear", func(dc *DiskCache) error { return dc.Clear() }},
|
||||
}, true},
|
||||
{"Remove", func(dc *DiskCache) error { return dc.Remove("key") }, true},
|
||||
{"Clear", func(dc *DiskCache) error { return dc.Clear() }, true},
|
||||
{"AddPath", func(dc *DiskCache) error {
|
||||
_, err := dc.AddPath(source, "key", map[string][]byte{"data": []byte("updated")})
|
||||
return err
|
||||
}},
|
||||
}, true},
|
||||
{"Add", func(dc *DiskCache) error {
|
||||
_, err := dc.Add("key", map[string][]byte{"data": []byte("updated")})
|
||||
return err
|
||||
}},
|
||||
}, true},
|
||||
{"NewDiskCache", func(dc *DiskCache) error {
|
||||
_, err := NewDiskCache(dc.Path, dc.MaxSize)
|
||||
return err
|
||||
}},
|
||||
}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dc, err := NewDiskCache(t.TempDir(), 64)
|
||||
|
|
@ -246,10 +311,15 @@ func TestDiskCacheLockErrors(t *testing.T) {
|
|||
if err = tc.run(dc); err == nil {
|
||||
t.Error("cache operation ignored failure to acquire the lock")
|
||||
}
|
||||
if !dc.lock_mutex.TryLock() {
|
||||
t.Fatal("failed lock attempt did not release the mutex")
|
||||
if tc.uses_dc {
|
||||
if !dc.lock_mutex.TryLock() {
|
||||
t.Fatal("failed lock attempt did not release the mutex")
|
||||
}
|
||||
dc.lock_mutex.Unlock()
|
||||
if dc.lock_file != nil {
|
||||
t.Fatal("failed lock attempt did not close the lock file")
|
||||
}
|
||||
}
|
||||
dc.lock_mutex.Unlock()
|
||||
if err = os.Remove(lock_path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue