Merge branch 'codex/fix-calibre-preview-responses' of https://github.com/hezhizhen/kitty

This commit is contained in:
Kovid Goyal 2026-11-10 19:28:04 +05:30
commit 245681507a
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
2 changed files with 304 additions and 78 deletions

View file

@ -32,6 +32,27 @@ type calibre_server_process struct {
stdout io.ReadCloser
stdin io.WriteCloser
file_extensions *utils.Set[string]
reader_done chan struct{}
lock sync.Mutex
request_id uint64
pending map[uint64]chan calibre_result
read_err error
}
type calibre_request struct {
ID uint64 `json:"id"`
Path string `json:"path"`
Cover string `json:"cover,omitempty"`
}
type calibre_reply struct {
ID uint64 `json:"id"`
Response json.RawMessage `json:"response"`
}
type calibre_result struct {
data json.RawMessage
err error
}
type CalibreMetadata struct {
@ -53,85 +74,163 @@ type CalibreResponse struct {
Metadata CalibreMetadata `json:"metadata"`
}
func ReadLineWithTimeout(r io.Reader, timeout time.Duration) (string, error) {
type result struct {
line string
err error
// The path can be reused, and calibre only echoes the cover path if it writes
// a cover. Attach a request ID to every response in the forked worker.
const calibre_server_program = `
def serve():
# calibre-debug runs -c with exec() inside a function. Keep imports local.
import io
import json
import sys
from contextlib import nullcontext
from calibre.ebooks.metadata import cli
read_metadata = cli.one
def reply_with_id(line, lock):
request_id = json.loads(line)['id']
stdout = sys.__stdout__
with io.StringIO() as output:
try:
sys.__stdout__ = output
read_metadata(line, nullcontext())
finally:
sys.__stdout__ = stdout
response = json.loads(output.getvalue())
with lock:
print(json.dumps({'id': request_id, 'response': response}), file=stdout, flush=True)
cli.one = reply_with_id
cli.simple_metadata_server()
serve()
`
func new_calibre_server_process(proc *exec.Cmd, stdin io.WriteCloser, stdout io.ReadCloser) *calibre_server_process {
s := &calibre_server_process{
proc: proc, stdin: stdin, stdout: stdout, reader_done: make(chan struct{}),
pending: make(map[uint64]chan calibre_result),
}
ch := make(chan result, 1)
go s.read_replies()
return s
}
go func() {
br := bufio.NewReader(r)
line, err := br.ReadString('\n')
ch <- result{strings.TrimRight(line, "\n"), err}
}()
select {
case res := <-ch:
return res.line, res.err
case <-time.After(timeout):
return "", os.ErrDeadlineExceeded
func (s *calibre_server_process) read_replies() {
defer close(s.reader_done)
reader := bufio.NewReader(s.stdout)
for {
line, err := reader.ReadBytes('\n')
if err != nil {
s.lock.Lock()
s.read_err = fmt.Errorf("error reading from calibre metadata server: %w", err)
for _, ch := range s.pending {
ch <- calibre_result{err: s.read_err}
}
clear(s.pending)
s.lock.Unlock()
return
}
var reply calibre_reply
if err := json.Unmarshal(line, &reply); err != nil {
continue
}
s.lock.Lock()
if ch := s.pending[reply.ID]; ch != nil {
delete(s.pending, reply.ID)
ch <- calibre_result{data: reply.Response}
}
s.lock.Unlock()
}
}
var calibre_server = sync.OnceValues(func() (ans *calibre_server_process, err error) {
func (s *calibre_server_process) request(path, cover string, timeout time.Duration) (json.RawMessage, error) {
s.lock.Lock()
if err := s.read_err; err != nil {
s.lock.Unlock()
return nil, err
}
s.request_id++
id := s.request_id
ch := make(chan calibre_result, 1)
s.pending[id] = ch
s.lock.Unlock()
defer func() {
s.lock.Lock()
delete(s.pending, id)
s.lock.Unlock()
}()
payload, err := json.Marshal(calibre_request{ID: id, Path: path, Cover: cover})
if err != nil {
return nil, err
}
if _, err := s.stdin.Write(append(payload, '\n')); err != nil {
return nil, fmt.Errorf("error writing to calibre metadata server: %w", err)
}
select {
case reply := <-ch:
return reply.data, reply.err
case <-time.After(timeout):
return nil, os.ErrDeadlineExceeded
}
}
func (s *calibre_server_process) close() {
s.stdin.Close()
s.stdout.Close()
// The request workers share the server's process group and can outlive it.
unix.Kill(-s.proc.Process.Pid, unix.SIGKILL)
<-s.reader_done
s.proc.Wait()
}
var calibre_server = sync.OnceValues(func() (_ *calibre_server_process, err error) {
defer func() { calibre_server_load_finished.Store(true) }()
ans = &calibre_server_process{}
ans.proc = exec.Command("calibre-debug", "-c", "from calibre.ebooks.metadata.cli import *; simple_metadata_server()")
ans.proc.Stderr = nil
if ans.stdout, err = ans.proc.StdoutPipe(); err != nil {
cmd := exec.Command("calibre-debug", "-c", calibre_server_program)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
if ans.stdin, err = ans.proc.StdinPipe(); err != nil {
ans.stdout.Close()
stdin, err := cmd.StdinPipe()
if err != nil {
stdout.Close()
return nil, err
}
ans.proc.SysProcAttr = &unix.SysProcAttr{Setsid: true}
if err = ans.proc.Start(); err != nil {
err = fmt.Errorf("calibre-debug executable not found in PATH, you must install the calibre program to preview these file types: %w", err)
return
cmd.SysProcAttr = &unix.SysProcAttr{Setsid: true}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("calibre-debug executable not found in PATH, you must install the calibre program to preview these file types: %w", err)
}
server := new_calibre_server_process(cmd, stdin, stdout)
calibre_needs_cleanup.Store(true)
payload, _ := json.Marshal(map[string]string{"path": ""})
if _, err = ans.stdin.Write(append(payload, '\n')); err != nil {
err = fmt.Errorf("error writing to calibre metadata server: %w", err)
return
}
line, err := ReadLineWithTimeout(ans.stdout, 2*time.Second)
defer func() {
if err != nil {
server.close()
}
}()
data, err := server.request("", "", 2*time.Second)
if err != nil {
if errors.Is(err, os.ErrDeadlineExceeded) {
err = fmt.Errorf("timed out waiting for response from calibre metadata server, make sure you are using calibre version >= 8.16")
} else {
err = fmt.Errorf("error reading from calibre metadata server: %w", err)
return nil, fmt.Errorf("timed out waiting for response from calibre metadata server, make sure you are using calibre version >= 8.16")
}
return
return nil, err
}
var r CalibreResponse
if err = json.Unmarshal([]byte(line), &r); err != nil {
err = fmt.Errorf("unexpected response from calibre metadata server: %#v", line)
return
r, err := unmarshal_calibre_response(data, "")
if err != nil {
return nil, fmt.Errorf("unexpected response from calibre metadata server: %w", err)
}
ans.file_extensions = utils.NewSet[string](len(r.Filetypes))
server.file_extensions = utils.NewSet[string](len(r.Filetypes))
for _, ft := range r.Filetypes {
ans.file_extensions.Add("." + ft)
server.file_extensions.Add("." + ft)
}
return
return server, nil
})
func calibre_cleanup() {
if !calibre_needs_cleanup.Load() {
if !calibre_needs_cleanup.Swap(false) {
return
}
calibre_needs_cleanup.Store(false)
calibre, _ := calibre_server()
if calibre.stdin != nil {
calibre.stdin.Close()
}
if calibre.stdout != nil {
calibre.stdout.Close()
}
if calibre.proc != nil {
calibre.proc.Wait()
if calibre, err := calibre_server(); err == nil {
calibre.close()
}
}
@ -155,57 +254,66 @@ func IsSupportedByCalibre(path string) bool {
const CALIBRE_METADATA_KEY = "calibre-metadata.json"
type calibre_renderer int
type calibre_renderer struct {
path string
}
func unmarshal_calibre_response(data []byte, path string) (*CalibreResponse, error) {
var ans CalibreResponse
if err := json.Unmarshal(data, &ans); err != nil {
return nil, err
}
if ans.Path != path {
return nil, fmt.Errorf("calibre metadata is for %q, expected %q", ans.Path, path)
}
if ans.Error != "" {
return nil, errors.New(ans.Error)
}
return &ans, nil
}
func (c calibre_renderer) Unmarshall(m map[string]string) (any, error) {
data, err := os.ReadFile(m[CALIBRE_METADATA_KEY])
if err != nil {
return nil, err
}
var ans CalibreResponse
if err = json.Unmarshal(data, &ans); err != nil {
ans, err := unmarshal_calibre_response(data, c.path)
if err != nil {
return nil, err
}
return &ans, nil
return ans, nil
}
func (c calibre_renderer) Render(path string) (m map[string][]byte, mi metadata, img *images.ImageData, err error) {
cpath, err := os.CreateTemp("", "")
if err != nil {
return
}
defer func() {
cpath.Close()
os.Remove(cpath.Name())
}()
calibre, err := calibre_server()
if err != nil {
return
}
payload, err := json.Marshal(map[string]string{"path": path, "cover": cpath.Name()})
cover_dir, err := os.MkdirTemp("", "kitty-calibre-")
if err != nil {
return
}
calibre.stdin.Write(append(payload, '\n'))
line, err := ReadLineWithTimeout(calibre.stdout, 30*time.Second)
// A worker finishing after the request times out cannot recreate this directory.
defer os.RemoveAll(cover_dir)
cover_path := filepath.Join(cover_dir, "cover")
data, err := calibre.request(path, cover_path, 30*time.Second)
if err != nil {
return
}
lb := []byte(line)
var reply CalibreResponse
if err = json.Unmarshal(lb, &reply); err != nil {
reply, err := unmarshal_calibre_response(data, path)
if err != nil {
return
}
if reply.Cover == cpath.Name() {
if reply.Cover == cover_path {
var ip ImagePreviewRenderer
if m, mi, img, err = ip.Render(cpath.Name()); err != nil {
if m, mi, img, err = ip.Render(cover_path); err != nil {
return
}
} else {
m = make(map[string][]byte)
}
mi.custom = &reply
m[CALIBRE_METADATA_KEY] = lb
mi.custom = reply
m[CALIBRE_METADATA_KEY] = data
return
}
@ -238,7 +346,7 @@ func (c calibre_renderer) String() string {
func NewCalibrePreview(
abspath string, metadata fs.FileInfo, opts Settings, WakeupMainThread func() bool,
) Preview {
c := calibre_renderer(0)
c := calibre_renderer{path: abspath}
if ans, err := NewImagePreview(abspath, metadata, opts, WakeupMainThread, c); err == nil {
return ans
} else {

View file

@ -0,0 +1,118 @@
package choose_files
import (
"bytes"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"testing"
"testing/synctest"
"time"
)
func TestCalibreResponseRouting(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
requests, stdin := io.Pipe()
stdout, replies := io.Pipe()
server := new_calibre_server_process(nil, stdin, stdout)
defer func() {
stdin.Close()
stdout.Close()
<-server.reader_done
}()
decoder := json.NewDecoder(requests)
start_request := func(path string, timeout time.Duration) (calibre_request, chan calibre_result) {
t.Helper()
done := make(chan calibre_result, 1)
go func() {
data, err := server.request(path, "", timeout)
done <- calibre_result{data: data, err: err}
}()
var request calibre_request
if err := decoder.Decode(&request); err != nil {
t.Fatal(err)
}
return request, done
}
make_reply := func(request calibre_request, title string) calibre_reply {
t.Helper()
data, err := json.Marshal(CalibreResponse{Path: request.Path, Metadata: CalibreMetadata{Title: title}})
if err != nil {
t.Fatal(err)
}
return calibre_reply{ID: request.ID, Response: data}
}
send := func(responses ...calibre_reply) {
t.Helper()
var data bytes.Buffer
encoder := json.NewEncoder(&data)
for _, response := range responses {
if err := encoder.Encode(response); err != nil {
t.Fatal(err)
}
}
if _, err := replies.Write(data.Bytes()); err != nil {
t.Fatal(err)
}
}
check := func(done chan calibre_result, path, title string) {
t.Helper()
result := <-done
if result.err != nil {
t.Fatal(result.err)
}
response, err := unmarshal_calibre_response(result.data, path)
if err != nil {
t.Fatal(err)
}
if response.Metadata.Title != title {
t.Fatalf("metadata for %q: got %q, want %q", path, response.Metadata.Title, title)
}
}
a, a_done := start_request("a.epub", time.Second)
b, b_done := start_request("b.epub", time.Second)
send(make_reply(b, "Book B"))
check(b_done, b.Path, "Book B")
send(make_reply(a, "Book A"))
check(a_done, a.Path, "Book A")
expired, expired_done := start_request(a.Path, time.Second)
if err := (<-expired_done).err; !errors.Is(err, os.ErrDeadlineExceeded) {
t.Fatalf("expected timeout, got %v", err)
}
retry, retry_done := start_request(a.Path, time.Second)
next, next_done := start_request(b.Path, time.Second)
// The stale and live replies arrive in one read, with no cover paths
// to distinguish the two requests for the same book.
send(make_reply(expired, "Expired"), make_reply(next, "Book B"), make_reply(retry, "Retried"))
check(retry_done, retry.Path, "Retried")
check(next_done, next.Path, "Book B")
_, pending := start_request(a.Path, time.Second)
replies.Close()
if err := (<-pending).err; !errors.Is(err, io.EOF) {
t.Fatalf("expected read error, got %v", err)
}
})
}
func TestCalibreCachedMetadataPath(t *testing.T) {
path := filepath.Join(t.TempDir(), CALIBRE_METADATA_KEY)
if err := os.WriteFile(path, []byte(`{"path":"/book.epub","metadata":{"title":"Book"}}`), 0600); err != nil {
t.Fatal(err)
}
files := map[string]string{CALIBRE_METADATA_KEY: path}
if _, err := (calibre_renderer{path: "/other.epub"}).Unmarshall(files); err == nil {
t.Fatal("accepted metadata for another book")
}
result, err := (calibre_renderer{path: "/book.epub"}).Unmarshall(files)
if err != nil {
t.Fatal(err)
}
if title := result.(*CalibreResponse).Metadata.Title; title != "Book" {
t.Fatalf("expected cached title %q, got %q", "Book", title)
}
}