transfer kitten: Fix receiving empty files
Some checks are pending
CI / Linux (python=3.14 cc=clang sanitize=1) (push) Waiting to run
CI / Linux (python=3.12 cc=gcc sanitize=0) (push) Waiting to run
CI / Linux (python=3.13 cc=gcc sanitize=1) (push) Waiting to run
CI / Linux package (push) Waiting to run
CI / Bundle test (macos-latest) (push) Waiting to run
CI / Bundle test (ubuntu-latest) (push) Waiting to run
CI / macOS Brew (push) Waiting to run
CI / Test ./dev.sh and benchmark (push) Waiting to run
CodeQL / CodeQL-Build (actions, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, macos-latest) (push) Waiting to run
CodeQL / CodeQL-Build (c, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (go, ubuntu-latest) (push) Waiting to run
CodeQL / CodeQL-Build (python, ubuntu-latest) (push) Waiting to run
Depscan / Scan dependencies for vulnerabilities (push) Waiting to run

The output file for a regular file was only created when the
decompressor first emitted data, which never happens for an empty file.
So empty files were not created and existing destination files were
left with their old contents. Open the output file as soon as the first
chunk for a regular file arrives instead.

Closes #10689
This commit is contained in:
Kovid Goyal 2026-10-18 14:33:13 +05:30
parent 3494fbda51
commit 56deb9e86d
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
4 changed files with 177 additions and 21 deletions

View file

@ -259,6 +259,9 @@ Detailed list of changes
- diff kitten: Fix comparing remote directories over SSH failing when they
contain read-only directories (:pull:`10676`)
- transfer kitten: Fix receiving an empty file not creating it, or leaving an
existing destination file unchanged instead of truncating it
- Vertical tab bars: When :opt:`tab_title_max_length` is set, size the tab bar
to fit the decorations of the current :opt:`tab_bar_style` rather than always
adding eight cells, allowing much narrower tab bars (:iss:`10680`)

View file

@ -172,32 +172,42 @@ func (self *remote_file) Write(data []byte) (n int, err error) {
self.remote_symlink_value += string(data)
return len(data), nil
case FileType_regular:
if self.actual_file == nil {
parent := filepath.Dir(self.expanded_local_path)
if parent != "" {
if err = os.MkdirAll(parent, 0o755); err != nil {
return 0, err
}
}
if self.expect_diff {
if pf, err := new_patch_file(self.expanded_local_path, self.patcher); err != nil {
return 0, err
} else {
self.actual_file = pf
}
} else {
if ff, err := os.Create(self.expanded_local_path); err != nil {
return 0, err
} else {
f := filesystem_file{f: ff}
self.actual_file = &f
}
}
if err = self.ensure_open(); err != nil {
return 0, err
}
return self.actual_file.write(data)
}
}
// Create the local output file for a regular file if not already done. This
// must happen even if no bytes are ever written, so that empty files are
// created and existing files are truncated.
func (self *remote_file) ensure_open() (err error) {
if self.actual_file != nil {
return
}
parent := filepath.Dir(self.expanded_local_path)
if parent != "" {
if err = os.MkdirAll(parent, 0o755); err != nil {
return
}
}
if self.expect_diff {
pf, err := new_patch_file(self.expanded_local_path, self.patcher)
if err != nil {
return err
}
self.actual_file = pf
} else {
ff, err := os.Create(self.expanded_local_path)
if err != nil {
return err
}
self.actual_file = &filesystem_file{f: ff}
}
return
}
func (self *remote_file) write_data(data []byte, is_last bool) (amt_written int64, err error) {
self.received_bytes += int64(len(data))
var base, pos int64
@ -206,6 +216,14 @@ func (self *remote_file) write_data(data []byte, is_last bool) (amt_written int6
err = fmt.Errorf("Failed writing to %s with error: %w", self.expanded_local_path, err)
}
}()
if self.ftype == FileType_regular {
// Open the output file when its first chunk arrives rather than
// relying on the decompressor producing output, which it does not do
// for empty files.
if err = self.ensure_open(); err != nil {
return 0, err
}
}
if self.actual_file != nil {
base, err = self.actual_file.tell()
if err != nil {

View file

@ -0,0 +1,109 @@
// License: GPLv3 Copyright: 2026, Kovid Goyal, <kovid at kovidgoyal.net>
package transfer
import (
"bytes"
"compress/zlib"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
var _ = fmt.Print
func TestReceiveEmptyFile(t *testing.T) {
tdir := t.TempDir()
zlib_compress := func(data []byte) []byte {
var b bytes.Buffer
w := zlib.NewWriter(&b)
if _, err := w.Write(data); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
receive := func(name, compress string, size int64, expected_compression Compression, chunks ...[]byte) string {
t.Helper()
ftc := &FileTransmissionCommand{Action: Action_file, File_id: "0", Name: name, Ftype: FileType_regular, Size: size}
f, err := new_remote_file(&Options{Compress: compress}, ftc, 1)
if err != nil {
t.Fatal(err)
}
if f.compression_type != expected_compression {
t.Fatalf("Unexpected compression type for %s: %s != %s", name, f.compression_type, expected_compression)
}
f.expanded_local_path = filepath.Join(tdir, name)
for i, chunk := range chunks {
is_last := i == len(chunks)-1
if _, err = f.write_data(chunk, is_last); err != nil {
t.Fatalf("Writing chunk %d of %s failed with error: %s", i, name, err)
}
}
if f.actual_file != nil {
t.Fatalf("Output file for %s not closed after last chunk", name)
}
return f.expanded_local_path
}
assert_contents := func(path string, expected []byte) {
t.Helper()
actual, err := os.ReadFile(path)
if err != nil {
t.Fatalf("Failed to read %s with error: %s", path, err)
}
if !bytes.Equal(actual, expected) {
t.Fatalf("Contents of %s incorrect: %q != %q", path, actual, expected)
}
}
// A single end_data with no payload, as sent for empty files
assert_contents(receive("empty", "never", 0, Compression_none, nil), nil)
// Parent directories must be created for empty files too
assert_contents(receive(filepath.Join("sub", "dir", "empty"), "never", 0, Compression_none, nil), nil)
// An existing file must be truncated when the source is empty
existing := filepath.Join(tdir, "existing")
if err := os.WriteFile(existing, []byte("stale contents"), 0o600); err != nil {
t.Fatal(err)
}
assert_contents(receive("existing", "never", 0, Compression_none, nil), nil)
// A zlib stream that decompresses to nothing, which happens when the
// file shrinks to empty after its metadata was sent
assert_contents(receive("zempty", "always", 8192, Compression_zlib, zlib_compress(nil)), nil)
zstream := zlib_compress(nil)
assert_contents(receive("zempty-split", "always", 8192, Compression_zlib, zstream[:1], zstream[1:]), nil)
// Non-empty files continue to work, including when the last chunk is empty
assert_contents(receive("nonempty", "never", 3, Compression_none, []byte("abc"), nil), []byte("abc"))
assert_contents(receive("znonempty", "always", 8192, Compression_zlib, zlib_compress([]byte("abc"))), []byte("abc"))
}
func TestReceiveEmptyFileError(t *testing.T) {
tdir := t.TempDir()
// Make the destination impossible to create by having its parent be a regular file
blocker := filepath.Join(tdir, "blocker")
if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
ftc := &FileTransmissionCommand{Action: Action_file, File_id: "0", Name: "empty", Ftype: FileType_regular}
f, err := new_remote_file(&Options{Compress: "never"}, ftc, 1)
if err != nil {
t.Fatal(err)
}
f.expanded_local_path = filepath.Join(blocker, "empty")
_, err = f.write_data(nil, true)
if err == nil {
t.Fatalf("Receiving an empty file into an uncreatable path did not fail")
}
if !strings.Contains(err.Error(), f.expanded_local_path) {
t.Fatalf("Error does not mention the destination path: %s", err)
}
}

View file

@ -517,6 +517,28 @@ class TestFileTransmission(BaseTest):
single_file('--compress=always')
single_file('--transmit-deltas', '--compress=never')
def empty_file(*cmd, existing_dest_data=None):
esrc = os.path.join(self.tdir, 'esrc')
edest = os.path.join(self.tdir, 'edest')
open(esrc, 'wb').close()
if existing_dest_data is None:
if os.path.exists(edest):
os.remove(edest)
else:
with open(edest, 'wb') as d:
d.write(existing_dest_data)
with self.run_kitten(list(cmd) + [esrc, edest]) as pty:
pty.wait_till_child_exits(require_exit_code=0)
self.assertTrue(os.path.isfile(edest), f'{edest} was not created by: {cmd}')
self.ae(os.path.getsize(edest), 0, f'{edest} is not empty after: {cmd}')
empty_file()
# an existing destination must be truncated
empty_file(existing_dest_data=b'stale')
# an existing destination larger than the signature threshold causes an rsync transfer
empty_file('--transmit-deltas', existing_dest_data=os.urandom(8192))
empty_file('--transmit-deltas')
def multiple_files(*cmd):
src = os.path.join(self.tdir, 'msrc')
dest = os.path.join(self.tdir, 'mdest')
@ -549,6 +571,10 @@ class TestFileTransmission(BaseTest):
se(str(b / 'hardlink'))
os.mkdir(b / 'empty')
se(str(b / 'empty'))
with open(b / 'emptyfile', 'wb') as f:
os.fchmod(f.fileno(), 0o640)
os.utime(f.name, (2345.6, 2345.6))
se(f.name)
s = b / 'sub'
os.mkdir(s)
with open(s / 'reg', 'wb') as f: