mirror of
https://github.com/kovidgoyal/kitty.git
synced 2026-11-19 14:45:29 +00:00
Receiving a regular file over an existing symlink or hard link used to truncate and overwrite the file the link points to. Instead, write received files to a temporary file in the destination directory and atomically rename it into place once the transfer completes, as was already done for delta transfers. This also means interrupted transfers no longer leave truncated files behind. If the destination directory is not writable, fall back to writing in place, but only for regular files with a single link, so that no other file can be affected. Also use stat rather than lstat when deciding whether to use delta transfers, so that symlinks to large files can use them, and remove the temporary file of failed or aborted delta transfers. Closes #10713
796 lines
32 KiB
Python
796 lines
32 KiB
Python
#!/usr/bin/env python
|
|
# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
|
|
|
|
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import tempfile
|
|
from collections import namedtuple
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
|
|
from kittens.transfer.rsync import Differ, Hasher, Patcher, parse_ftc
|
|
from kittens.transfer.utils import set_paths
|
|
from kitty.constants import kitten_exe
|
|
from kitty.file_transmission import Action, Compression, FileTransmissionCommand, FileType, TransmissionType, ZlibDecompressor
|
|
from kitty.file_transmission import TestFileTransmission as FileTransmission
|
|
|
|
from .base import PTY, BaseTest
|
|
|
|
|
|
def response(id='test', msg='', file_id='', name='', action='status', status='', size=-1):
|
|
ans = {'action': 'status'}
|
|
if id:
|
|
ans['id'] = id
|
|
if file_id:
|
|
ans['file_id'] = file_id
|
|
if name:
|
|
ans['name'] = name
|
|
if status:
|
|
ans['status'] = status
|
|
if size > -1:
|
|
ans['size'] = size
|
|
return ans
|
|
|
|
|
|
def names_in(path):
|
|
for dirpath, dirnames, filenames in os.walk(path):
|
|
for d in dirnames + filenames:
|
|
yield os.path.relpath(os.path.join(dirpath, d), path)
|
|
|
|
|
|
def serialized_cmd(**fields) -> str:
|
|
if 'id' not in fields:
|
|
fields['id'] = 'test'
|
|
for k, A in (('action', Action), ('ftype', FileType), ('ttype', TransmissionType), ('compression', Compression)):
|
|
if k in fields:
|
|
fields[k] = A[fields[k]]
|
|
if isinstance(fields.get('data'), str):
|
|
fields['data'] = fields['data'].encode('utf-8')
|
|
ans = FileTransmissionCommand(**fields)
|
|
return ans.serialize()
|
|
|
|
|
|
def generate_data(block_size, num_blocks, *extra) -> bytes:
|
|
extra = ''.join(extra)
|
|
b = b'_' * (block_size * num_blocks) + extra.encode()
|
|
ans = bytearray(b)
|
|
for i in range(num_blocks):
|
|
offset = i * block_size
|
|
p = str(i).encode()
|
|
ans[offset : offset + len(p)] = p
|
|
return bytes(ans)
|
|
|
|
|
|
def patch_data(data, *patches):
|
|
total_patch_size = 0
|
|
ans = bytearray(data)
|
|
for patch in patches:
|
|
o, sep, r = patch.partition(':')
|
|
r = r.encode()
|
|
total_patch_size += len(r)
|
|
offset = int(o)
|
|
ans[offset : offset + len(r)] = r
|
|
return bytes(ans), len(patches), total_patch_size
|
|
|
|
|
|
def run_roundtrip_test(self: 'TestFileTransmission', src_data, changed, num_of_patches, total_patch_size):
|
|
buf = memoryview(bytearray(30))
|
|
signature = bytearray(0)
|
|
p = Patcher(len(changed))
|
|
n = p.signature_header(buf)
|
|
signature.extend(buf[:n])
|
|
src = memoryview(changed)
|
|
bs = p.block_size
|
|
while src:
|
|
n = p.sign_block(src[:bs], buf)
|
|
signature.extend(buf[:n])
|
|
src = src[bs:]
|
|
d = Differ()
|
|
src = memoryview(signature)
|
|
while src:
|
|
d.add_signature_data(src[:13])
|
|
src = src[13:]
|
|
d.finish_signature_data()
|
|
del src, signature
|
|
src = memoryview(src_data)
|
|
delta = bytearray(0)
|
|
|
|
def read_into(b):
|
|
nonlocal src
|
|
n = min(len(b), len(src))
|
|
if n > 0:
|
|
b[:n] = src[:n]
|
|
src = src[n:]
|
|
return n
|
|
|
|
def write_delta(b):
|
|
delta.extend(b)
|
|
|
|
while d.next_op(read_into, write_delta):
|
|
pass
|
|
delta = memoryview(delta)
|
|
del src
|
|
|
|
def read_at(pos, output) -> int:
|
|
b = changed[pos:]
|
|
amt = min(len(output), len(b))
|
|
output[:amt] = b[:amt]
|
|
return amt
|
|
|
|
output = bytearray(0)
|
|
|
|
def write_changes(b):
|
|
output.extend(b)
|
|
|
|
def debug_msg():
|
|
return f'\n\nsrc:\n{src_data.decode()}\nchanged:\n{changed.decode()}\noutput:\n{output.decode()}'
|
|
|
|
try:
|
|
while delta:
|
|
p.apply_delta_data(delta[:11], read_at, write_changes)
|
|
delta = delta[11:]
|
|
p.finish_delta_data()
|
|
except Exception as err:
|
|
self.fail(f'{err}\n{debug_msg()}')
|
|
self.assertEqual(src_data, bytes(output), debug_msg())
|
|
limit = 2 * (p.block_size * num_of_patches)
|
|
if limit > -1:
|
|
self.assertLessEqual(p.total_data_in_delta, limit, f'Unexpectedly poor delta performance: {total_patch_size=} {p.total_data_in_delta=} {limit=}')
|
|
|
|
|
|
def test_rsync_roundtrip(self: 'TestFileTransmission') -> None:
|
|
block_size = 16
|
|
src_data = generate_data(block_size, 16)
|
|
changed, num_of_patches, total_patch_size = patch_data(src_data, '3:patch1', '16:patch2', '130:ptch3', '176:patch4', '222:XXYY')
|
|
|
|
run_roundtrip_test(self, src_data, src_data[block_size:], 1, block_size)
|
|
run_roundtrip_test(self, src_data, changed, num_of_patches, total_patch_size)
|
|
run_roundtrip_test(self, src_data, b'', -1, 0)
|
|
run_roundtrip_test(self, src_data, src_data, 0, 0)
|
|
run_roundtrip_test(self, src_data, changed[: len(changed) - 3], num_of_patches, total_patch_size)
|
|
run_roundtrip_test(self, src_data, changed[:37] + changed[81:], num_of_patches, total_patch_size)
|
|
|
|
block_size = 13
|
|
src_data = generate_data(block_size, 17, 'trailer')
|
|
changed, num_of_patches, total_patch_size = patch_data(src_data, '0:patch1', '19:patch2')
|
|
run_roundtrip_test(self, src_data, changed, num_of_patches, total_patch_size)
|
|
run_roundtrip_test(self, src_data, changed[: len(changed) - 3], num_of_patches, total_patch_size)
|
|
run_roundtrip_test(self, src_data, changed + b'xyz...', num_of_patches, total_patch_size)
|
|
|
|
|
|
class PtyFileTransmission(FileTransmission):
|
|
def __init__(self, pty, allow=True):
|
|
self.pty = pty
|
|
super().__init__(allow=allow)
|
|
self.pty.callbacks.ftc = self
|
|
|
|
def write_ftc_to_child(self, payload: FileTransmissionCommand, appendleft: bool = False, use_pending: bool = True) -> bool:
|
|
# print('to kitten:', payload)
|
|
self.pty.write_to_child('\x1b]' + payload.serialize(prefix_with_osc_code=True) + '\x1b\\', flush=False)
|
|
return True
|
|
|
|
|
|
class TransferPTY(PTY):
|
|
def __init__(self, cmd, cwd, allow=True, env=None):
|
|
super().__init__(cmd, cwd=cwd, env=env, rows=200, columns=120)
|
|
self.fc = PtyFileTransmission(self, allow=allow)
|
|
|
|
|
|
class TestFileTransmission(BaseTest):
|
|
def setUp(self):
|
|
self.direction_receive = False
|
|
self.kitty_home = self.kitty_cwd = self.kitten_home = self.kitten_cwd = ''
|
|
super().setUp()
|
|
self.tdir = os.path.realpath(tempfile.mkdtemp())
|
|
self.responses = []
|
|
self.orig_home = os.environ.get('HOME')
|
|
|
|
def tearDown(self):
|
|
self.rmtree_ignoring_errors(self.tdir)
|
|
self.responses = []
|
|
if self.orig_home is None:
|
|
os.environ.pop('HOME', None)
|
|
else:
|
|
os.environ['HOME'] = self.orig_home
|
|
super().tearDown()
|
|
|
|
def clean_tdir(self):
|
|
for x in os.listdir(self.tdir):
|
|
x = os.path.join(self.tdir, x)
|
|
if os.path.isdir(x):
|
|
shutil.rmtree(x)
|
|
else:
|
|
os.remove(x)
|
|
self.responses = []
|
|
|
|
def cr(self, a, b):
|
|
def f(r):
|
|
r.pop('size', None)
|
|
return r
|
|
|
|
a = tuple(f(r) for r in a if r.get('status') != 'PROGRESS')
|
|
b = tuple(f(r) for r in b if r.get('status') != 'PROGRESS')
|
|
self.ae(a, b)
|
|
|
|
def assertResponses(self, ft, limit=1024, **kw):
|
|
self.responses.append(response(**kw))
|
|
self.cr(ft.test_responses[:limit], self.responses[:limit])
|
|
|
|
def assertPathEqual(self, a, b):
|
|
a = os.path.abspath(os.path.realpath(a))
|
|
b = os.path.abspath(os.path.realpath(b))
|
|
self.ae(a, b)
|
|
|
|
def test_rsync_roundtrip(self):
|
|
test_rsync_roundtrip(self)
|
|
|
|
def test_rsync_invalid_block_size(self):
|
|
from kittens.transfer.rsync import RsyncError
|
|
|
|
for block_size in (0, 1024 * 1024 + 1):
|
|
d = Differ()
|
|
with self.assertRaises(RsyncError):
|
|
d.add_signature_data(bytes(8) + block_size.to_bytes(4, 'little'))
|
|
|
|
def test_file_put_differential(self):
|
|
from kitty.file_transmission import PatchFile
|
|
|
|
def start_receiving():
|
|
ft = FileTransmission()
|
|
ft.handle_serialized_command(serialized_cmd(action='send'))
|
|
ft.test_responses = []
|
|
return ft
|
|
|
|
def started_ttype(ft):
|
|
q = [r for r in ft.test_responses if r.get('status') == 'STARTED']
|
|
self.ae(len(q), 1)
|
|
return q[0].get('ttype', 'simple')
|
|
|
|
def signature_sent(ft):
|
|
return b''.join(r['data'] for r in ft.test_responses if r.get('action') == 'data' and r.get('data'))
|
|
|
|
base = os.path.join(self.tdir, 'diff')
|
|
os.mkdir(base)
|
|
|
|
# a differential transfer to a plain regular file works as usual
|
|
dest = os.path.join(base, 'f.bin')
|
|
before = b'hello world ' * 5000
|
|
after = (b'hello world ' * 4000) + (b'goodbye!' * 2000)
|
|
with open(dest, 'wb') as f:
|
|
f.write(before)
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='r', name=dest, ftype='regular', ttype='rsync'))
|
|
self.ae(started_ttype(ft), 'rsync')
|
|
d = Differ()
|
|
d.add_signature_data(signature_sent(ft))
|
|
d.finish_signature_data()
|
|
src = memoryview(after)
|
|
delta = bytearray()
|
|
|
|
def read_into(b):
|
|
nonlocal src
|
|
n = min(len(b), len(src))
|
|
b[:n] = src[:n]
|
|
src = src[n:]
|
|
return n
|
|
|
|
while d.next_op(read_into, delta.extend):
|
|
pass
|
|
chunks = memoryview(bytes(delta))
|
|
while len(chunks) > 4096:
|
|
ft.handle_serialized_command(serialized_cmd(action='data', file_id='r', data=bytes(chunks[:4096])))
|
|
chunks = chunks[4096:]
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='r', data=bytes(chunks)))
|
|
with open(dest, 'rb') as f:
|
|
self.ae(f.read(), after)
|
|
self.ae(sorted(names_in(base)), ['f.bin'])
|
|
|
|
# a sender must not be able to use a differential transfer to read
|
|
# through a symlink it plants at the destination in the same transfer
|
|
secret_dir = os.path.join(base, 'secret')
|
|
os.mkdir(secret_dir)
|
|
secret = os.path.join(secret_dir, 'secret.txt')
|
|
secret_data = b'super secret data' * 1000
|
|
with open(secret, 'wb') as f:
|
|
f.write(secret_data)
|
|
sdest = os.path.join(base, 'innocent.bin')
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='s', name=sdest, ftype='symlink'))
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='s', data='path:' + secret))
|
|
self.assertTrue(os.path.islink(sdest))
|
|
ft.test_responses = []
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='r', name=sdest, ftype='regular', ttype='rsync'))
|
|
self.ae(started_ttype(ft), 'simple')
|
|
self.ae(signature_sent(ft), b'')
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='r', data='replacement'))
|
|
self.assertFalse(os.path.islink(sdest))
|
|
with open(sdest, 'rb') as f:
|
|
self.ae(f.read(), b'replacement')
|
|
with open(secret, 'rb') as f:
|
|
self.ae(f.read(), secret_data)
|
|
self.ae(sorted(names_in(secret_dir)), ['secret.txt'])
|
|
|
|
# likewise for a destination with more than one link to it
|
|
hdest = os.path.join(base, 'hard.bin')
|
|
other = os.path.join(base, 'other.bin')
|
|
with open(hdest, 'wb') as f:
|
|
f.write(b'original')
|
|
os.link(hdest, other)
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='r', name=hdest, ftype='regular', ttype='rsync'))
|
|
self.ae(started_ttype(ft), 'simple')
|
|
self.ae(signature_sent(ft), b'')
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='r', data='replacement'))
|
|
with open(other, 'rb') as f:
|
|
self.ae(f.read(), b'original')
|
|
|
|
# and PatchFile itself must never follow a symlink, in case the
|
|
# destination is swapped after it is created but before it is read
|
|
pf = PatchFile(sdest, len(secret_data))
|
|
os.remove(sdest)
|
|
os.symlink(secret, sdest)
|
|
with self.assertRaises(OSError):
|
|
pf.next_signature_block(memoryview(bytearray(64)))
|
|
# its temporary file goes next to the destination, not next to the
|
|
# symlink target
|
|
tf = pf.dest_file
|
|
tf.close()
|
|
self.assertPathEqual(os.path.dirname(tf.name), base)
|
|
os.remove(tf.name)
|
|
|
|
def test_file_put_existing_directory(self):
|
|
def start_receiving():
|
|
ft = FileTransmission()
|
|
ft.handle_serialized_command(serialized_cmd(action='send'))
|
|
ft.test_responses = []
|
|
return ft
|
|
|
|
def statuses(ft):
|
|
return [r['status'] for r in ft.test_responses if r.get('action') == 'status']
|
|
|
|
base = os.path.join(self.tdir, 'existing')
|
|
dest_dir = os.path.join(base, 'dir')
|
|
os.makedirs(os.path.join(dest_dir, 'child'))
|
|
dir_link = os.path.join(base, 'dir_link')
|
|
os.symlink(dest_dir, dir_link)
|
|
|
|
# a regular file must not replace an existing directory or a symlink
|
|
# to one, and data sent for it must be silently discarded
|
|
for dest in (dest_dir, dir_link):
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='f', name=dest, ftype='regular'))
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='f', data='xyz'))
|
|
self.ae(statuses(ft), [f'EISDIR:Cannot replace the existing directory {dest} with a file'])
|
|
self.assertTrue(os.path.isdir(dest))
|
|
self.assertTrue(os.path.islink(dir_link))
|
|
self.ae(set(names_in(dest_dir)), {'child'})
|
|
|
|
# a symlink must not replace an existing directory
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='s', name=dest_dir, ftype='symlink'))
|
|
self.ae(statuses(ft), [f'EISDIR:Cannot replace the existing directory {dest_dir} with a file'])
|
|
self.assertTrue(os.path.isdir(dest_dir))
|
|
|
|
# but it may replace a symlink to a directory
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='s', name=dir_link, ftype='symlink'))
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='s', data='path:elsewhere'))
|
|
self.ae(statuses(ft)[-1], 'OK')
|
|
self.ae(os.readlink(dir_link), 'elsewhere')
|
|
|
|
# failure to create parent directories gives a clear error
|
|
blocker = os.path.join(base, 'blocker')
|
|
with open(blocker, 'w'):
|
|
pass
|
|
dest = os.path.join(blocker, 'sub', 'f')
|
|
ft = start_receiving()
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='f', name=dest, ftype='regular'))
|
|
ft.handle_serialized_command(serialized_cmd(action='end_data', file_id='f', data='xyz'))
|
|
self.ae(statuses(ft), ['STARTED', f'ENOTDIR:Failed to create the directory {os.path.dirname(dest)}: Not a directory'])
|
|
|
|
def test_file_get(self):
|
|
# send refusal
|
|
for quiet in (0, 1, 2):
|
|
ft = FileTransmission(allow=False)
|
|
ft.handle_serialized_command(serialized_cmd(action='receive', id='x', quiet=quiet))
|
|
self.cr(ft.test_responses, [] if quiet == 2 else [response(id='x', status='EPERM:User refused the transfer')])
|
|
self.assertFalse(ft.active_sends)
|
|
# reading metadata for specs
|
|
cwd = os.path.join(self.tdir, 'cwd')
|
|
home = os.path.join(self.tdir, 'home')
|
|
os.mkdir(cwd), os.mkdir(home)
|
|
with set_paths(cwd=cwd, home=home):
|
|
ft = FileTransmission()
|
|
self.responses = []
|
|
ft.handle_serialized_command(serialized_cmd(action='receive', size=1))
|
|
self.assertResponses(ft, status='OK')
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='missing', name='XXX'))
|
|
self.responses.append(response(status='ENOENT:Failed to read spec', file_id='missing'))
|
|
self.assertResponses(ft, status='OK', name=home)
|
|
ft = FileTransmission()
|
|
self.responses = []
|
|
ft.handle_serialized_command(serialized_cmd(action='receive', size=2))
|
|
self.assertResponses(ft, status='OK')
|
|
with open(os.path.join(home, 'a'), 'w') as f:
|
|
f.write('a')
|
|
os.mkdir(f.name + 'd')
|
|
with open(os.path.join(f.name + 'd', 'b'), 'w') as f2:
|
|
f2.write('bbb')
|
|
os.symlink(f.name, f.name + 'd/s')
|
|
os.link(f.name, f.name + 'd/h')
|
|
os.symlink('XXX', f.name + 'd/q')
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='a', name='a'))
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='b', name='ad'))
|
|
files = {r['name']: r for r in ft.test_responses if r['action'] == 'file'}
|
|
self.ae(len(files), 6)
|
|
q = files[f.name]
|
|
tgt = q['status'].encode('ascii')
|
|
self.ae(q['size'], 1), self.assertNotIn('ftype', q)
|
|
q = files[f.name + 'd']
|
|
self.ae(q['ftype'], 'directory')
|
|
q = files[f.name + 'd/b']
|
|
self.ae(q['size'], 3)
|
|
q = files[f.name + 'd/s']
|
|
self.ae(q['ftype'], 'symlink')
|
|
self.ae(q['data'], tgt)
|
|
q = files[f.name + 'd/h']
|
|
self.ae(q['ftype'], 'link')
|
|
self.ae(q['data'], tgt)
|
|
q = files[f.name + 'd/q']
|
|
self.ae(q['ftype'], 'symlink')
|
|
self.assertNotIn('data', q)
|
|
base = os.path.join(self.tdir, 'base')
|
|
os.mkdir(base)
|
|
src = os.path.join(base, 'src.bin')
|
|
data = os.urandom(16 * 1024)
|
|
with open(src, 'wb') as f:
|
|
f.write(data)
|
|
sl = os.path.join(base, 'src.link')
|
|
os.symlink(src, sl)
|
|
for compress in ('none', 'zlib'):
|
|
ft = FileTransmission()
|
|
self.responses = []
|
|
ft.handle_serialized_command(serialized_cmd(action='receive', size=1))
|
|
self.assertResponses(ft, status='OK')
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='src', name=src))
|
|
ft.active_sends['test'].metadata_sent = True
|
|
ft.test_responses = []
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='src', name=src, compression=compress))
|
|
received = b''.join(x['data'] for x in ft.test_responses)
|
|
if compress == 'zlib':
|
|
received = ZlibDecompressor()(received, True)
|
|
self.ae(data, received)
|
|
ft.test_responses = []
|
|
ft.handle_serialized_command(serialized_cmd(action='file', file_id='sl', name=sl, compression=compress))
|
|
received = b''.join(x['data'] for x in ft.test_responses)
|
|
self.ae(received.decode('utf-8'), src)
|
|
|
|
def test_check_bypass(self):
|
|
import json
|
|
from base64 import b85encode
|
|
from time import time_ns
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from kitty.fast_data_types import AES256GCMEncrypt, EllipticCurveKey
|
|
from kitty.file_transmission import check_bypass, encode_bypass
|
|
|
|
from .crypto import is_rlimit_memlock_too_low
|
|
|
|
if is_rlimit_memlock_too_low():
|
|
self.skipTest('RLIMIT_MEMLOCK is too low')
|
|
kitty_key, remote_key = EllipticCurveKey(), EllipticCurveKey()
|
|
|
|
def kitty_1(request_id: str, password: str) -> str:
|
|
e = AES256GCMEncrypt(remote_key.derive_secret(kitty_key.public))
|
|
encrypted = e.add_data_to_be_encrypted(f'{time_ns()}:{request_id};{password}'.encode(), True)
|
|
return 'kitty-1:' + json.dumps(
|
|
{k: b85encode(v).decode() for k, v in {'pubkey': remote_key.public, 'iv': e.iv, 'tag': e.tag, 'encrypted': encrypted}.items()}
|
|
)
|
|
|
|
with patch('kitty.file_transmission.get_boss', return_value=SimpleNamespace(encryption_key=kitty_key)):
|
|
self.assertTrue(check_bypass('secret', 'rid', kitty_1('rid', 'secret')))
|
|
self.assertFalse(check_bypass('secret', 'rid', kitty_1('rid', 'wrong')))
|
|
self.assertFalse(check_bypass('secret', 'rid', kitty_1('other', 'secret')))
|
|
self.assertTrue(check_bypass('secret', 'rid', encode_bypass('rid', 'secret')))
|
|
# when no bypass password is configured, bypass must never succeed
|
|
self.assertFalse(check_bypass('', 'rid', kitty_1('rid', '')))
|
|
self.assertFalse(check_bypass('', 'rid', encode_bypass('rid', '')))
|
|
|
|
def test_parse_ftc(self):
|
|
def t(raw, *expected):
|
|
a = []
|
|
|
|
def c(k, v):
|
|
a.append(str(k, 'utf-8'))
|
|
a.append(str(v, 'utf-8'))
|
|
|
|
parse_ftc(raw, c)
|
|
self.ae(tuple(a), expected)
|
|
|
|
t('a=b', 'a', 'b')
|
|
t('a=b;', 'a', 'b')
|
|
t('a1=b1;c=d;;', 'a1', 'b1', 'c', 'd')
|
|
t('a1=b1;c=d;;e', 'a1', 'b1', 'c', 'd')
|
|
t('a1=b1;c=d;;;1=1', 'a1', 'b1', 'c', 'd', '1', '1')
|
|
|
|
def test_rsync_hashers(self):
|
|
h = Hasher('xxh3-64')
|
|
h.update(b'abcd')
|
|
self.assertEqual(h.hexdigest(), '6497a96f53a89890')
|
|
self.assertEqual(h.digest64(), 7248448420886124688)
|
|
h128 = Hasher('xxh3-128')
|
|
h128.update(b'abcd')
|
|
self.assertEqual(h128.hexdigest(), '8d6b60383dfa90c21be79eecd1b1353d')
|
|
|
|
@contextmanager
|
|
def run_kitten(self, cmd, home_dir='', allow=True, cwd=''):
|
|
cwd = cwd or self.kitten_cwd or self.tdir
|
|
cmd = [kitten_exe(), 'transfer'] + (['--direction=receive'] if self.direction_receive else []) + cmd
|
|
env = {'PWD': cwd}
|
|
env['HOME'] = home_dir or self.kitten_home or self.tdir
|
|
with set_paths(home=self.kitty_home, cwd=self.kitty_cwd):
|
|
pty = TransferPTY(cmd, cwd=cwd, allow=allow, env=env)
|
|
i = 10
|
|
while i > 0 and not pty.screen_contents().strip():
|
|
pty.process_input_from_child()
|
|
i -= 1
|
|
yield pty
|
|
|
|
def basic_transfer_tests(self):
|
|
src = os.path.join(self.tdir, 'src')
|
|
self.src_data = os.urandom(11113)
|
|
with open(src, 'wb') as s:
|
|
s.write(self.src_data)
|
|
dest = os.path.join(self.tdir, 'dest')
|
|
with self.run_kitten([src, dest], allow=False) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=1)
|
|
self.assertFalse(os.path.exists(dest))
|
|
|
|
def single_file(*cmd):
|
|
with self.run_kitten(list(cmd) + [src, dest]) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
with open(dest, 'rb') as f:
|
|
self.assertEqual(self.src_data, f.read())
|
|
|
|
single_file()
|
|
single_file()
|
|
single_file('--transmit-deltas')
|
|
with open(dest, 'wb') as d:
|
|
d.write(os.urandom(1023))
|
|
single_file('--transmit-deltas')
|
|
os.remove(dest)
|
|
single_file('--transmit-deltas')
|
|
single_file('--compress=never')
|
|
single_file('--compress=always')
|
|
single_file('--transmit-deltas', '--compress=never')
|
|
|
|
def empty_file(*cmd, existing_dest_data=None):
|
|
esrc = os.path.join(self.tdir, 'esrc')
|
|
edest = os.path.join(self.tdir, 'edest')
|
|
open(esrc, 'wb').close()
|
|
if existing_dest_data is None:
|
|
if os.path.exists(edest):
|
|
os.remove(edest)
|
|
else:
|
|
with open(edest, 'wb') as d:
|
|
d.write(existing_dest_data)
|
|
with self.run_kitten(list(cmd) + [esrc, edest]) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
self.assertTrue(os.path.isfile(edest), f'{edest} was not created by: {cmd}')
|
|
self.ae(os.path.getsize(edest), 0, f'{edest} is not empty after: {cmd}')
|
|
|
|
empty_file()
|
|
# an existing destination must be truncated
|
|
empty_file(existing_dest_data=b'stale')
|
|
# an existing destination larger than the signature threshold causes an rsync transfer
|
|
empty_file('--transmit-deltas', existing_dest_data=os.urandom(8192))
|
|
empty_file('--transmit-deltas')
|
|
|
|
def multiple_files(*cmd):
|
|
src = os.path.join(self.tdir, 'msrc')
|
|
dest = os.path.join(self.tdir, 'mdest')
|
|
if os.path.exists(src):
|
|
shutil.rmtree(src)
|
|
os.mkdir(src)
|
|
os.makedirs(dest, exist_ok=True)
|
|
|
|
expected = {}
|
|
Entry = namedtuple('Entry', 'relpath mtime mode nlink')
|
|
|
|
def entry(path, base=src):
|
|
st = os.stat(path, follow_symlinks=False)
|
|
mtime = st.st_mtime_ns
|
|
if stat.S_ISDIR(st.st_mode):
|
|
mtime = 0 # mtime is flaky for dirs on CI even empty ones
|
|
return Entry(os.path.relpath(path, base), mtime, oct(st.st_mode), st.st_nlink)
|
|
|
|
def se(path):
|
|
e = entry(path)
|
|
expected[e.relpath] = e
|
|
|
|
b = Path(src)
|
|
with open(b / 'simple', 'wb') as f:
|
|
f.write(os.urandom(1317))
|
|
os.fchmod(f.fileno(), 0o766)
|
|
os.link(f.name, b / 'hardlink')
|
|
os.utime(f.name, (1.3, 1.3))
|
|
se(f.name)
|
|
se(str(b / 'hardlink'))
|
|
os.mkdir(b / 'empty')
|
|
se(str(b / 'empty'))
|
|
with open(b / 'emptyfile', 'wb') as f:
|
|
os.fchmod(f.fileno(), 0o640)
|
|
os.utime(f.name, (2345.6, 2345.6))
|
|
se(f.name)
|
|
s = b / 'sub'
|
|
os.mkdir(s)
|
|
with open(s / 'reg', 'wb') as f:
|
|
f.write(os.urandom(113))
|
|
os.utime(f.name, (1171.3, 1171.3))
|
|
se(f.name)
|
|
se(str(s))
|
|
os.symlink('/', b / 'abssym')
|
|
os.utime(b / 'abssym', (1234.5, 1234.5), follow_symlinks=False)
|
|
se(b / 'abssym')
|
|
os.symlink('sub/reg', b / 'sym')
|
|
os.utime(b / 'sym', (6789.1, 6789.1), follow_symlinks=False)
|
|
se(b / 'sym')
|
|
|
|
with self.run_kitten(list(cmd) + [src, dest]) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
|
|
actual = {}
|
|
|
|
def de(path):
|
|
e = entry(path, os.path.join(dest, os.path.basename(src)))
|
|
if e.relpath != '.':
|
|
actual[e.relpath] = e
|
|
|
|
for dirpath, dirnames, filenames in os.walk(dest):
|
|
for x in dirnames:
|
|
de(os.path.join(dirpath, x))
|
|
for x in filenames:
|
|
de(os.path.join(dirpath, x))
|
|
|
|
self.assertEqual(expected, actual)
|
|
|
|
for key, e in expected.items():
|
|
ex = os.path.join(src, key)
|
|
ax = os.path.join(dest, os.path.basename(src), key)
|
|
if os.path.islink(ex):
|
|
self.ae(os.readlink(ex), os.readlink(ax))
|
|
elif os.path.isfile(ex):
|
|
with open(ex, 'rb') as ef, open(ax, 'rb') as af:
|
|
self.assertEqual(ef.read(), af.read())
|
|
|
|
multiple_files()
|
|
multiple_files('--compress=always')
|
|
self.clean_tdir()
|
|
multiple_files('--transmit-deltas')
|
|
multiple_files('--transmit-deltas')
|
|
|
|
def setup_dirs(self):
|
|
self.clean_tdir()
|
|
self.kitty_home = os.path.join(self.tdir, 'kitty-home')
|
|
self.kitty_cwd = os.path.join(self.tdir, 'kitty-cwd')
|
|
self.kitten_home = os.path.join(self.tdir, 'kitten-home')
|
|
self.kitten_cwd = os.path.join(self.tdir, 'kitten-cwd')
|
|
tuple(map(os.mkdir, (self.kitty_home, self.kitty_cwd, self.kitten_home, self.kitten_cwd)))
|
|
|
|
def create_src(self, base):
|
|
src = os.path.join(base, 'src')
|
|
with open(src, 'wb') as s:
|
|
s.write(self.src_data)
|
|
return src
|
|
|
|
def mirror_test(self, src, dest, prefix=''):
|
|
self.create_src(src)
|
|
os.symlink('/', os.path.join(src, 'sym'))
|
|
os.mkdir(os.path.join(src, 'sub'))
|
|
os.link(os.path.join(src, 'src'), os.path.join(src, 'sub', 'hardlink'))
|
|
with self.run_kitten(['--mode=mirror', f'{prefix}src', f'{prefix}sym', f'{prefix}sub']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(dest, 'src'))
|
|
os.remove(os.path.join(dest, 'sym'))
|
|
shutil.rmtree(os.path.join(dest, 'sub'))
|
|
|
|
def test_transfer_receive_existing_links(self):
|
|
self.direction_receive = True
|
|
for link in (os.symlink, os.link):
|
|
for args in ([], ['--compress=always'], ['--transmit-deltas']):
|
|
for data in (b'', b'incoming contents' * 1024):
|
|
with self.subTest(link=link.__name__, args=args, size=len(data)):
|
|
self.clean_tdir()
|
|
base = Path(self.tdir)
|
|
src = base / 'source'
|
|
src.write_bytes(data)
|
|
src.chmod(0o644)
|
|
target = base / 'unrelated'
|
|
original = b'keep these contents' * 1024
|
|
target.write_bytes(original)
|
|
target.chmod(0o600)
|
|
before = target.stat()
|
|
dest = base / 'downloads' / 'received'
|
|
dest.parent.mkdir()
|
|
link(target, dest)
|
|
with self.run_kitten([*args, str(src), str(dest)]) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
self.assertFalse(dest.is_symlink())
|
|
self.assertFalse(os.path.samefile(target, dest))
|
|
self.ae(dest.read_bytes(), data)
|
|
self.ae(stat.S_IMODE(dest.stat().st_mode), 0o644)
|
|
self.ae(target.read_bytes(), original)
|
|
self.ae(target.stat().st_mtime_ns, before.st_mtime_ns)
|
|
self.ae(stat.S_IMODE(target.stat().st_mode), 0o600)
|
|
self.ae(os.listdir(dest.parent), ['received'])
|
|
|
|
def test_transfer_receive(self):
|
|
self.direction_receive = True
|
|
self.basic_transfer_tests()
|
|
|
|
self.setup_dirs()
|
|
self.create_src(self.kitty_home)
|
|
|
|
# dir expansion with single transfer
|
|
with self.run_kitten(['~/src', '~/src']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitten_home, 'src'))
|
|
|
|
with self.run_kitten(['src', 'src']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitten_cwd, 'src'))
|
|
|
|
# dir expansion with multiple transfers
|
|
os.symlink('/', os.path.join(self.kitty_home, 'sym'))
|
|
with self.run_kitten(['~/src', '~/sym', '~']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitten_home, 'src'))
|
|
os.remove(os.path.join(self.kitten_home, 'sym'))
|
|
|
|
with self.run_kitten(['src', 'sym', '.']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitten_cwd, 'src'))
|
|
os.remove(os.path.join(self.kitten_cwd, 'sym'))
|
|
|
|
# mirroring
|
|
self.setup_dirs()
|
|
self.mirror_test(self.kitty_home, self.kitten_home)
|
|
|
|
def test_transfer_send(self):
|
|
self.basic_transfer_tests()
|
|
src = os.path.join(self.tdir, 'src')
|
|
with open(src, 'wb') as s:
|
|
s.write(self.src_data)
|
|
|
|
self.setup_dirs()
|
|
self.create_src(self.kitten_home)
|
|
|
|
# dir expansion with single transfer
|
|
with self.run_kitten(['~/src', '~/src']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitty_home, 'src'))
|
|
|
|
self.create_src(self.kitten_cwd)
|
|
with self.run_kitten(['src', 'src']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitty_home, 'src'))
|
|
|
|
# dir expansion with multiple transfers
|
|
os.symlink('/', os.path.join(self.kitten_home, 'sym'))
|
|
with self.run_kitten(['~/src', '~/sym', '~']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitty_home, 'src'))
|
|
os.remove(os.path.join(self.kitty_home, 'sym'))
|
|
|
|
os.symlink('/', os.path.join(self.kitten_cwd, 'sym'))
|
|
with self.run_kitten(['src', 'sym', '.']) as pty:
|
|
pty.wait_till_child_exits(require_exit_code=0)
|
|
os.remove(os.path.join(self.kitty_home, 'src'))
|
|
os.remove(os.path.join(self.kitty_home, 'sym'))
|
|
|
|
# mirroring
|
|
self.setup_dirs()
|
|
self.mirror_test(self.kitten_home, self.kitty_home, prefix='~/')
|