From 125ea08143fa697d0668ce8b214efefb61037a8e Mon Sep 17 00:00:00 2001 From: Ron Frederick Date: Sat, 28 Sep 2024 16:40:46 -0700 Subject: [PATCH] Add support for OpenSSH "limits" extension This commit adds client and server support for the OpenSSH "limits" extension, which allows the client to query server limits such as the maximum supported read and write size, improving performance between clients and servers that can support larger sizes. When AsyncSSH is acting as a server, it advertises support for up to 4 MiB reads and write, and a large enough packet size to hold such write requests. As a client, it will query these values from servers supporting the extension and automatically default to the largest supported size. When a server does not support this extension, AsyncSSH will fall back to a "safe" maxmium size of 16 KiB for both reads and writes. SCP has also been adjusted from a default size of 16 KiB to 256 KiB, which seemed to be the sweet spot after some local performance testing. As before, callers can always choose to override this default with the block_size parameter on calls to open() or to the higher-level get/put/copy functions, but generally speaking this should not be necessary. --- asyncssh/scp.py | 15 ++-- asyncssh/sftp.py | 195 +++++++++++++++++++++++++++++++++++---------- tests/test_sftp.py | 69 ++++++++++------ 3 files changed, 208 insertions(+), 71 deletions(-) diff --git a/asyncssh/scp.py b/asyncssh/scp.py index cb5e1b2..4472305 100644 --- a/asyncssh/scp.py +++ b/asyncssh/scp.py @@ -42,7 +42,7 @@ from .sftp import SFTPAttrs, SFTPGlob, SFTPName, SFTPServer, SFTPServerFS from .sftp import SFTPFileProtocol, SFTPError, SFTPFailure, SFTPBadMessage from .sftp import SFTPConnectionLost, SFTPErrorHandler, SFTPProgressHandler -from .sftp import SFTP_BLOCK_SIZE, local_fs +from .sftp import local_fs if TYPE_CHECKING: @@ -57,6 +57,9 @@ _SCPConnPath = Union[Tuple[_SCPConn, _SCPPath], _SCPConn, _SCPPath] +_SCP_BLOCK_SIZE = 256*1024 # 256 KiB + + class _SCPFSProtocol(Protocol): """Protocol for accessing a filesystem during an SCP copy""" @@ -409,7 +412,7 @@ class _SCPSource(_SCPHandler): def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]', preserve: bool, recurse: bool, - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None, server: bool = False): super().__init__(reader, writer, error_handler, server) @@ -568,7 +571,7 @@ class _SCPSink(_SCPHandler): def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]', must_be_dir: bool, preserve: bool, - recurse: bool, block_size: int = SFTP_BLOCK_SIZE, + recurse: bool, block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None, server: bool = False): super().__init__(reader, writer, error_handler, server) @@ -736,7 +739,7 @@ def __init__(self, src_reader: 'SSHReader[bytes]', src_writer: 'SSHWriter[bytes]', dst_reader: 'SSHReader[bytes]', dst_writer: 'SSHWriter[bytes]', - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None): self._source = _SCPHandler(src_reader, src_writer) @@ -898,7 +901,7 @@ async def run(self) -> None: async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]], dstpath: _SCPConnPath = None, *, preserve: bool = False, - recurse: bool = False, block_size: int = SFTP_BLOCK_SIZE, + recurse: bool = False, block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None, **kwargs) -> None: """Copy files using SCP @@ -955,7 +958,7 @@ async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]], SFTP instead. The block_size value controls the size of read and write operations - issued to copy the files. It defaults to 16 KB. + issued to copy the files. It defaults to 256 KB. If progress_handler is specified, it will be called after each block of a file is successfully copied. The arguments passed to diff --git a/asyncssh/sftp.py b/asyncssh/sftp.py index 3eb115c..48bb9c9 100644 --- a/asyncssh/sftp.py +++ b/asyncssh/sftp.py @@ -148,20 +148,24 @@ SFTPErrorHandler = Union[None, Literal[False], Callable[[Exception], None]] SFTPProgressHandler = Optional[Callable[[bytes, bytes, int, int], None]] +_T = TypeVar('_T') + MIN_SFTP_VERSION = 3 MAX_SFTP_VERSION = 6 -SFTP_BLOCK_SIZE = 16384 -_MAX_SFTP_READ_SIZE = 4*1024*1024 # 4 MiB +SAFE_SFTP_READ_LEN = 16*1024 # 16 KiB +SAFE_SFTP_WRITE_LEN = 16*1024 # 16 KiB + +MAX_SFTP_READ_LEN = 4*1024*1024 # 4 MiB +MAX_SFTP_WRITE_LEN = 4*1024*1024 # 4 MiB +MAX_SFTP_PACKET_LEN = MAX_SFTP_WRITE_LEN + 1024 _MAX_SFTP_REQUESTS = 128 _MAX_READDIR_NAMES = 128 _NSECS_IN_SEC = 1_000_000_000 -_T = TypeVar('_T') - _const_dict: Mapping[str, int] = constants.__dict__ @@ -229,6 +233,14 @@ async def close(self) -> None: class _SFTPFSProtocol(Protocol): """Protocol for accessing a filesystem via an SFTP server""" + @property + def max_read_len(self) -> int: + """Maximum read length associated with this SFTP session""" + + @property + def max_write_len(self) -> int: + """Maximum write length associated with this SFTP session""" + @staticmethod def basename(path: bytes) -> bytes: """Return the final component of a POSIX-style path""" @@ -666,7 +678,7 @@ async def iter(self) -> AsyncIterator[Tuple[int, _T]]: offset, size, count, result = task.result() yield offset, result - if count < size: + if count and count < size: self._pending.add(asyncio.ensure_future( self._start_task(offset+count, size-count))) except SFTPEOFError: @@ -795,20 +807,24 @@ async def run(self) -> None: if self._progress_handler and self._total_bytes == 0: self._progress_handler(self._srcpath, self._dstpath, 0, 0) - async for offset, datalen in self.iter(): - if not datalen: - exc = SFTPFailure('Unexpected EOF during file copy') + async for _, datalen in self.iter(): + if datalen: + self._bytes_copied += datalen - setattr(exc, 'filename', self._srcpath) - setattr(exc, 'offset', offset) + if self._progress_handler: + self._progress_handler(self._srcpath, self._dstpath, + self._bytes_copied, + self._total_bytes) + + if self._bytes_copied != self._total_bytes: + exc = SFTPFailure('Unexpected EOF during file copy') + + setattr(exc, 'filename', self._srcpath) + setattr(exc, 'offset', self._bytes_copied) + + raise exc - raise exc - if self._progress_handler: - self._bytes_copied += datalen - self._progress_handler(self._srcpath, self._dstpath, - self._bytes_copied, - self._total_bytes) finally: if self._src: # pragma: no branch await self._src.close() @@ -2030,6 +2046,37 @@ def decode(cls, packet: SSHPacket, sftp_version: int) -> 'SFTPName': return cls(filename, longname, attrs) +class SFTPLimits(Record): + """SFTP server limits""" + + max_packet_len: int + max_read_len: int + max_write_len: int + max_open_handles: int + + def encode(self, sftp_version: int) -> bytes: + """Encode SFTP server limits in an SSH packet""" + + # pylint: disable=unused-argument + + return (UInt64(self.max_packet_len) + UInt64(self.max_read_len) + + UInt64(self.max_write_len) + UInt64(self.max_open_handles)) + + @classmethod + def decode(cls, packet: SSHPacket, sftp_version: int) -> 'SFTPLimits': + """Decode bytes in an SSH packet as SFTP server limits""" + + # pylint: disable=unused-argument + + max_packet_len = packet.get_uint64() + max_read_len = packet.get_uint64() + max_write_len = packet.get_uint64() + max_open_handles = packet.get_uint64() + + return cls(max_packet_len, max_read_len, + max_write_len, max_open_handles) + + class SFTPGlob: """SFTP glob matcher""" @@ -2240,7 +2287,8 @@ class SFTPHandler(SSHPacketLogger): FXP_STAT: FXP_ATTRS, FXP_READLINK: FXP_NAME, b'statvfs@openssh.com': FXP_EXTENDED_REPLY, - b'fstatvfs@openssh.com': FXP_EXTENDED_REPLY + b'fstatvfs@openssh.com': FXP_EXTENDED_REPLY, + b'limits@openssh.com': FXP_EXTENDED_REPLY } def __init__(self, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]'): @@ -2248,6 +2296,9 @@ def __init__(self, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]'): self._writer: Optional['SSHWriter[bytes]'] = writer self._logger = reader.logger.get_child('sftp') + self.max_read_len = SAFE_SFTP_READ_LEN + self.max_write_len = SAFE_SFTP_WRITE_LEN + @property def logger(self) -> SSHLogger: """A logger associated with this SFTP handler""" @@ -2328,6 +2379,14 @@ def _log_extensions(self, extensions: Sequence[Tuple[bytes, bytes]]): self.logger.debug1(' %s%s%s', name, ': ' if data else '', data) + def _log_limits(self, limits: SFTPLimits) -> None: + """Log SFTP server limits""" + + self.logger.debug1(' Max packet len: %d', limits.max_packet_len) + self.logger.debug1(' Max read len: %d', limits.max_read_len) + self.logger.debug1(' Max write len: %d', limits.max_write_len) + self.logger.debug1(' Max open handles: %d', limits.max_open_handles) + async def _process_packet(self, pkttype: int, pktid: int, packet: SSHPacket) -> None: """Abstract method for processing SFTP packets""" @@ -2401,6 +2460,7 @@ def __init__(self, loop: asyncio.AbstractEventLoop, self._supports_hardlink = False self._supports_fsync = False self._supports_lsetstat = False + self._supports_limits = False @property def version(self) -> int: @@ -2619,6 +2679,8 @@ async def start(self) -> None: self._supports_fsync = True elif name == b'lsetstat@openssh.com' and data == b'1': self._supports_lsetstat = True + elif name == b'limits@openssh.com' and data == b'1': + self._supports_limits = True if version == 3: # Check if the server has a buggy SYMLINK implementation @@ -2632,6 +2694,25 @@ async def start(self) -> None: 'implementation') self._nonstandard_symlink = True + async def request_limits(self) -> None: + """Request SFTP server limits""" + + if self._supports_limits: + packet = cast(SSHPacket, await self._make_request( + b'limits@openssh.com')) + + limits = SFTPLimits.decode(packet, self._version) + packet.check_end() + + self.logger.debug1('Received server limits:') + self._log_limits(limits) + + if limits.max_read_len: + self.max_read_len = limits.max_read_len + + if limits.max_write_len: + self.max_write_len = limits.max_write_len + async def open(self, filename: bytes, pflags: int, attrs: SFTPAttrs) -> bytes: """Make an SFTP open request""" @@ -3029,10 +3110,14 @@ def __init__(self, handler: SFTPClientHandler, handle: bytes, self._appending = appending self._encoding = encoding self._errors = errors - self._block_size = block_size self._max_requests = max_requests self._offset = None if appending else 0 + self.read_len = \ + handler.max_read_len if block_size == -1 else block_size + self.write_len = \ + handler.max_write_len if block_size == -1 else block_size + async def __aenter__(self) -> Self: """Allow SFTPClientFile to be used as an async context manager""" @@ -3104,9 +3189,9 @@ async def read(self, size: int = -1, size = (await self._end()) - offset try: - if self._block_size and size > self._block_size: + if self.read_len and size > self.read_len: data = await _SFTPFileReader( - self._block_size, self._max_requests, self._handler, + self.read_len, self._max_requests, self._handler, self._handle, offset, size).run() else: data, _ = await self._handler.read(self._handle, @@ -3177,7 +3262,7 @@ async def read_parallel(self, size: int = -1, offset = 0 size = 0 - return _SFTPFileReader(self._block_size, self._max_requests, + return _SFTPFileReader(self.read_len, self._max_requests, self._handler, self._handle, offset, size).iter() @@ -3224,9 +3309,9 @@ async def write(self, data: AnyStr, offset: Optional[int] = None) -> int: datalen = len(data_bytes) - if self._block_size and datalen > self._block_size: + if self.write_len and datalen > self.write_len: await _SFTPFileWriter( - self._block_size, self._max_requests, self._handler, + self.write_len, self._max_requests, self._handler, self._handle, offset, data_bytes).run() else: await self._handler.write(self._handle, offset, data_bytes) @@ -3523,6 +3608,18 @@ def version(self) -> int: return self._handler.version + @property + def max_read_len(self) -> int: + """Maximum read length associated with this SFTP session""" + + return self._handler.max_read_len + + @property + def max_write_len(self) -> int: + """Maximum write length associated with this SFTP session""" + + return self._handler.max_write_len + @staticmethod def basename(path: bytes) -> bytes: """Return the final component of a POSIX-style path""" @@ -3687,6 +3784,9 @@ async def _begin_copy(self, srcfs: _SFTPFSProtocol, dstfs: _SFTPFSProtocol, error_handler: SFTPErrorHandler) -> None: """Begin a new file upload, download, or copy""" + if block_size == -1: + block_size = min(srcfs.max_read_len, dstfs.max_write_len) + if isinstance(srcpaths, (bytes, str, PurePath)): srcpaths = [srcpaths] elif not isinstance(srcpaths, list): @@ -3742,8 +3842,7 @@ async def _begin_copy(self, srcfs: _SFTPFSProtocol, dstfs: _SFTPFSProtocol, async def get(self, remotepaths: _SFTPPaths, localpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -3847,8 +3946,7 @@ async def get(self, remotepaths: _SFTPPaths, async def put(self, localpaths: _SFTPPaths, remotepath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -3952,8 +4050,7 @@ async def put(self, localpaths: _SFTPPaths, async def copy(self, srcpaths: _SFTPPaths, dstpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int =SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4057,8 +4154,7 @@ async def copy(self, srcpaths: _SFTPPaths, async def mget(self, remotepaths: _SFTPPaths, localpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4081,8 +4177,7 @@ async def mget(self, remotepaths: _SFTPPaths, async def mput(self, localpaths: _SFTPPaths, remotepath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4105,8 +4200,7 @@ async def mput(self, localpaths: _SFTPPaths, async def mcopy(self, srcpaths: _SFTPPaths, dstpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int =SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4366,7 +4460,7 @@ async def open(self, path: _SFTPPath, pflags_or_mode: Union[int, str] = FXF_READ, attrs: SFTPAttrs = SFTPAttrs(), encoding: Optional[str] = 'utf-8', errors: str = 'strict', - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS) -> SFTPClientFile: """Open a remote file @@ -4498,7 +4592,7 @@ async def open56(self, path: _SFTPPath, flags: int = FXF_OPEN_EXISTING, attrs: SFTPAttrs = SFTPAttrs(), encoding: Optional[str] = 'utf-8', errors: str = 'strict', - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS) -> SFTPClientFile: """Open a remote file using SFTP v5/v6 flags @@ -5473,7 +5567,8 @@ class SFTPServerHandler(SFTPHandler): (b'posix-rename@openssh.com', b'1'), (b'hardlink@openssh.com', b'1'), (b'fsync@openssh.com', b'1'), - (b'lsetstat@openssh.com', b'1')] + (b'lsetstat@openssh.com', b'1'), + (b'limits@openssh.com', b'1')] _attrib_extensions: List[bytes] = [] @@ -5576,6 +5671,10 @@ async def _process_packet(self, pkttype: int, pktid: int, response = (UInt32(len(names)) + b''.join(name.encode(self._version) for name in names) + end) + elif isinstance(result, SFTPLimits): + self.logger.debug1('Sending server limits:') + self._log_limits(result) + response = result.encode(self._version) else: attrs: _SupportsEncode @@ -6314,6 +6413,14 @@ async def _process_lsetstat(self, packet: SSHPacket) -> None: assert result is not None await result + async def _process_limits(self, packet: SSHPacket) -> SFTPLimits: + """Process an incoming SFTP fstatvfs request""" + + packet.check_end() + + return SFTPLimits(MAX_SFTP_PACKET_LEN, MAX_SFTP_READ_LEN, + MAX_SFTP_WRITE_LEN, 0) + _packet_handlers: Dict[Union[int, bytes], _SFTPPacketHandler] = { FXP_OPEN: _process_open, FXP_CLOSE: _process_close, @@ -6341,7 +6448,8 @@ async def _process_lsetstat(self, packet: SSHPacket) -> None: b'fstatvfs@openssh.com': _process_fstatvfs, b'hardlink@openssh.com': _process_openssh_link, b'fsync@openssh.com': _process_fsync, - b'lsetstat@openssh.com': _process_lsetstat + b'lsetstat@openssh.com': _process_lsetstat, + b'limits@openssh.com': _process_limits } async def run(self) -> None: @@ -6394,7 +6502,7 @@ async def run(self) -> None: UInt32(self._supported_attrib_mask) + \ UInt32(self._supported_open_flags) + \ UInt32(self._supported_access_mask) + \ - UInt32(_MAX_SFTP_READ_SIZE) + ext_names + \ + UInt32(MAX_SFTP_READ_LEN) + ext_names + \ attrib_ext_names extensions.append((b'supported', supported)) @@ -6405,7 +6513,7 @@ async def run(self) -> None: UInt32(self._supported_attrib_mask) + \ UInt32(self._supported_open_flags) + \ UInt32(self._supported_access_mask) + \ - UInt32(_MAX_SFTP_READ_SIZE) + \ + UInt32(MAX_SFTP_READ_LEN) + \ UInt16(self._supported_open_block_vector) + \ UInt16(self._supported_block_vector) + \ UInt32(len(self._attrib_extensions)) + \ @@ -7420,6 +7528,9 @@ async def close(self) -> None: class LocalFS: """An async wrapper around local filesystem access""" + max_read_len = MAX_SFTP_READ_LEN + max_write_len = MAX_SFTP_WRITE_LEN + @staticmethod def basename(path: bytes) -> bytes: """Return the final component of a local file path""" @@ -7668,6 +7779,8 @@ async def start_sftp_client(conn: 'SSHClientConnection', conn.create_task(handler.recv_packets(), handler.logger) + await handler.request_limits() + return SFTPClient(handler, path_encoding, path_errors) diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 884d594..1657e58 100644 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -68,7 +68,9 @@ from asyncssh import FX_OK, scp from asyncssh.packet import SSHPacket, String, UInt32 -from asyncssh.sftp import LocalFile, SFTPHandler, SFTPServerHandler + +from asyncssh.sftp import SAFE_SFTP_READ_LEN, SAFE_SFTP_WRITE_LEN +from asyncssh.sftp import LocalFile, SFTPHandler, SFTPLimits, SFTPServerHandler from .server import ServerTestCase from .util import asynctest @@ -292,17 +294,17 @@ class _IOErrorSFTPServer(SFTPServer): """Return an I/O error during file writing""" async def read(self, file_obj, offset, size): - """Return an error for reads past 64 KB in a file""" + """Return an error for reads past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: raise SFTPFailure('I/O error') else: return super().read(file_obj, offset, size) async def write(self, file_obj, offset, data): - """Return an error for writes past 64 KB in a file""" + """Return an error for writes past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: raise SFTPFailure('I/O error') else: super().write(file_obj, offset, data) @@ -2219,7 +2221,7 @@ async def _test_read_out_of_order(self, sftp): f = None try: - random_data = os.urandom(4*1024*1024) + random_data = os.urandom(12*1024*1024) self._create_file('file', random_data) async with sftp.open('file', 'rb') as f: @@ -3709,6 +3711,24 @@ async def _unsupported_extensions_v6(self, sftp): # pylint: disable=no-value-for-parameter _unsupported_extensions_v6(self) + @asynctest + async def test_zero_limits(self): + """Test sending a server limits response with zero read/write length""" + + async def _send_zero_read_write_len(self, packet): + """Send a server limits response with zero read/write length""" + + # pylint: disable=unused-argument + + return SFTPLimits(0, 0, 0, 0) + + with patch.dict('asyncssh.sftp.SFTPServerHandler._packet_handlers', + {b'limits@openssh.com': _send_zero_read_write_len}): + async with self.connect() as conn: + async with conn.start_sftp_client() as sftp: + self.assertEqual(sftp.max_read_len, SAFE_SFTP_READ_LEN) + self.assertEqual(sftp.max_write_len, SAFE_SFTP_WRITE_LEN) + def test_write_close(self): """Test session cleanup in the middle of a write request""" @@ -4103,6 +4123,7 @@ async def test_chroot_realpath_v6(self, sftp): with self.assertRaises(SFTPInvalidParameter): await sftp.realpath('.', check=99) + @sftp_test async def test_getcwd_and_chdir(self, sftp): """Test changing directory on an SFTP server with a changed root""" @@ -4284,7 +4305,7 @@ async def test_put_error(self, sftp): for method in ('get', 'put', 'copy'): with self.subTest(method=method): try: - self._create_file('src', 4*1024*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await getattr(sftp, method)('src', 'dst') @@ -4296,14 +4317,14 @@ async def test_read_error(self, sftp): """Test error when reading a file on an SFTP server""" try: - self._create_file('file', 4*1024*1024*'\0') + self._create_file('file', 8*1024*1024*'\0') async with sftp.open('file') as f: with self.assertRaises(SFTPFailure): - await f.read(4*1024*1024) + await f.read(8*1024*1024) with self.assertRaises(SFTPFailure): - async for _ in await f.read_parallel(4*1024*1024): + async for _ in await f.read_parallel(8*1024*1024): pass finally: remove('file') @@ -4315,7 +4336,7 @@ async def test_write_error(self, sftp): try: with self.assertRaises(SFTPFailure): async with sftp.open('file', 'w') as f: - await f.write(4*1024*1024*'\0') + await f.write(8*1024*1024*'\0') finally: remove('file') @@ -4338,10 +4359,10 @@ async def test_read(self, sftp): data = os.urandom(65536) self._create_file('file', data) - async with sftp.open('file', 'rb') as f: - result = await f.read(32768, 16384) + async with sftp.open('file', 'rb', block_size=16384) as f: + result = await f.read(65536, 16384) - self.assertEqual(result, data[16384:49152]) + self.assertEqual(result, data[16384:]) finally: remove('file') @@ -4350,7 +4371,7 @@ async def test_get(self, sftp): """Test getting a file from an SFTP server with a small block size""" try: - data = os.urandom(65536) + data = os.urandom(8*1024*1024) self._create_file('src', data) await sftp.get('src', 'dst') self._check_file('src', 'dst') @@ -4372,7 +4393,7 @@ async def test_get(self, sftp): """Test getting a file from an SFTP server truncated during the copy""" try: - self._create_file('src', 65536*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await sftp.get('src', 'dst') @@ -5041,15 +5062,15 @@ async def test_put_read_error(self): """Test read errors when putting a file over SCP""" async def _read_error(self, size, offset): - """Return an error for reads past 64 KB in a file""" + """Return an error for reads past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: raise OSError(errno.EIO, 'I/O error') else: return await orig_read(self, size, offset) try: - self._create_file('src', 128*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') orig_read = LocalFile.read @@ -5064,15 +5085,15 @@ async def test_put_read_early_eof(self): """Test getting early EOF when putting a file over SCP""" async def _read_early_eof(self, size, offset): - """Return an early EOF for reads past 64 KB in a file""" + """Return an early EOF for reads past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: return b'' else: return await orig_read(self, size, offset) try: - self._create_file('src', 128*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') orig_read = LocalFile.read @@ -5405,7 +5426,7 @@ async def test_put_error(self): """Test error when putting a file over SCP""" try: - self._create_file('src', 4*1024*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await scp('src', (self._scp_server, 'dst')) @@ -5417,7 +5438,7 @@ async def test_copy_error(self): """Test error when copying a file over SCP""" try: - self._create_file('src', 4*1024*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await scp((self._scp_server, 'src'),