diff --git a/asyncssh/scp.py b/asyncssh/scp.py index cb5e1b2..4472305 100644 --- a/asyncssh/scp.py +++ b/asyncssh/scp.py @@ -42,7 +42,7 @@ from .sftp import SFTPAttrs, SFTPGlob, SFTPName, SFTPServer, SFTPServerFS from .sftp import SFTPFileProtocol, SFTPError, SFTPFailure, SFTPBadMessage from .sftp import SFTPConnectionLost, SFTPErrorHandler, SFTPProgressHandler -from .sftp import SFTP_BLOCK_SIZE, local_fs +from .sftp import local_fs if TYPE_CHECKING: @@ -57,6 +57,9 @@ _SCPConnPath = Union[Tuple[_SCPConn, _SCPPath], _SCPConn, _SCPPath] +_SCP_BLOCK_SIZE = 256*1024 # 256 KiB + + class _SCPFSProtocol(Protocol): """Protocol for accessing a filesystem during an SCP copy""" @@ -409,7 +412,7 @@ class _SCPSource(_SCPHandler): def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]', preserve: bool, recurse: bool, - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None, server: bool = False): super().__init__(reader, writer, error_handler, server) @@ -568,7 +571,7 @@ class _SCPSink(_SCPHandler): def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]', must_be_dir: bool, preserve: bool, - recurse: bool, block_size: int = SFTP_BLOCK_SIZE, + recurse: bool, block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None, server: bool = False): super().__init__(reader, writer, error_handler, server) @@ -736,7 +739,7 @@ def __init__(self, src_reader: 'SSHReader[bytes]', src_writer: 'SSHWriter[bytes]', dst_reader: 'SSHReader[bytes]', dst_writer: 'SSHWriter[bytes]', - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None): self._source = _SCPHandler(src_reader, src_writer) @@ -898,7 +901,7 @@ async def run(self) -> None: async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]], dstpath: _SCPConnPath = None, *, preserve: bool = False, - recurse: bool = False, block_size: int = SFTP_BLOCK_SIZE, + recurse: bool = False, block_size: int = _SCP_BLOCK_SIZE, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None, **kwargs) -> None: """Copy files using SCP @@ -955,7 +958,7 @@ async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]], SFTP instead. The block_size value controls the size of read and write operations - issued to copy the files. It defaults to 16 KB. + issued to copy the files. It defaults to 256 KB. If progress_handler is specified, it will be called after each block of a file is successfully copied. The arguments passed to diff --git a/asyncssh/sftp.py b/asyncssh/sftp.py index 3eb115c..48bb9c9 100644 --- a/asyncssh/sftp.py +++ b/asyncssh/sftp.py @@ -148,20 +148,24 @@ SFTPErrorHandler = Union[None, Literal[False], Callable[[Exception], None]] SFTPProgressHandler = Optional[Callable[[bytes, bytes, int, int], None]] +_T = TypeVar('_T') + MIN_SFTP_VERSION = 3 MAX_SFTP_VERSION = 6 -SFTP_BLOCK_SIZE = 16384 -_MAX_SFTP_READ_SIZE = 4*1024*1024 # 4 MiB +SAFE_SFTP_READ_LEN = 16*1024 # 16 KiB +SAFE_SFTP_WRITE_LEN = 16*1024 # 16 KiB + +MAX_SFTP_READ_LEN = 4*1024*1024 # 4 MiB +MAX_SFTP_WRITE_LEN = 4*1024*1024 # 4 MiB +MAX_SFTP_PACKET_LEN = MAX_SFTP_WRITE_LEN + 1024 _MAX_SFTP_REQUESTS = 128 _MAX_READDIR_NAMES = 128 _NSECS_IN_SEC = 1_000_000_000 -_T = TypeVar('_T') - _const_dict: Mapping[str, int] = constants.__dict__ @@ -229,6 +233,14 @@ async def close(self) -> None: class _SFTPFSProtocol(Protocol): """Protocol for accessing a filesystem via an SFTP server""" + @property + def max_read_len(self) -> int: + """Maximum read length associated with this SFTP session""" + + @property + def max_write_len(self) -> int: + """Maximum write length associated with this SFTP session""" + @staticmethod def basename(path: bytes) -> bytes: """Return the final component of a POSIX-style path""" @@ -666,7 +678,7 @@ async def iter(self) -> AsyncIterator[Tuple[int, _T]]: offset, size, count, result = task.result() yield offset, result - if count < size: + if count and count < size: self._pending.add(asyncio.ensure_future( self._start_task(offset+count, size-count))) except SFTPEOFError: @@ -795,20 +807,24 @@ async def run(self) -> None: if self._progress_handler and self._total_bytes == 0: self._progress_handler(self._srcpath, self._dstpath, 0, 0) - async for offset, datalen in self.iter(): - if not datalen: - exc = SFTPFailure('Unexpected EOF during file copy') + async for _, datalen in self.iter(): + if datalen: + self._bytes_copied += datalen - setattr(exc, 'filename', self._srcpath) - setattr(exc, 'offset', offset) + if self._progress_handler: + self._progress_handler(self._srcpath, self._dstpath, + self._bytes_copied, + self._total_bytes) + + if self._bytes_copied != self._total_bytes: + exc = SFTPFailure('Unexpected EOF during file copy') + + setattr(exc, 'filename', self._srcpath) + setattr(exc, 'offset', self._bytes_copied) + + raise exc - raise exc - if self._progress_handler: - self._bytes_copied += datalen - self._progress_handler(self._srcpath, self._dstpath, - self._bytes_copied, - self._total_bytes) finally: if self._src: # pragma: no branch await self._src.close() @@ -2030,6 +2046,37 @@ def decode(cls, packet: SSHPacket, sftp_version: int) -> 'SFTPName': return cls(filename, longname, attrs) +class SFTPLimits(Record): + """SFTP server limits""" + + max_packet_len: int + max_read_len: int + max_write_len: int + max_open_handles: int + + def encode(self, sftp_version: int) -> bytes: + """Encode SFTP server limits in an SSH packet""" + + # pylint: disable=unused-argument + + return (UInt64(self.max_packet_len) + UInt64(self.max_read_len) + + UInt64(self.max_write_len) + UInt64(self.max_open_handles)) + + @classmethod + def decode(cls, packet: SSHPacket, sftp_version: int) -> 'SFTPLimits': + """Decode bytes in an SSH packet as SFTP server limits""" + + # pylint: disable=unused-argument + + max_packet_len = packet.get_uint64() + max_read_len = packet.get_uint64() + max_write_len = packet.get_uint64() + max_open_handles = packet.get_uint64() + + return cls(max_packet_len, max_read_len, + max_write_len, max_open_handles) + + class SFTPGlob: """SFTP glob matcher""" @@ -2240,7 +2287,8 @@ class SFTPHandler(SSHPacketLogger): FXP_STAT: FXP_ATTRS, FXP_READLINK: FXP_NAME, b'statvfs@openssh.com': FXP_EXTENDED_REPLY, - b'fstatvfs@openssh.com': FXP_EXTENDED_REPLY + b'fstatvfs@openssh.com': FXP_EXTENDED_REPLY, + b'limits@openssh.com': FXP_EXTENDED_REPLY } def __init__(self, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]'): @@ -2248,6 +2296,9 @@ def __init__(self, reader: 'SSHReader[bytes]', writer: 'SSHWriter[bytes]'): self._writer: Optional['SSHWriter[bytes]'] = writer self._logger = reader.logger.get_child('sftp') + self.max_read_len = SAFE_SFTP_READ_LEN + self.max_write_len = SAFE_SFTP_WRITE_LEN + @property def logger(self) -> SSHLogger: """A logger associated with this SFTP handler""" @@ -2328,6 +2379,14 @@ def _log_extensions(self, extensions: Sequence[Tuple[bytes, bytes]]): self.logger.debug1(' %s%s%s', name, ': ' if data else '', data) + def _log_limits(self, limits: SFTPLimits) -> None: + """Log SFTP server limits""" + + self.logger.debug1(' Max packet len: %d', limits.max_packet_len) + self.logger.debug1(' Max read len: %d', limits.max_read_len) + self.logger.debug1(' Max write len: %d', limits.max_write_len) + self.logger.debug1(' Max open handles: %d', limits.max_open_handles) + async def _process_packet(self, pkttype: int, pktid: int, packet: SSHPacket) -> None: """Abstract method for processing SFTP packets""" @@ -2401,6 +2460,7 @@ def __init__(self, loop: asyncio.AbstractEventLoop, self._supports_hardlink = False self._supports_fsync = False self._supports_lsetstat = False + self._supports_limits = False @property def version(self) -> int: @@ -2619,6 +2679,8 @@ async def start(self) -> None: self._supports_fsync = True elif name == b'lsetstat@openssh.com' and data == b'1': self._supports_lsetstat = True + elif name == b'limits@openssh.com' and data == b'1': + self._supports_limits = True if version == 3: # Check if the server has a buggy SYMLINK implementation @@ -2632,6 +2694,25 @@ async def start(self) -> None: 'implementation') self._nonstandard_symlink = True + async def request_limits(self) -> None: + """Request SFTP server limits""" + + if self._supports_limits: + packet = cast(SSHPacket, await self._make_request( + b'limits@openssh.com')) + + limits = SFTPLimits.decode(packet, self._version) + packet.check_end() + + self.logger.debug1('Received server limits:') + self._log_limits(limits) + + if limits.max_read_len: + self.max_read_len = limits.max_read_len + + if limits.max_write_len: + self.max_write_len = limits.max_write_len + async def open(self, filename: bytes, pflags: int, attrs: SFTPAttrs) -> bytes: """Make an SFTP open request""" @@ -3029,10 +3110,14 @@ def __init__(self, handler: SFTPClientHandler, handle: bytes, self._appending = appending self._encoding = encoding self._errors = errors - self._block_size = block_size self._max_requests = max_requests self._offset = None if appending else 0 + self.read_len = \ + handler.max_read_len if block_size == -1 else block_size + self.write_len = \ + handler.max_write_len if block_size == -1 else block_size + async def __aenter__(self) -> Self: """Allow SFTPClientFile to be used as an async context manager""" @@ -3104,9 +3189,9 @@ async def read(self, size: int = -1, size = (await self._end()) - offset try: - if self._block_size and size > self._block_size: + if self.read_len and size > self.read_len: data = await _SFTPFileReader( - self._block_size, self._max_requests, self._handler, + self.read_len, self._max_requests, self._handler, self._handle, offset, size).run() else: data, _ = await self._handler.read(self._handle, @@ -3177,7 +3262,7 @@ async def read_parallel(self, size: int = -1, offset = 0 size = 0 - return _SFTPFileReader(self._block_size, self._max_requests, + return _SFTPFileReader(self.read_len, self._max_requests, self._handler, self._handle, offset, size).iter() @@ -3224,9 +3309,9 @@ async def write(self, data: AnyStr, offset: Optional[int] = None) -> int: datalen = len(data_bytes) - if self._block_size and datalen > self._block_size: + if self.write_len and datalen > self.write_len: await _SFTPFileWriter( - self._block_size, self._max_requests, self._handler, + self.write_len, self._max_requests, self._handler, self._handle, offset, data_bytes).run() else: await self._handler.write(self._handle, offset, data_bytes) @@ -3523,6 +3608,18 @@ def version(self) -> int: return self._handler.version + @property + def max_read_len(self) -> int: + """Maximum read length associated with this SFTP session""" + + return self._handler.max_read_len + + @property + def max_write_len(self) -> int: + """Maximum write length associated with this SFTP session""" + + return self._handler.max_write_len + @staticmethod def basename(path: bytes) -> bytes: """Return the final component of a POSIX-style path""" @@ -3687,6 +3784,9 @@ async def _begin_copy(self, srcfs: _SFTPFSProtocol, dstfs: _SFTPFSProtocol, error_handler: SFTPErrorHandler) -> None: """Begin a new file upload, download, or copy""" + if block_size == -1: + block_size = min(srcfs.max_read_len, dstfs.max_write_len) + if isinstance(srcpaths, (bytes, str, PurePath)): srcpaths = [srcpaths] elif not isinstance(srcpaths, list): @@ -3742,8 +3842,7 @@ async def _begin_copy(self, srcfs: _SFTPFSProtocol, dstfs: _SFTPFSProtocol, async def get(self, remotepaths: _SFTPPaths, localpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -3847,8 +3946,7 @@ async def get(self, remotepaths: _SFTPPaths, async def put(self, localpaths: _SFTPPaths, remotepath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -3952,8 +4050,7 @@ async def put(self, localpaths: _SFTPPaths, async def copy(self, srcpaths: _SFTPPaths, dstpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int =SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4057,8 +4154,7 @@ async def copy(self, srcpaths: _SFTPPaths, async def mget(self, remotepaths: _SFTPPaths, localpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4081,8 +4177,7 @@ async def mget(self, remotepaths: _SFTPPaths, async def mput(self, localpaths: _SFTPPaths, remotepath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int = SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4105,8 +4200,7 @@ async def mput(self, localpaths: _SFTPPaths, async def mcopy(self, srcpaths: _SFTPPaths, dstpath: Optional[_SFTPPath] = None, *, preserve: bool = False, recurse: bool = False, - follow_symlinks: bool = False, - block_size: int =SFTP_BLOCK_SIZE, + follow_symlinks: bool = False, block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS, progress_handler: SFTPProgressHandler = None, error_handler: SFTPErrorHandler = None) -> None: @@ -4366,7 +4460,7 @@ async def open(self, path: _SFTPPath, pflags_or_mode: Union[int, str] = FXF_READ, attrs: SFTPAttrs = SFTPAttrs(), encoding: Optional[str] = 'utf-8', errors: str = 'strict', - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS) -> SFTPClientFile: """Open a remote file @@ -4498,7 +4592,7 @@ async def open56(self, path: _SFTPPath, flags: int = FXF_OPEN_EXISTING, attrs: SFTPAttrs = SFTPAttrs(), encoding: Optional[str] = 'utf-8', errors: str = 'strict', - block_size: int = SFTP_BLOCK_SIZE, + block_size: int = -1, max_requests: int = _MAX_SFTP_REQUESTS) -> SFTPClientFile: """Open a remote file using SFTP v5/v6 flags @@ -5473,7 +5567,8 @@ class SFTPServerHandler(SFTPHandler): (b'posix-rename@openssh.com', b'1'), (b'hardlink@openssh.com', b'1'), (b'fsync@openssh.com', b'1'), - (b'lsetstat@openssh.com', b'1')] + (b'lsetstat@openssh.com', b'1'), + (b'limits@openssh.com', b'1')] _attrib_extensions: List[bytes] = [] @@ -5576,6 +5671,10 @@ async def _process_packet(self, pkttype: int, pktid: int, response = (UInt32(len(names)) + b''.join(name.encode(self._version) for name in names) + end) + elif isinstance(result, SFTPLimits): + self.logger.debug1('Sending server limits:') + self._log_limits(result) + response = result.encode(self._version) else: attrs: _SupportsEncode @@ -6314,6 +6413,14 @@ async def _process_lsetstat(self, packet: SSHPacket) -> None: assert result is not None await result + async def _process_limits(self, packet: SSHPacket) -> SFTPLimits: + """Process an incoming SFTP fstatvfs request""" + + packet.check_end() + + return SFTPLimits(MAX_SFTP_PACKET_LEN, MAX_SFTP_READ_LEN, + MAX_SFTP_WRITE_LEN, 0) + _packet_handlers: Dict[Union[int, bytes], _SFTPPacketHandler] = { FXP_OPEN: _process_open, FXP_CLOSE: _process_close, @@ -6341,7 +6448,8 @@ async def _process_lsetstat(self, packet: SSHPacket) -> None: b'fstatvfs@openssh.com': _process_fstatvfs, b'hardlink@openssh.com': _process_openssh_link, b'fsync@openssh.com': _process_fsync, - b'lsetstat@openssh.com': _process_lsetstat + b'lsetstat@openssh.com': _process_lsetstat, + b'limits@openssh.com': _process_limits } async def run(self) -> None: @@ -6394,7 +6502,7 @@ async def run(self) -> None: UInt32(self._supported_attrib_mask) + \ UInt32(self._supported_open_flags) + \ UInt32(self._supported_access_mask) + \ - UInt32(_MAX_SFTP_READ_SIZE) + ext_names + \ + UInt32(MAX_SFTP_READ_LEN) + ext_names + \ attrib_ext_names extensions.append((b'supported', supported)) @@ -6405,7 +6513,7 @@ async def run(self) -> None: UInt32(self._supported_attrib_mask) + \ UInt32(self._supported_open_flags) + \ UInt32(self._supported_access_mask) + \ - UInt32(_MAX_SFTP_READ_SIZE) + \ + UInt32(MAX_SFTP_READ_LEN) + \ UInt16(self._supported_open_block_vector) + \ UInt16(self._supported_block_vector) + \ UInt32(len(self._attrib_extensions)) + \ @@ -7420,6 +7528,9 @@ async def close(self) -> None: class LocalFS: """An async wrapper around local filesystem access""" + max_read_len = MAX_SFTP_READ_LEN + max_write_len = MAX_SFTP_WRITE_LEN + @staticmethod def basename(path: bytes) -> bytes: """Return the final component of a local file path""" @@ -7668,6 +7779,8 @@ async def start_sftp_client(conn: 'SSHClientConnection', conn.create_task(handler.recv_packets(), handler.logger) + await handler.request_limits() + return SFTPClient(handler, path_encoding, path_errors) diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 884d594..1657e58 100644 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -68,7 +68,9 @@ from asyncssh import FX_OK, scp from asyncssh.packet import SSHPacket, String, UInt32 -from asyncssh.sftp import LocalFile, SFTPHandler, SFTPServerHandler + +from asyncssh.sftp import SAFE_SFTP_READ_LEN, SAFE_SFTP_WRITE_LEN +from asyncssh.sftp import LocalFile, SFTPHandler, SFTPLimits, SFTPServerHandler from .server import ServerTestCase from .util import asynctest @@ -292,17 +294,17 @@ class _IOErrorSFTPServer(SFTPServer): """Return an I/O error during file writing""" async def read(self, file_obj, offset, size): - """Return an error for reads past 64 KB in a file""" + """Return an error for reads past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: raise SFTPFailure('I/O error') else: return super().read(file_obj, offset, size) async def write(self, file_obj, offset, data): - """Return an error for writes past 64 KB in a file""" + """Return an error for writes past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: raise SFTPFailure('I/O error') else: super().write(file_obj, offset, data) @@ -2219,7 +2221,7 @@ async def _test_read_out_of_order(self, sftp): f = None try: - random_data = os.urandom(4*1024*1024) + random_data = os.urandom(12*1024*1024) self._create_file('file', random_data) async with sftp.open('file', 'rb') as f: @@ -3709,6 +3711,24 @@ async def _unsupported_extensions_v6(self, sftp): # pylint: disable=no-value-for-parameter _unsupported_extensions_v6(self) + @asynctest + async def test_zero_limits(self): + """Test sending a server limits response with zero read/write length""" + + async def _send_zero_read_write_len(self, packet): + """Send a server limits response with zero read/write length""" + + # pylint: disable=unused-argument + + return SFTPLimits(0, 0, 0, 0) + + with patch.dict('asyncssh.sftp.SFTPServerHandler._packet_handlers', + {b'limits@openssh.com': _send_zero_read_write_len}): + async with self.connect() as conn: + async with conn.start_sftp_client() as sftp: + self.assertEqual(sftp.max_read_len, SAFE_SFTP_READ_LEN) + self.assertEqual(sftp.max_write_len, SAFE_SFTP_WRITE_LEN) + def test_write_close(self): """Test session cleanup in the middle of a write request""" @@ -4103,6 +4123,7 @@ async def test_chroot_realpath_v6(self, sftp): with self.assertRaises(SFTPInvalidParameter): await sftp.realpath('.', check=99) + @sftp_test async def test_getcwd_and_chdir(self, sftp): """Test changing directory on an SFTP server with a changed root""" @@ -4284,7 +4305,7 @@ async def test_put_error(self, sftp): for method in ('get', 'put', 'copy'): with self.subTest(method=method): try: - self._create_file('src', 4*1024*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await getattr(sftp, method)('src', 'dst') @@ -4296,14 +4317,14 @@ async def test_read_error(self, sftp): """Test error when reading a file on an SFTP server""" try: - self._create_file('file', 4*1024*1024*'\0') + self._create_file('file', 8*1024*1024*'\0') async with sftp.open('file') as f: with self.assertRaises(SFTPFailure): - await f.read(4*1024*1024) + await f.read(8*1024*1024) with self.assertRaises(SFTPFailure): - async for _ in await f.read_parallel(4*1024*1024): + async for _ in await f.read_parallel(8*1024*1024): pass finally: remove('file') @@ -4315,7 +4336,7 @@ async def test_write_error(self, sftp): try: with self.assertRaises(SFTPFailure): async with sftp.open('file', 'w') as f: - await f.write(4*1024*1024*'\0') + await f.write(8*1024*1024*'\0') finally: remove('file') @@ -4338,10 +4359,10 @@ async def test_read(self, sftp): data = os.urandom(65536) self._create_file('file', data) - async with sftp.open('file', 'rb') as f: - result = await f.read(32768, 16384) + async with sftp.open('file', 'rb', block_size=16384) as f: + result = await f.read(65536, 16384) - self.assertEqual(result, data[16384:49152]) + self.assertEqual(result, data[16384:]) finally: remove('file') @@ -4350,7 +4371,7 @@ async def test_get(self, sftp): """Test getting a file from an SFTP server with a small block size""" try: - data = os.urandom(65536) + data = os.urandom(8*1024*1024) self._create_file('src', data) await sftp.get('src', 'dst') self._check_file('src', 'dst') @@ -4372,7 +4393,7 @@ async def test_get(self, sftp): """Test getting a file from an SFTP server truncated during the copy""" try: - self._create_file('src', 65536*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await sftp.get('src', 'dst') @@ -5041,15 +5062,15 @@ async def test_put_read_error(self): """Test read errors when putting a file over SCP""" async def _read_error(self, size, offset): - """Return an error for reads past 64 KB in a file""" + """Return an error for reads past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: raise OSError(errno.EIO, 'I/O error') else: return await orig_read(self, size, offset) try: - self._create_file('src', 128*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') orig_read = LocalFile.read @@ -5064,15 +5085,15 @@ async def test_put_read_early_eof(self): """Test getting early EOF when putting a file over SCP""" async def _read_early_eof(self, size, offset): - """Return an early EOF for reads past 64 KB in a file""" + """Return an early EOF for reads past 4 MB in a file""" - if offset >= 65536: + if offset >= 4*1024*1024: return b'' else: return await orig_read(self, size, offset) try: - self._create_file('src', 128*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') orig_read = LocalFile.read @@ -5405,7 +5426,7 @@ async def test_put_error(self): """Test error when putting a file over SCP""" try: - self._create_file('src', 4*1024*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await scp('src', (self._scp_server, 'dst')) @@ -5417,7 +5438,7 @@ async def test_copy_error(self): """Test error when copying a file over SCP""" try: - self._create_file('src', 4*1024*1024*'\0') + self._create_file('src', 8*1024*1024*'\0') with self.assertRaises(SFTPFailure): await scp((self._scp_server, 'src'),