From d5fc0e9db2380e9d004012d5fb5261e9e9a47479 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Sep 2019 21:57:06 +0800 Subject: [PATCH 1/6] set blocksize max 2G --- Lib/shutil.py | 1 + Lib/socket.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Lib/shutil.py b/Lib/shutil.py index 5c1255a6713de6..acf4010994a40e 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -136,6 +136,7 @@ def _fastcopy_sendfile(fsrc, fdst): # changes while being copied. try: blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MB + blocksize = min(blocksize, 2 ** 31 - 2) # max 2GB except Exception: blocksize = 2 ** 27 # 128MB diff --git a/Lib/socket.py b/Lib/socket.py index e5989d9dfd4094..cd89f4528dd9f5 100755 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -357,6 +357,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): if not fsize: return 0 # empty file blocksize = fsize if not count else count + blocksize = min(blocksize, 2 ** 31 - 2) # max 2GB timeout = self.gettimeout() if timeout == 0: From 5ea0ee10d31e3b5a02db164b0f327ba82dc7400e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Sep 2019 22:21:04 +0800 Subject: [PATCH 2/6] add news entry --- .../next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst diff --git a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst new file mode 100644 index 00000000000000..8abca8c8da72c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst @@ -0,0 +1,2 @@ +sendfile() (used by socket and shutil modules) was raising OverflowError for +files > 2GB on 32-bit systems. From 549e508c26776dd8193a2a64888ebacb11ef2208 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Sep 2019 22:29:20 +0800 Subject: [PATCH 3/6] use power of 2; catch OSError --- Lib/shutil.py | 4 ++-- Lib/socket.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index acf4010994a40e..16dc86f96bf4c1 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -136,8 +136,8 @@ def _fastcopy_sendfile(fsrc, fdst): # changes while being copied. try: blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MB - blocksize = min(blocksize, 2 ** 31 - 2) # max 2GB - except Exception: + blocksize = min(blocksize, 2 ** 30) # max 1GB + except OSError: blocksize = 2 ** 27 # 128MB offset = 0 diff --git a/Lib/socket.py b/Lib/socket.py index cd89f4528dd9f5..928aff25bc0717 100755 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -357,7 +357,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): if not fsize: return 0 # empty file blocksize = fsize if not count else count - blocksize = min(blocksize, 2 ** 31 - 2) # max 2GB + blocksize = min(blocksize, 2 ** 30) # max 1GB timeout = self.gettimeout() if timeout == 0: From 49e6684c79b548518e2e640091efa6a7415e7d68 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Sep 2019 22:37:12 +0800 Subject: [PATCH 4/6] use binary prefixes --- Lib/shutil.py | 4 ++-- Lib/socket.py | 3 +-- .../next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 16dc86f96bf4c1..91eee9363293a0 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -135,8 +135,8 @@ def _fastcopy_sendfile(fsrc, fdst): # should not make any difference, also in case the file content # changes while being copied. try: - blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MB - blocksize = min(blocksize, 2 ** 30) # max 1GB + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB + blocksize = min(blocksize, 2 ** 30) # max 1GiB except OSError: blocksize = 2 ** 27 # 128MB diff --git a/Lib/socket.py b/Lib/socket.py index 928aff25bc0717..7cf8e97de5e1e0 100755 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -356,8 +356,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): raise _GiveupOnSendfile(err) # not a regular file if not fsize: return 0 # empty file - blocksize = fsize if not count else count - blocksize = min(blocksize, 2 ** 30) # max 1GB + blocksize = min(count or fsize, 2 ** 30) # max 1GiB timeout = self.gettimeout() if timeout == 0: diff --git a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst index 8abca8c8da72c9..bba0e193a02141 100644 --- a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst +++ b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst @@ -1,2 +1,2 @@ sendfile() (used by socket and shutil modules) was raising OverflowError for -files > 2GB on 32-bit systems. +files >= 2GiB on 32-bit systems. From 222a267e5aa4f72f36be1b188f207e7f0803ee5d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Sep 2019 22:55:35 +0800 Subject: [PATCH 5/6] only limit blocksize to 1GiB for 32 archs --- Lib/shutil.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 91eee9363293a0..9e08c6be01c4fa 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -136,7 +136,8 @@ def _fastcopy_sendfile(fsrc, fdst): # changes while being copied. try: blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB - blocksize = min(blocksize, 2 ** 30) # max 1GiB + if sys.maxsize < 2 ** 32: # 32-bit architecture + blocksize = min(blocksize, 2 ** 30) # max 1GiB except OSError: blocksize = 2 ** 27 # 128MB From 66149969e1266db27c9e25698555c3a965d65893 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 1 Oct 2019 10:48:03 +0800 Subject: [PATCH 6/6] add comments --- Lib/shutil.py | 8 +++++--- Lib/socket.py | 4 ++-- .../next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 9e08c6be01c4fa..1e89256cc34443 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -136,10 +136,12 @@ def _fastcopy_sendfile(fsrc, fdst): # changes while being copied. try: blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB - if sys.maxsize < 2 ** 32: # 32-bit architecture - blocksize = min(blocksize, 2 ** 30) # max 1GiB except OSError: - blocksize = 2 ** 27 # 128MB + blocksize = 2 ** 27 # 128MiB + # On 32-bit architectures truncate to 1GiB to avoid OverflowError, + # see bpo-38319. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) offset = 0 while True: diff --git a/Lib/socket.py b/Lib/socket.py index 7cf8e97de5e1e0..84a5dcb0daf29a 100755 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -356,8 +356,8 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): raise _GiveupOnSendfile(err) # not a regular file if not fsize: return 0 # empty file - blocksize = min(count or fsize, 2 ** 30) # max 1GiB - + # Truncate to 1GiB to avoid OverflowError, see bpo-38319. + blocksize = min(count or fsize, 2 ** 30) timeout = self.gettimeout() if timeout == 0: raise ValueError("non-blocking sockets are not supported") diff --git a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst index bba0e193a02141..376a9e459de27c 100644 --- a/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst +++ b/Misc/NEWS.d/next/Library/2019-09-30-22-06-33.bpo-38319.5QjiDa.rst @@ -1,2 +1,2 @@ -sendfile() (used by socket and shutil modules) was raising OverflowError for -files >= 2GiB on 32-bit systems. +sendfile() used in socket and shutil modules was raising OverflowError for +files >= 2GiB on 32-bit architectures. (patch by Giampaolo Rodola)