From 555651de9141e7df6f0473eed81bb9e122a93a9a Mon Sep 17 00:00:00 2001 From: Rohith Raj Srinivasan Date: Wed, 28 Feb 2024 16:44:57 -0800 Subject: [PATCH 1/2] added support for handling 206 response --- eta/core/web.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/eta/core/web.py b/eta/core/web.py index c5f78bc8..8de77777 100644 --- a/eta/core/web.py +++ b/eta/core/web.py @@ -18,6 +18,8 @@ # pragma pylint: enable=unused-wildcard-import # pragma pylint: enable=wildcard-import +# pylint: disable=E1121 + import io import logging import re @@ -150,6 +152,8 @@ def write(self, path, url, params=None): self._do_download(r, f) def _get_streaming_response(self, url, headers=None, params=None): + headers = headers or {} + r = self.sess.get( url, headers=headers, @@ -165,14 +169,37 @@ def _get_streaming_response(self, url, headers=None, params=None): def _do_download(self, r, f): size_bytes = _get_content_length(r) + total_downloaded_bytes = 0 size_bits = 8 * size_bytes if size_bytes is not None else None + with etau.ProgressBar( size_bits, use_bits=True, quiet=self.quiet ) as pb: for chunk in r.iter_content(chunk_size=self.chunk_size): f.write(chunk) + total_downloaded_bytes += len(chunk) pb.update(8 * len(chunk)) + while True: + remaining_bytes = size_bytes - total_downloaded_bytes + if remaining_bytes > 0: + logger.debug( + "Continuing download...Total downloaded bytes: %d, Remaining bytes: %d" + % (total_downloaded_bytes, remaining_bytes) + ) + r = self._get_streaming_response( + r.url, + headers={ + "Range": "bytes=%d-" % total_downloaded_bytes + }, + ) + for chunk in r.iter_content(chunk_size=self.chunk_size): + f.write(chunk) + total_downloaded_bytes += len(chunk) + pb.update(8 * len(chunk)) + else: + break + class WebSessionError(Exception): """Exception raised when there is a problem with a web session.""" From d56be35d105403f8a6435668a39dd606eb709a46 Mon Sep 17 00:00:00 2001 From: Rohith Raj Srinivasan Date: Fri, 1 Mar 2024 09:51:33 -0800 Subject: [PATCH 2/2] review comments addressed --- eta/core/web.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eta/core/web.py b/eta/core/web.py index 8de77777..a95caec4 100644 --- a/eta/core/web.py +++ b/eta/core/web.py @@ -152,7 +152,6 @@ def write(self, path, url, params=None): self._do_download(r, f) def _get_streaming_response(self, url, headers=None, params=None): - headers = headers or {} r = self.sess.get( url, @@ -180,7 +179,10 @@ def _do_download(self, r, f): total_downloaded_bytes += len(chunk) pb.update(8 * len(chunk)) - while True: + while size_bytes is not None and ( + "Accept-Ranges" in r.headers + and r.headers["Accept-Ranges"] is not None + ): remaining_bytes = size_bytes - total_downloaded_bytes if remaining_bytes > 0: logger.debug(