diff --git a/.changes/0.8.2.json b/.changes/0.8.2.json new file mode 100644 index 00000000..c26b7f25 --- /dev/null +++ b/.changes/0.8.2.json @@ -0,0 +1,7 @@ +[ + { + "category": "Subscribers", + "description": "Added caching for Subscribers to improve throughput by up to 24% in high volume transfer", + "type": "bugfix" + } +] \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a6c5a36e..72168c52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ CHANGELOG ========= +0.8.2 +===== + +* bugfix:Subscribers: Added caching for Subscribers to improve throughput by up to 24% in high volume transfer + + 0.8.1 ===== diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py index 97bb46d7..fa251490 100644 --- a/s3transfer/__init__.py +++ b/s3transfer/__init__.py @@ -144,7 +144,7 @@ def __call__(self, bytes_amount): from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError __author__ = 'Amazon Web Services' -__version__ = '0.8.1' +__version__ = '0.8.2' class NullHandler(logging.Handler): diff --git a/s3transfer/crt.py b/s3transfer/crt.py index 24fa7976..367f1f6f 100644 --- a/s3transfer/crt.py +++ b/s3transfer/crt.py @@ -15,6 +15,7 @@ from io import BytesIO import awscrt.http +import awscrt.s3 import botocore.awsrequest import botocore.session from awscrt.auth import AwsCredentials, AwsCredentialsProvider @@ -25,13 +26,7 @@ EventLoopGroup, TlsContextOptions, ) -from awscrt.s3 import ( - S3Client, - S3RequestTlsMode, - S3RequestType, - S3ResponseError, - get_recommended_throughput_target_gbps, -) +from awscrt.s3 import S3Client, S3RequestTlsMode, S3RequestType from botocore import UNSIGNED from botocore.compat import urlsplit from botocore.config import Config @@ -124,7 +119,6 @@ def create_s3_crt_client( use. Specify this argument if you want to use a custom CA cert bundle instead of the default one on your system. """ - event_loop_group = EventLoopGroup(num_threads) host_resolver = DefaultHostResolver(event_loop_group) bootstrap = ClientBootstrap(event_loop_group, host_resolver) @@ -159,7 +153,7 @@ def create_s3_crt_client( def _get_crt_throughput_target_gbps(provided_throughput_target_bytes=None): if provided_throughput_target_bytes is None: - target_gbps = get_recommended_throughput_target_gbps() + target_gbps = awscrt.s3.get_recommended_throughput_target_gbps() logger.debug( 'Recommended CRT throughput target in gbps: %s', target_gbps ) @@ -544,7 +538,7 @@ def serialize_http_request(self, transfer_type, future): return crt_request def translate_crt_exception(self, exception): - if isinstance(exception, S3ResponseError): + if isinstance(exception, awscrt.s3.S3ResponseError): return self._translate_crt_s3_response_error(exception) else: return None diff --git a/s3transfer/subscribers.py b/s3transfer/subscribers.py index cf0dbaa0..473d5d94 100644 --- a/s3transfer/subscribers.py +++ b/s3transfer/subscribers.py @@ -10,6 +10,8 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from functools import lru_cache + from s3transfer.compat import accepts_kwargs from s3transfer.exceptions import InvalidSubscriberMethodError @@ -28,6 +30,7 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls) @classmethod + @lru_cache() def _validate_subscriber_methods(cls): for subscriber_type in cls.VALID_SUBSCRIBER_TYPES: subscriber_method = getattr(cls, 'on_' + subscriber_type) diff --git a/tests/unit/test_crt.py b/tests/unit/test_crt.py index da899289..b0bb1970 100644 --- a/tests/unit/test_crt.py +++ b/tests/unit/test_crt.py @@ -54,7 +54,7 @@ def mock_s3_crt_client(): @pytest.fixture def mock_get_recommended_throughput_target_gbps(): with mock.patch( - 's3transfer.crt.get_recommended_throughput_target_gbps' + 'awscrt.s3.get_recommended_throughput_target_gbps' ) as mock_get_target_gbps: yield mock_get_target_gbps