From 7b625b80710e3fd1bf2fc509f0434cc1be483866 Mon Sep 17 00:00:00 2001 From: Josh Schneier Date: Sat, 26 Aug 2023 17:13:16 -0400 Subject: [PATCH] [s3] add setting to customize TransferConfig and deprecate AWS_S3_USE_THREADS (#1280) --- docs/backends/amazon-S3.rst | 6 ++++++ storages/backends/s3boto3.py | 17 ++++++++++++++--- tests/test_s3boto3.py | 33 ++++++++++++++++++++++++--------- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/docs/backends/amazon-S3.rst b/docs/backends/amazon-S3.rst index 17045f59..9775abf2 100644 --- a/docs/backends/amazon-S3.rst +++ b/docs/backends/amazon-S3.rst @@ -134,6 +134,11 @@ searches for them: support the legacy ``s3`` (also known as ``v2``) version. You can check to see if your region is one of them in the `S3 region list`_. +``AWS_S3_TRANSFER_CONFIG`` (optional, default is ``None``) + + Set this to customize the transfer config options such as disabling threads for ``gevent`` compatibility; + See the `Boto3 docs for TransferConfig` for more info. + .. note:: The signature versions are not backwards compatible so be careful about url endpoints if making this change @@ -143,6 +148,7 @@ searches for them: .. _S3 region list: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region .. _list of canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Boto3 docs for uploading files: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object +.. _Boto3 docs for TransferConfig: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig .. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/3.1/ref/contrib/staticfiles/#manifeststaticfilesstorage CloudFront diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index 03c18871..700af57d 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -3,6 +3,7 @@ import posixpath import tempfile import threading +import warnings from datetime import datetime from datetime import timedelta from tempfile import SpooledTemporaryFile @@ -145,7 +146,7 @@ def _get_file(self): ) if 'r' in self._mode: self._is_dirty = False - self.obj.download_fileobj(self._file, Config=self._storage._transfer_config) + self.obj.download_fileobj(self._file, Config=self._storage.transfer_config) self._file.seek(0) if self._storage.gzip and self.obj.content_encoding == 'gzip': self._file = self._decompress_file(mode=self._mode, file=self._file) @@ -269,7 +270,16 @@ def __init__(self, **settings): signature_version=self.signature_version, proxies=self.proxies, ) - self._transfer_config = TransferConfig(use_threads=self.use_threads) + + if self.use_threads is False: + warnings.warn( + "The AWS_S3_USE_THREADS setting is deprecated. Use " + "AWS_S3_TRANSFER_CONFIG to customize any of the " + "boto.s3.transfer.TransferConfig parameters.", DeprecationWarning + ) + + if self.transfer_config is None: + self.transfer_config = TransferConfig(use_threads=self.use_threads) def get_cloudfront_signer(self, key_id, key): return _cloud_front_signer_from_pem(key_id, key) @@ -330,6 +340,7 @@ def get_default_settings(self): 'max_memory_size': setting('AWS_S3_MAX_MEMORY_SIZE', 0), 'default_acl': setting('AWS_DEFAULT_ACL', None), 'use_threads': setting('AWS_S3_USE_THREADS', True), + 'transfer_config': setting('AWS_S3_TRANSFER_CONFIG', None), } def __getstate__(self): @@ -419,7 +430,7 @@ def _save(self, name, content): params['ContentEncoding'] = 'gzip' obj = self.bucket.Object(name) - obj.upload_fileobj(content, ExtraArgs=params, Config=self._transfer_config) + obj.upload_fileobj(content, ExtraArgs=params, Config=self.transfer_config) return cleaned_name def delete(self, name): diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py index 6e003292..43491c16 100644 --- a/tests/test_s3boto3.py +++ b/tests/test_s3boto3.py @@ -7,6 +7,7 @@ from unittest import skipIf from urllib.parse import urlparse +import boto3.s3.transfer from botocore.exceptions import ClientError from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -96,7 +97,7 @@ def test_storage_save(self): ExtraArgs={ 'ContentType': 'text/plain', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_storage_save_non_seekable(self): @@ -114,7 +115,7 @@ def test_storage_save_non_seekable(self): ExtraArgs={ 'ContentType': 'text/plain', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_storage_save_with_default_acl(self): @@ -134,7 +135,7 @@ def test_storage_save_with_default_acl(self): 'ContentType': 'text/plain', 'ACL': 'private', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_storage_object_parameters_not_overwritten_by_default(self): @@ -155,7 +156,7 @@ def test_storage_object_parameters_not_overwritten_by_default(self): 'ContentType': 'text/plain', 'ACL': 'private', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_content_type(self): @@ -174,7 +175,7 @@ def test_content_type(self): ExtraArgs={ 'ContentType': 'image/jpeg', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_storage_save_gzipped(self): @@ -191,7 +192,7 @@ def test_storage_save_gzipped(self): 'ContentType': 'application/octet-stream', 'ContentEncoding': 'gzip', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_storage_save_gzipped_non_seekable(self): @@ -208,7 +209,7 @@ def test_storage_save_gzipped_non_seekable(self): 'ContentType': 'application/octet-stream', 'ContentEncoding': 'gzip', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) def test_storage_save_gzip(self): @@ -226,7 +227,7 @@ def test_storage_save_gzip(self): 'ContentType': 'text/css', 'ContentEncoding': 'gzip', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) args, kwargs = obj.upload_fileobj.call_args content = args[0] @@ -254,7 +255,7 @@ def test_storage_save_gzip_twice(self): 'ContentType': 'text/css', 'ContentEncoding': 'gzip', }, - Config=self.storage._transfer_config + Config=self.storage.transfer_config ) args, kwargs = obj.upload_fileobj.call_args content = args[0] @@ -767,6 +768,20 @@ def test_override_init_argument(self): storage = s3boto3.S3Boto3Storage(location='foo2') self.assertEqual(storage.location, 'foo2') + def test_use_threads_false(self): + with override_settings(AWS_S3_USE_THREADS=False): + storage = s3boto3.S3Boto3Storage() + self.assertFalse(storage.transfer_config.use_threads) + + def test_transfer_config(self): + storage = s3boto3.S3Boto3Storage() + self.assertTrue(storage.transfer_config.use_threads) + + transfer_config = boto3.s3.transfer.TransferConfig(use_threads=False) + with override_settings(AWS_S3_TRANSFER_CONFIG=transfer_config): + storage = s3boto3.S3Boto3Storage() + self.assertFalse(storage.transfer_config.use_threads) + class S3StaticStorageTests(TestCase): def setUp(self):