From 8bb6b7eb1bd1cc00e72fc0bcbbdad2c5013e16a5 Mon Sep 17 00:00:00 2001 From: nitely Date: Mon, 30 Jul 2018 02:44:07 -0300 Subject: [PATCH] azure backend --- .travis.yml | 4 + docs/backends/azure.rst | 108 +++++- manage.py | 11 + setup.py | 2 +- storages/backends/azure_storage.py | 356 +++++++++++++++---- tests/integration/__init__.py | 1 + tests/integration/migrations/0001_initial.py | 21 ++ tests/integration/migrations/__init__.py | 0 tests/integration/models.py | 8 + tests/integration/settings.py | 29 ++ tests/integration/test_azure.py | 261 ++++++++++++++ tests/test_azure.py | 267 ++++++++++++++ tox.ini | 25 +- 13 files changed, 1007 insertions(+), 86 deletions(-) create mode 100644 manage.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/migrations/0001_initial.py create mode 100644 tests/integration/migrations/__init__.py create mode 100644 tests/integration/models.py create mode 100644 tests/integration/settings.py create mode 100644 tests/integration/test_azure.py create mode 100644 tests/test_azure.py diff --git a/.travis.yml b/.travis.yml index 5715781f0..065c68ff8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ matrix: fast_finish: true include: - env: TOXENV=flake8 + - python: 2.7 + env: TOXENV=integration + - python: 3.5 + env: TOXENV=integration - python: 2.7 env: TOXENV=py27-django111 - python: 3.4 diff --git a/docs/backends/azure.rst b/docs/backends/azure.rst index b9fa2734b..c84bcde86 100644 --- a/docs/backends/azure.rst +++ b/docs/backends/azure.rst @@ -3,37 +3,125 @@ Azure Storage A custom storage system for Django using Windows Azure Storage backend. -Before you start configuration, you will need to install the Azure SDK for Python. -Install the package:: +Notes +***** - pip install azure +Be aware Azure file names have some extra restrictions. They can't: -Add to your requirements file:: + - end with dot (``.``) or slash (``/``) + - contain more than 256 slashes (``/``) + - be longer than 1024 characters - pip freeze > requirements.txt +This is usually not an issue, since some file-systems won't +allow this anyway. +There's ``default_storage.get_name_max_len()`` method +to get the ``max_length`` allowed. This is useful +for form inputs. It usually returns +``1024 - len(azure_location_setting)``. +There's ``default_storage.get_valid_name(...)`` method +to clean up file names when migrating to Azure. + +Gzipping for static files must be done through Azure CDN. + + +Install +******* + +Install Azure SDK:: + + pip install django-storage[azure] + + +Private VS Public Access +************************ + +The ``AzureStorage`` allows a single container. The container may have either +public access or private access. When dealing with a private container, the +``AZURE_URL_EXPIRATION_SECS`` must be set to get temporary URLs. + +A common setup is having private media files and public static files, +since public files allow for better caching (i.e: no query-string within the URL). + +One way to support this is having two backends, a regular ``AzureStorage`` +with the private container and expiration setting set, and a custom +backend (i.e: a subclass of ``AzureStorage``) for the public container. + +Custom backend:: + + # file: ./custom_storage/custom_azure.py + class PublicAzureStorage(AzureStorage): + account_name = 'myaccount' + account_key = 'mykey' + azure_container = 'mypublic_container' + expiration_secs = None + +Then on settings set:: + + DEFAULT_FILE_STORAGE = 'storages.backends.azure_storage.AzureStorage' + STATICFILES_STORAGE = 'custom_storage.custom_azure.PublicAzureStorage' Settings ******** -To use `AzureStorage` set:: +The following settings should be set within the standard django +configuration file, usually `settings.py`. + +Set the default storage (i.e: for media files) and the static storage +(i.e: fo static files) to use the azure backend:: DEFAULT_FILE_STORAGE = 'storages.backends.azure_storage.AzureStorage' + STATICFILES_STORAGE = 'storages.backends.azure_storage.AzureStorage' The following settings are available: + is_emulated = setting('AZURE_EMULATED_MODE', False) + ``AZURE_ACCOUNT_NAME`` - This setting is the Windows Azure Storage Account name, which in many cases is also the first part of the url for instance: http://azure_account_name.blob.core.windows.net/ would mean:: + This setting is the Windows Azure Storage Account name, which in many cases + is also the first part of the url for instance: http://azure_account_name.blob.core.windows.net/ + would mean:: AZURE_ACCOUNT_NAME = "azure_account_name" ``AZURE_ACCOUNT_KEY`` - This is the private key that gives your Django app access to your Windows Azure Account. + This is the private key that gives Django access to the Windows Azure Account. ``AZURE_CONTAINER`` - This is where the files uploaded through your Django app will be uploaded. - The container must be already created as the storage system will not attempt to create it. + This is where the files uploaded through Django will be uploaded. + The container must be already created, since the storage system will not attempt to create it. + +``AZURE_SSL`` + + Set a secure connection (HTTPS), otherwise it's makes an insecure connection (HTTP). Default is ``True`` + +``AZURE_UPLOAD_MAX_CONN`` + + Number of connections to make when uploading a single file. Default is ``2`` + +``AZURE_CONNECTION_TIMEOUT_SECS`` + + Global connection timeout in seconds. Default is ``20`` + +``AZURE_BLOB_MAX_MEMORY_SIZE`` + + Maximum memory used by a downloaded file before dumping it to disk. Unit is in bytes. Default is ``2MB`` + +``AZURE_URL_EXPIRATION_SECS`` + + Seconds before a URL expires, set to ``None`` to never expire it. + Be aware the container must have public read permissions in order + to access a URL without expiration date. Default is ``None`` + +``AZURE_OVERWRITE_FILES`` + + Overwrite an existing file when it has the same name as the file being uploaded. + Otherwise, rename it. Default is ``False`` + +``AZURE_LOCATION`` + + Default location for the uploaded files. This is a path that gets prepended to every file name. diff --git a/manage.py b/manage.py new file mode 100644 index 000000000..ea3f17b79 --- /dev/null +++ b/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +# XXX we need manage.py until pytest-django is fixed +# https://github.com/pytest-dev/pytest-django/issues/639 + +import sys + +if __name__ == "__main__": + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/setup.py b/setup.py index a6ce4c075..b7ef4c28d 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def read(filename): python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", install_requires=['Django>=1.11'], extras_require={ - 'azure': ['azure'], + 'azure': ['azure>=3.0.0', 'azure-storage-blob>=1.3.1'], 'boto': ['boto>=2.32.0'], 'boto3': ['boto3>=1.2.3'], 'dropbox': ['dropbox>=7.2.1'], diff --git a/storages/backends/azure_storage.py b/storages/backends/azure_storage.py index ea5d71c3f..b29db8a66 100644 --- a/storages/backends/azure_storage.py +++ b/storages/backends/azure_storage.py @@ -1,120 +1,328 @@ +from __future__ import unicode_literals + import mimetypes -import os.path -import time -from datetime import datetime -from time import mktime +import re +from datetime import datetime, timedelta +from tempfile import SpooledTemporaryFile -from django.core.exceptions import ImproperlyConfigured -from django.core.files.base import ContentFile +from azure.common import AzureMissingResourceHttpError +from azure.storage.blob import BlobPermissions, ContentSettings +from azure.storage.common import CloudStorageAccount +from django.core.exceptions import SuspiciousOperation +from django.core.files.base import File from django.core.files.storage import Storage +from django.utils import timezone from django.utils.deconstruct import deconstructible +from django.utils.encoding import force_bytes, force_text + +from storages.utils import clean_name, safe_join, setting + + +@deconstructible +class AzureStorageFile(File): + + def __init__(self, name, mode, storage): + self.name = name + self._mode = mode + self._storage = storage + self._is_dirty = False + self._file = None + self._path = storage._get_valid_path(name) + + def _get_file(self): + if self._file is not None: + return self._file + + file = SpooledTemporaryFile( + max_size=self._storage.max_memory_size, + suffix=".AzureStorageFile", + dir=setting("FILE_UPLOAD_TEMP_DIR", None)) + + if 'r' in self._mode or 'a' in self._mode: + # I set max connection to 1 since spooledtempfile is + # not seekable which is required if we use max_connections > 1 + self._storage.service.get_blob_to_stream( + container_name=self._storage.azure_container, + blob_name=self._path, + stream=file, + max_connections=1, + timeout=10) + if 'r' in self._mode: + file.seek(0) + + self._file = file + return self._file -from storages.utils import setting + def _set_file(self, value): + self._file = value -try: - import azure # noqa -except ImportError: - raise ImproperlyConfigured( - "Could not load Azure bindings. " - "See https://github.com/WindowsAzure/azure-sdk-for-python") + file = property(_get_file, _set_file) -try: - # azure-storage 0.20.0 - from azure.storage.blob.blobservice import BlobService - from azure.common import AzureMissingResourceHttpError -except ImportError: - from azure.storage import BlobService - from azure import WindowsAzureMissingResourceError as AzureMissingResourceHttpError + def read(self, *args, **kwargs): + if 'r' not in self._mode and 'a' not in self._mode: + raise AttributeError("File was not opened in read mode.") + return super(AzureStorageFile, self).read(*args, **kwargs) + def write(self, content): + if len(content) > 100*1024*1024: + raise ValueError("Max chunk size is 100MB") + if ('w' not in self._mode and + '+' not in self._mode and + 'a' not in self._mode): + raise AttributeError("File was not opened in write mode.") + self._is_dirty = True + return super(AzureStorageFile, self).write(force_bytes(content)) -def clean_name(name): - return os.path.normpath(name).replace("\\", "/") + def close(self): + if self._file is None: + return + if self._is_dirty: + self._file.seek(0) + self._storage._save(self.name, self._file) + self._is_dirty = False + self._file.close() + self._file = None + + +def _content_type(content): + try: + return content.file.content_type + except AttributeError: + pass + try: + return content.content_type + except AttributeError: + pass + return None + + +def _get_valid_path(s): + # A blob name: + # * must not end with dot or slash + # * can contain any character + # * must escape URL reserved characters + # We allow a subset of this to avoid + # illegal file names. We must ensure it is idempotent. + s = force_text(s).strip().replace(' ', '_') + s = re.sub(r'(?u)[^-\w./]', '', s) + s = s.strip('./') + if len(s) > _AZURE_NAME_MAX_LEN: + raise ValueError( + "File name max len is %d" % _AZURE_NAME_MAX_LEN) + if not len(s): + raise ValueError( + "File name must contain one or more " + "printable characters") + if s.count('/') > 256: + raise ValueError( + "File name must not contain " + "more than 256 slashes") + return s + + +def _clean_name_dance(name): + # `get_valid_path` may return `foo/../bar` + name = name.replace('\\', '/') + return clean_name(_get_valid_path(clean_name(name))) + + +# Max len according to azure's docs +_AZURE_NAME_MAX_LEN = 1024 @deconstructible class AzureStorage(Storage): + account_name = setting("AZURE_ACCOUNT_NAME") account_key = setting("AZURE_ACCOUNT_KEY") azure_container = setting("AZURE_CONTAINER") - azure_ssl = setting("AZURE_SSL") + azure_ssl = setting("AZURE_SSL", True) + upload_max_conn = setting("AZURE_UPLOAD_MAX_CONN", 2) + timeout = setting('AZURE_CONNECTION_TIMEOUT_SECS', 20) + max_memory_size = setting('AZURE_BLOB_MAX_MEMORY_SIZE', 2*1024*1024) + expiration_secs = setting('AZURE_URL_EXPIRATION_SECS') + overwrite_files = setting('AZURE_OVERWRITE_FILES', False) + location = setting('AZURE_LOCATION', '') + default_content_type = 'application/octet-stream' + is_emulated = setting('AZURE_EMULATED_MODE', False) - def __init__(self, *args, **kwargs): - super(AzureStorage, self).__init__(*args, **kwargs) - self._connection = None + def __init__(self): + self._service = None @property - def connection(self): - if self._connection is None: - self._connection = BlobService( - self.account_name, self.account_key) - return self._connection + def service(self): + # This won't open a connection or anything, + # it's akin to a client + if self._service is None: + account = CloudStorageAccount( + self.account_name, + self.account_key, + is_emulated=self.is_emulated) + self._service = account.create_block_blob_service() + return self._service @property def azure_protocol(self): if self.azure_ssl: return 'https' - return 'http' if self.azure_ssl is not None else None + else: + return 'http' - def __get_blob_properties(self, name): + def _path(self, name): + name = _clean_name_dance(name) try: - return self.connection.get_blob_properties( - self.azure_container, - name - ) - except AzureMissingResourceHttpError: - return None + return safe_join(self.location, name) + except ValueError: + raise SuspiciousOperation("Attempted access to '%s' denied." % name) + + def _get_valid_path(self, name): + # Must be idempotent + return _get_valid_path(self._path(name)) def _open(self, name, mode="rb"): - contents = self.connection.get_blob(self.azure_container, name) - return ContentFile(contents) + return AzureStorageFile(name, mode, self) + + def get_valid_name(self, name): + return _clean_name_dance(name) + + def get_available_name(self, name, max_length=_AZURE_NAME_MAX_LEN): + """ + Returns a filename that's free on the target storage system, and + available for new content to be written to. + """ + if self.overwrite_files: + return self.get_valid_name(name) + return super(AzureStorage, self).get_available_name( + self.get_valid_name(name), max_length) def exists(self, name): - return self.__get_blob_properties(name) is not None + return self.service.exists( + self.azure_container, + self._get_valid_path(name), + timeout=self.timeout) def delete(self, name): try: - self.connection.delete_blob(self.azure_container, name) + self.service.delete_blob( + container_name=self.azure_container, + blob_name=self._get_valid_path(name), + timeout=self.timeout) except AzureMissingResourceHttpError: pass def size(self, name): - properties = self.connection.get_blob_properties( - self.azure_container, name) - return properties["content-length"] + properties = self.service.get_blob_properties( + self.azure_container, + self._get_valid_path(name), + timeout=self.timeout).properties + return properties.content_length def _save(self, name, content): - if hasattr(content.file, 'content_type'): - content_type = content.file.content_type - else: - content_type = mimetypes.guess_type(name)[0] + name_only = self.get_valid_name(name) + name = self._get_valid_path(name) + guessed_type, content_encoding = mimetypes.guess_type(name) + content_type = ( + _content_type(content) or + guessed_type or + self.default_content_type) - if hasattr(content, 'chunks'): - content_data = b''.join(chunk for chunk in content.chunks()) - else: - content_data = content.read() + # Unwrap django file (wrapped by parent's save call) + if isinstance(content, File): + content = content.file - self.connection.put_blob(self.azure_container, name, - content_data, "BlockBlob", - x_ms_blob_content_type=content_type) - return name + self.service.create_blob_from_stream( + container_name=self.azure_container, + blob_name=name, + stream=content, + content_settings=ContentSettings( + content_type=content_type, + content_encoding=content_encoding), + max_connections=self.upload_max_conn, + timeout=self.timeout) + return name_only - def url(self, name): - if hasattr(self.connection, 'make_blob_url'): - return self.connection.make_blob_url( - container_name=self.azure_container, - blob_name=name, - protocol=self.azure_protocol, - ) - else: - return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name) + def _expire_at(self, expire): + # azure expects time in UTC + return datetime.utcnow() + timedelta(seconds=expire) + + def url(self, name, expire=None): + name = self._get_valid_path(name) + + if expire is None: + expire = self.expiration_secs + + make_blob_url_kwargs = {} + if expire: + sas_token = self.service.generate_blob_shared_access_signature( + self.azure_container, name, BlobPermissions.READ, expiry=self._expire_at(expire)) + make_blob_url_kwargs['sas_token'] = sas_token + + if self.azure_protocol: + make_blob_url_kwargs['protocol'] = self.azure_protocol + return self.service.make_blob_url( + container_name=self.azure_container, + blob_name=name, + **make_blob_url_kwargs) + + def get_modified_time(self, name): + """ + Returns an (aware) datetime object containing the last modified time if + USE_TZ is True, otherwise returns a naive datetime in the local timezone. + """ + properties = self.service.get_blob_properties( + self.azure_container, + self._get_valid_path(name), + timeout=self.timeout).properties + if not setting('USE_TZ', False): + return timezone.make_naive(properties.last_modified) + + tz = timezone.get_current_timezone() + if timezone.is_naive(properties.last_modified): + return timezone.make_aware(properties.last_modified, tz) + + # `last_modified` is in UTC time_zone, we + # must convert it to settings time_zone + return properties.last_modified.astimezone(tz) def modified_time(self, name): - try: - modified = self.__get_blob_properties(name)['last-modified'] - except (TypeError, KeyError): - return super(AzureStorage, self).modified_time(name) + """Returns a naive datetime object containing the last modified time.""" + mtime = self.get_modified_time(name) + if timezone.is_naive(mtime): + return mtime + return timezone.make_naive(mtime) + + def list_all(self, path=''): + """Return all files for a given path""" + if path: + path = self._get_valid_path(path) + if path and not path.endswith('/'): + path += '/' + # XXX make generator, add start, end + return [ + blob.name + for blob in self.service.list_blobs( + self.azure_container, + prefix=path, + timeout=self.timeout)] - modified = time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z') - modified = datetime.fromtimestamp(mktime(modified)) + def listdir(self, path=''): + """ + Return directories and files for a given path. + Leave the path empty to list the root. + Order of dirs and files is undefined. + """ + files = [] + dirs = set() + for name in self.list_all(path): + n = name[len(path):] + if '/' in n: + dirs.add(n.split('/', 1)[0]) + else: + files.append(n) + return list(dirs), files - return modified + def get_name_max_len(self): + max_len = _AZURE_NAME_MAX_LEN - len(self._get_valid_path('foo')) - len('foo') + if not self.overwrite_files: + max_len -= len('_1234567') + return max_len diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/integration/migrations/0001_initial.py b/tests/integration/migrations/0001_initial.py new file mode 100644 index 000000000..40071c8f0 --- /dev/null +++ b/tests/integration/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 2.0.7 on 2018-08-22 08:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SimpleFileModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('foo_file', models.FileField(upload_to='foo_uploads/')), + ], + ), + ] diff --git a/tests/integration/migrations/__init__.py b/tests/integration/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/models.py b/tests/integration/models.py new file mode 100644 index 000000000..0a1f2266f --- /dev/null +++ b/tests/integration/models.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +from django.db import models + + +class SimpleFileModel(models.Model): + + foo_file = models.FileField(upload_to='foo_uploads/') diff --git a/tests/integration/settings.py b/tests/integration/settings.py new file mode 100644 index 000000000..d2ae387fd --- /dev/null +++ b/tests/integration/settings.py @@ -0,0 +1,29 @@ + +AZURE_EMULATED_MODE = True +AZURE_ACCOUNT_NAME = "XXX" +AZURE_ACCOUNT_KEY = "KXXX" +AZURE_CONTAINER = "test" + +SECRET_KEY = 'test' + +INSTALLED_APPS = ( + 'django.contrib.staticfiles', + 'storages', + 'tests.integration' +) + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': {}, + }, +] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:' + } +} diff --git a/tests/integration/test_azure.py b/tests/integration/test_azure.py new file mode 100644 index 000000000..edfc58e94 --- /dev/null +++ b/tests/integration/test_azure.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import io + +from django import forms +from django.core.files.storage import default_storage +from django.core.files.uploadedfile import SimpleUploadedFile +from django.template import Context, Template +from django.test import TestCase, override_settings +from django.utils import timezone +from tests.integration.models import SimpleFileModel + +from storages.backends import azure_storage + + +class AzureStorageTest(TestCase): + + def setUp(self, *args): + self.storage = azure_storage.AzureStorage() + self.storage.is_emulated = True + self.storage.account_name = "XXX" + self.storage.account_key = "KXXX" + self.storage.azure_container = "test" + self.storage.service.delete_container( + self.storage.azure_container, fail_not_exist=False) + self.storage.service.create_container( + self.storage.azure_container, public_access=False, fail_on_exist=False) + + def test_save(self): + expected_name = "some_blob_Ϊ.txt" + self.assertFalse(self.storage.exists(expected_name)) + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some blob Ϊ.txt', stream) + self.assertEqual(name, expected_name) + self.assertTrue(self.storage.exists(expected_name)) + + def test_delete(self): + self.storage.location = 'path' + expected_name = "some_blob_Ϊ.txt" + self.assertFalse(self.storage.exists(expected_name)) + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some blob Ϊ.txt', stream) + self.assertEqual(name, expected_name) + self.assertTrue(self.storage.exists(expected_name)) + self.storage.delete(expected_name) + self.assertFalse(self.storage.exists(expected_name)) + + def test_size(self): + self.storage.location = 'path' + expected_name = "some_path/some_blob_Ϊ.txt" + self.assertFalse(self.storage.exists(expected_name)) + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some path/some blob Ϊ.txt', stream) + self.assertEqual(name, expected_name) + self.assertTrue(self.storage.exists(expected_name)) + self.assertEqual(self.storage.size(expected_name), len(b'Im a stream')) + + def test_url(self): + self.assertTrue( + self.storage.url("my_file.txt").endswith("/test/my_file.txt")) + self.storage.expiration_secs = 360 + # has some query-string + self.assertTrue("/test/my_file.txt?" in self.storage.url("my_file.txt")) + + @override_settings(USE_TZ=True) + def test_get_modified_time_tz(self): + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some path/some blob Ϊ.txt', stream) + self.assertTrue(timezone.is_aware(self.storage.get_modified_time(name))) + + @override_settings(USE_TZ=False) + def test_get_modified_time_no_tz(self): + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some path/some blob Ϊ.txt', stream) + self.assertTrue(timezone.is_naive(self.storage.get_modified_time(name))) + + @override_settings(USE_TZ=True) + def test_modified_time_tz(self): + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some path/some blob Ϊ.txt', stream) + self.assertTrue(timezone.is_naive(self.storage.modified_time(name))) + + @override_settings(USE_TZ=False) + def test_modified_time_no_tz(self): + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('some path/some blob Ϊ.txt', stream) + self.assertTrue(timezone.is_naive(self.storage.modified_time(name))) + + def test_open_read(self): + self.storage.location = 'root' + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('path/some file.txt', stream) + fh = self.storage.open(name, 'r+b') + try: + self.assertEqual(fh.read(), b'Im a stream') + finally: + fh.close() + + stream = io.BytesIO() + self.storage.service.get_blob_to_stream( + container_name=self.storage.azure_container, + blob_name='root/path/some_file.txt', + stream=stream, + max_connections=1, + timeout=10) + stream.seek(0) + self.assertEqual(stream.read(), b'Im a stream') + + def test_open_write(self): + self.storage.location = 'root' + name = 'file.txt' + path = 'root/file.txt' + fh = self.storage.open(name, 'wb') + try: + fh.write(b'foo') + finally: + fh.close() + + stream = io.BytesIO() + self.storage.service.get_blob_to_stream( + container_name=self.storage.azure_container, + blob_name=path, + stream=stream, + max_connections=1, + timeout=10) + stream.seek(0) + self.assertEqual(stream.read(), b'foo') + + # Write again + + fh = self.storage.open(name, 'wb') + try: + fh.write(b'bar') + finally: + fh.close() + + stream = io.BytesIO() + self.storage.service.get_blob_to_stream( + container_name=self.storage.azure_container, + blob_name=path, + stream=stream, + max_connections=1, + timeout=10) + stream.seek(0) + self.assertEqual(stream.read(), b'bar') + + def test_open_read_write(self): + self.storage.location = 'root' + stream = io.BytesIO(b'Im a stream') + name = self.storage.save('file.txt', stream) + fh = self.storage.open(name, 'r+b') + try: + self.assertEqual(fh.read(), b'Im a stream') + fh.write(b' foo') + fh.seek(0) + self.assertEqual(fh.read(), b'Im a stream foo') + finally: + fh.close() + + stream = io.BytesIO() + self.storage.service.get_blob_to_stream( + container_name=self.storage.azure_container, + blob_name='root/file.txt', + stream=stream, + max_connections=1, + timeout=10) + stream.seek(0) + self.assertEqual(stream.read(), b'Im a stream foo') + + +class AzureStorageExpiry(azure_storage.AzureStorage): + + account_name = 'myaccount' + account_key = 'mykey' + azure_container = 'test_private' + expiration_secs = 360 + + +class FooFileForm(forms.Form): + + foo_file = forms.FileField() + + +class FooFileModelForm(forms.ModelForm): + class Meta: + model = SimpleFileModel + fields = ['foo_file'] + + +@override_settings( + DEFAULT_FILE_STORAGE='storages.backends.azure_storage.AzureStorage', + STATICFILES_STORAGE='storages.backends.azure_storage.AzureStorage') +class AzureStorageDjangoTest(TestCase): + + def setUp(self, *args): + default_storage.service.delete_container( + default_storage.azure_container, fail_not_exist=False) + default_storage.service.create_container( + default_storage.azure_container, public_access=False, fail_on_exist=False) + + def test_is_azure(self): + self.assertIsInstance(default_storage, azure_storage.AzureStorage) + + def test_template_static_file(self): + t = Template( + '{% load static from staticfiles %}' + '{% static "foo.txt" %}') + self.assertEqual( + t.render(Context({})).strip(), + "https://127.0.0.1:10000/devstoreaccount1/test/foo.txt") + + @override_settings( + DEFAULT_FILE_STORAGE='tests.integration.test_azure.AzureStorageExpiry') + def test_template_media_file(self): + t = Template('{{ file_url }}') + rendered = t.render(Context({ + 'file_url': default_storage.url('foo.txt')})).strip() + self.assertTrue( + "https://127.0.0.1:10000/devstoreaccount1/test_private/foo.txt?" in rendered) + self.assertTrue("&" in rendered) + + # check static files still work + t = Template( + '{% load static from staticfiles %}' + '{% static "foo.txt" %}') + self.assertEqual( + t.render(Context({})).strip(), + "https://127.0.0.1:10000/devstoreaccount1/test/foo.txt") + + def test_form(self): + files = { + 'foo_file': SimpleUploadedFile( + name='1234.pdf', + content=b'foo content', + content_type='application/pdf')} + form = FooFileForm(data={}, files=files) + self.assertTrue(form.is_valid()) + name = default_storage.save( + 'foo.pdf', form.cleaned_data['foo_file']) + self.assertEqual(name, 'foo.pdf') + self.assertTrue(default_storage.exists('foo.pdf')) + + def test_model_form(self): + files = { + 'foo_file': SimpleUploadedFile( + name='foo.pdf', + content=b'foo content', + content_type='application/pdf')} + form = FooFileModelForm(data={}, files=files) + self.assertTrue(form.is_valid()) + form.save() + self.assertTrue(default_storage.exists('foo_uploads/foo.pdf')) + + # check file content was saved + fh = default_storage.open('foo_uploads/foo.pdf', 'r+b') + try: + self.assertEqual(fh.read(), b'foo content') + finally: + fh.close() diff --git a/tests/test_azure.py b/tests/test_azure.py new file mode 100644 index 000000000..55e93813f --- /dev/null +++ b/tests/test_azure.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import datetime +from datetime import timedelta + +import pytz +from azure.storage.blob import Blob, BlobPermissions, BlobProperties +from django.core.files.base import ContentFile +from django.test import TestCase + +from storages.backends import azure_storage + +try: + from unittest import mock +except ImportError: # Python 3.2 and below + import mock + + +class AzureStorageTest(TestCase): + + def setUp(self, *args): + self.storage = azure_storage.AzureStorage() + self.storage._service = mock.MagicMock() + self.storage.overwrite_files = True + self.container_name = 'test' + self.storage.azure_container = self.container_name + + def test_get_valid_path(self): + self.assertEqual( + self.storage._get_valid_path("path/to/somewhere"), + "path/to/somewhere") + self.assertEqual( + self.storage._get_valid_path("path/to/../somewhere"), + "path/somewhere") + self.assertEqual( + self.storage._get_valid_path("path/to/../"), "path") + self.assertEqual( + self.storage._get_valid_path("path\\to\\..\\"), "path") + self.assertEqual( + self.storage._get_valid_path("path/name/"), "path/name") + self.assertEqual( + self.storage._get_valid_path("path\\to\\somewhere"), + "path/to/somewhere") + self.assertEqual( + self.storage._get_valid_path("some/$/path"), "some/path") + self.assertEqual( + self.storage._get_valid_path("/$/path"), "path") + self.assertEqual( + self.storage._get_valid_path("path/$/"), "path") + self.assertEqual( + self.storage._get_valid_path("path/$/$/$/path"), "path/path") + self.assertEqual( + self.storage._get_valid_path("some///path"), "some/path") + self.assertEqual( + self.storage._get_valid_path("some//path"), "some/path") + self.assertEqual( + self.storage._get_valid_path("some\\\\path"), "some/path") + self.assertEqual( + self.storage._get_valid_path("a" * 1024), "a" * 1024) + self.assertEqual( + self.storage._get_valid_path("a/a" * 256), "a/a" * 256) + self.assertRaises(ValueError, self.storage._get_valid_path, "") + self.assertRaises(ValueError, self.storage._get_valid_path, "/") + self.assertRaises(ValueError, self.storage._get_valid_path, "/../") + self.assertRaises(ValueError, self.storage._get_valid_path, "..") + self.assertRaises(ValueError, self.storage._get_valid_path, "///") + self.assertRaises(ValueError, self.storage._get_valid_path, "!!!") + self.assertRaises(ValueError, self.storage._get_valid_path, "a" * 1025) + self.assertRaises(ValueError, self.storage._get_valid_path, "a/a" * 257) + + def test_get_valid_path_idempotency(self): + self.assertEqual( + self.storage._get_valid_path("//$//a//$//"), "a") + self.assertEqual( + self.storage._get_valid_path( + self.storage._get_valid_path("//$//a//$//")), + self.storage._get_valid_path("//$//a//$//")) + self.assertEqual( + self.storage._get_valid_path("some path/some long name & then some.txt"), + "some_path/some_long_name__then_some.txt") + self.assertEqual( + self.storage._get_valid_path( + self.storage._get_valid_path("some path/some long name & then some.txt")), + self.storage._get_valid_path("some path/some long name & then some.txt")) + + def test_get_available_name(self): + self.storage.overwrite_files = False + self.storage._service.exists.side_effect = [True, False] + name = self.storage.get_available_name('foo.txt') + self.assertTrue(name.startswith('foo_')) + self.assertTrue(name.endswith('.txt')) + self.assertTrue(len(name) > len('foo.txt')) + self.assertEqual(self.storage._service.exists.call_count, 2) + + def test_get_available_name_first(self): + self.storage.overwrite_files = False + self.storage._service.exists.return_value = False + self.assertEqual( + self.storage.get_available_name('foo bar baz.txt'), + 'foo_bar_baz.txt') + self.assertEqual(self.storage._service.exists.call_count, 1) + + def test_get_available_name_max_len(self): + self.storage.overwrite_files = False + # if you wonder why this is, file-system + # storage will raise when file name is too long as well, + # the form should validate this + self.assertRaises(ValueError, self.storage.get_available_name, 'a' * 1025) + self.storage._service.exists.side_effect = [True, False] + name = self.storage.get_available_name('a' * 1000, max_length=100) # max_len == 1024 + self.assertEqual(len(name), 100) + self.assertTrue('_' in name) + self.assertEqual(self.storage._service.exists.call_count, 2) + + def test_get_available_invalid(self): + self.storage.overwrite_files = False + self.storage._service.exists.return_value = False + self.assertRaises(ValueError, self.storage.get_available_name, "") + self.assertRaises(ValueError, self.storage.get_available_name, "$$") + + def test_url(self): + self.storage._service.make_blob_url.return_value = 'ret_foo' + self.assertEqual(self.storage.url('some blob'), 'ret_foo') + self.storage._service.make_blob_url.assert_called_once_with( + container_name=self.container_name, + blob_name='some_blob', + protocol='https') + + def test_url_expire(self): + utc = pytz.timezone('UTC') + fixed_time = utc.localize(datetime.datetime(2016, 11, 6, 4)) + self.storage._service.generate_blob_shared_access_signature.return_value = 'foo_token' + self.storage._service.make_blob_url.return_value = 'ret_foo' + with mock.patch('storages.backends.azure_storage.datetime') as d_mocked: + d_mocked.utcnow.return_value = fixed_time + self.assertEqual(self.storage.url('some blob', 100), 'ret_foo') + self.storage._service.generate_blob_shared_access_signature.assert_called_once_with( + self.container_name, + 'some_blob', + BlobPermissions.READ, + expiry=fixed_time + timedelta(seconds=100)) + self.storage._service.make_blob_url.assert_called_once_with( + container_name=self.container_name, + blob_name='some_blob', + sas_token='foo_token', + protocol='https') + + # From boto3 + + def test_storage_save(self): + """ + Test saving a file + """ + name = 'test storage save.txt' + content = ContentFile('new content') + with mock.patch('storages.backends.azure_storage.ContentSettings') as c_mocked: + c_mocked.return_value = 'content_settings_foo' + self.assertEqual(self.storage.save(name, content), 'test_storage_save.txt') + self.storage._service.create_blob_from_stream.assert_called_once_with( + container_name=self.container_name, + blob_name='test_storage_save.txt', + stream=content.file, + content_settings='content_settings_foo', + max_connections=2, + timeout=20) + c_mocked.assert_called_once_with( + content_type='text/plain', + content_encoding=None) + + def test_storage_open_write(self): + """ + Test opening a file in write mode + """ + name = 'test_open_for_writïng.txt' + content = 'new content' + + file = self.storage.open(name, 'w') + file.write(content) + written_file = file.file + file.close() + self.storage._service.create_blob_from_stream.assert_called_once_with( + container_name=self.container_name, + blob_name=name, + stream=written_file, + content_settings=mock.ANY, + max_connections=2, + timeout=20) + + def test_storage_exists(self): + self.storage._service.exists.return_value = True + blob_name = "blob" + self.assertTrue(self.storage.exists(blob_name)) + self.storage._service.exists.assert_called_once_with( + self.container_name, blob_name, timeout=20) + + def test_delete_blob(self): + self.storage.delete("name") + self.storage._service.delete_blob.assert_called_once_with( + container_name=self.container_name, + blob_name="name", + timeout=20) + + def test_storage_listdir_base(self): + file_names = ["some/path/1.txt", "2.txt", "other/path/3.txt", "4.txt"] + + result = [] + for p in file_names: + obj = mock.MagicMock() + obj.name = p + result.append(obj) + self.storage._service.list_blobs.return_value = iter(result) + + dirs, files = self.storage.listdir("") + self.storage._service.list_blobs.assert_called_with( + self.container_name, prefix="", timeout=20) + + self.assertEqual(len(dirs), 2) + for directory in ["some", "other"]: + self.assertTrue( + directory in dirs, + """ "%s" not in directory list "%s".""" % (directory, dirs)) + + self.assertEqual(len(files), 2) + for filename in ["2.txt", "4.txt"]: + self.assertTrue( + filename in files, + """ "%s" not in file list "%s".""" % (filename, files)) + + def test_storage_listdir_subdir(self): + file_names = ["some/path/1.txt", "some/2.txt"] + + result = [] + for p in file_names: + obj = mock.MagicMock() + obj.name = p + result.append(obj) + self.storage._service.list_blobs.return_value = iter(result) + + dirs, files = self.storage.listdir("some/") + self.storage._service.list_blobs.assert_called_with( + self.container_name, prefix="some/", timeout=20) + + self.assertEqual(len(dirs), 1) + self.assertTrue( + 'path' in dirs, + """ "path" not in directory list "%s".""" % (dirs,)) + + self.assertEqual(len(files), 1) + self.assertTrue( + '2.txt' in files, + """ "2.txt" not in files list "%s".""" % (files,)) + + def test_size_of_file(self): + props = BlobProperties() + props.content_length = 12 + self.storage._service.get_blob_properties.return_value = Blob(props=props) + self.assertEqual(12, self.storage.size("name")) + + def test_last_modified_of_file(self): + props = BlobProperties() + accepted_time = datetime.datetime(2017, 5, 11, 8, 52, 4) + props.last_modified = accepted_time + self.storage._service.get_blob_properties.return_value = Blob(props=props) + time = self.storage.modified_time("name") + self.assertEqual(accepted_time, time) diff --git a/tox.ini b/tox.ini index 633e0ed8a..9257bebca 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = py35-django{111,20,21,master} py36-django{111,20,21,master} py37-django{20,21,master} + integration flake8 [testenv] @@ -12,7 +13,7 @@ setenv = DJANGO_SETTINGS_MODULE = tests.settings PYTHONWARNINGS = all PYTHONDONTWRITEBYTECODE = 1 -commands = py.test --cov=storages tests/ {posargs} +commands = py.test --cov=storages --ignore=tests/integration/ tests/ {posargs} deps = django111: Django>=1.11,<2.0 django20: Django>=2.0,<2.1 @@ -27,6 +28,28 @@ deps = dropbox google-cloud-storage paramiko + azure>=3.0.0 + azure-storage-blob>=1.3.1 + +[testenv:integration] +ignore_errors = True +whitelist_externals = + docker + py.test +commands = + docker pull arafato/azurite + docker run --name azure-storage -d -t -p 10000:10000 -p 10001:10001 -p 10002:10002 arafato/azurite + py.test tests/integration/ + docker stop azure-storage + docker rm azure-storage +setenv = + PYTHONDONTWRITEBYTECODE = 1 + DJANGO_SETTINGS_MODULE = tests.integration.settings +deps = + Django>=1.11, <1.12 + pytest-django + azure>=3.0.0 + azure-storage-blob>=1.3.1 [testenv:flake8] deps =