diff --git a/docs/backends/sftp.rst b/docs/backends/sftp.rst index 6f19f9f56..a754097d8 100644 --- a/docs/backends/sftp.rst +++ b/docs/backends/sftp.rst @@ -1,5 +1,68 @@ SFTP ==== -Take a look at the top of the backend's file for the documentation. +Settings +-------- +``SFTP_STORAGE_HOST`` + +The hostname where you want the files to be saved. + +``SFTP_STORAGE_ROOT`` + +The root directory on the remote host into which files should be placed. +Should work the same way that ``STATIC_ROOT`` works for local files. Must +include a trailing slash. + +``SFTP_STORAGE_PARAMS`` (Optional) + +A dictionary containing connection parameters to be passed as keyword +arguments to ``paramiko.SSHClient().connect()`` (do not include hostname here). +See `paramiko SSHClient.connect() documentation`_ for details + +.. _`paramiko SSHClient.connect() documentation`: http://docs.paramiko.org/en/latest/api/client.html#paramiko.client.SSHClient.connect + +``SFTP_STORAGE_INTERACTIVE`` (Optional) + +A boolean indicating whether to prompt for a password if the connection cannot +be made using keys, and there is not already a password in +``SFTP_STORAGE_PARAMS``. You can set this to ``True`` to enable interactive +login when running ``manage.py collectstatic``, for example. + +.. warning:: + + DO NOT set SFTP_STORAGE_INTERACTIVE to True if you are using this storage + for files being uploaded to your site by users, because you'll have no way + to enter the password when they submit the form.. + +``SFTP_STORAGE_FILE_MODE`` (Optional) + +A bitmask for setting permissions on newly-created files. See +`Python os.chmod documentation`_ for acceptable values. + + +``SFTP_STORAGE_DIR_MODE`` (Optional) + +A bitmask for setting permissions on newly-created directories. See +`Python os.chmod documentation`_ for acceptable values. + +.. note:: + + Hint: if you start the mode number with a 0 you can express it in octal + just like you would when doing "chmod 775 myfile" from bash. + +.. _`Python os.chmod documentation`: http://docs.python.org/library/os.html#os.chmod + +``SFTP_STORAGE_UID`` (Optional) + +UID of the account that should be set as owner of the files on the remote +host. You may have to be root to set this. + +``SFTP_STORAGE_GID`` (Optional) + +GID of the group that should be set on the files on the remote host. You have +to be a member of the group to set this. + +``SFTP_KNOWN_HOST_FILE`` (Optional) + +Absolute path of know host file, if it isn't set ``"~/.ssh/known_hosts"`` will be used. diff --git a/requirements-tests.txt b/requirements-tests.txt index 307b874a3..fba9c555b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -3,3 +3,4 @@ pytest-cov==2.2.1 boto>=2.32.0 dropbox>=3.24 mock +paramiko diff --git a/storages/backends/sftpstorage.py b/storages/backends/sftpstorage.py index f2fd4e561..f219b5950 100644 --- a/storages/backends/sftpstorage.py +++ b/storages/backends/sftpstorage.py @@ -4,49 +4,6 @@ # License: MIT # # Modeled on the FTP storage by Rafal Jonca -# -# Settings: -# -# SFTP_STORAGE_HOST - The hostname where you want the files to be saved. -# -# SFTP_STORAGE_ROOT - The root directory on the remote host into which files -# should be placed. Should work the same way that STATIC_ROOT works for local -# files. Must include a trailing slash. -# -# SFTP_STORAGE_PARAMS (Optional) - A dictionary containing connection -# parameters to be passed as keyword arguments to -# paramiko.SSHClient().connect() (do not include hostname here). See -# http://docs.paramiko.org/en/latest/api/client.html#paramiko.client.SSHClient.connect -# for details -# -# SFTP_STORAGE_INTERACTIVE (Optional) - A boolean indicating whether to prompt -# for a password if the connection cannot be made using keys, and there is not -# already a password in SFTP_STORAGE_PARAMS. You can set this to True to -# enable interactive login when running 'manage.py collectstatic', for example. -# -# DO NOT set SFTP_STORAGE_INTERACTIVE to True if you are using this storage -# for files being uploaded to your site by users, because you'll have no way -# to enter the password when they submit the form.. -# -# SFTP_STORAGE_FILE_MODE (Optional) - A bitmask for setting permissions on -# newly-created files. See http://docs.python.org/library/os.html#os.chmod for -# acceptable values. -# -# SFTP_STORAGE_DIR_MODE (Optional) - A bitmask for setting permissions on -# newly-created directories. See -# http://docs.python.org/library/os.html#os.chmod for acceptable values. -# -# Hint: if you start the mode number with a 0 you can express it in octal -# just like you would when doing "chmod 775 myfile" from bash. -# -# SFTP_STORAGE_UID (Optional) - uid of the account that should be set as owner -# of the files on the remote host. You have to be root to set this. -# -# SFTP_STORAGE_GID (Optional) - gid of the group that should be set on the -# files on the remote host. You have to be a member of the group to set this. -# SFTP_KNOWN_HOST_FILE (Optional) - absolute path of know host file, if it isn't -# set "~/.ssh/known_hosts" will be used - import getpass import os @@ -59,29 +16,32 @@ from django.core.files.base import File from storages.compat import urlparse, BytesIO, Storage +from storages.utils import setting class SFTPStorage(Storage): - def __init__(self): - self._host = settings.SFTP_STORAGE_HOST + def __init__(self, host, params=None, interactive=None, file_mode=None, + dir_mode=None, uid=None, gid=None, known_host_file=None, + root_path=None, base_url=None): + self._host = host or settings('SFTP_STORAGE_HOST') - # if present, settings.SFTP_STORAGE_PARAMS should be a dict with params - # matching the keyword arguments to paramiko.SSHClient().connect(). So - # you can put username/password there. Or you can omit all that if - # you're using keys. - self._params = getattr(settings, 'SFTP_STORAGE_PARAMS', {}) - self._interactive = getattr(settings, 'SFTP_STORAGE_INTERACTIVE', - False) - self._file_mode = getattr(settings, 'SFTP_STORAGE_FILE_MODE', None) - self._dir_mode = getattr(settings, 'SFTP_STORAGE_DIR_MODE', None) + self._params = params or setting('SFTP_STORAGE_PARAMS', {}) + self._interactive = setting('SFTP_STORAGE_INTERACTIVE', False) \ + if interactive is None else interactive + self._file_mode = setting('SFTP_STORAGE_FILE_MODE') \ + if file_mode is None else file_mode + self._dir_mode = setting('SFTP_STORAGE_DIR_MODE') if \ + dir_mode is None else dir_mode - self._uid = getattr(settings, 'SFTP_STORAGE_UID', None) - self._gid = getattr(settings, 'SFTP_STORAGE_GID', None) - self._known_host_file = getattr(settings, 'SFTP_KNOWN_HOST_FILE', None) + self._uid = setting('SFTP_STORAGE_UID') if uid is None else uid + self._gid = setting('SFTP_STORAGE_GID') if gid is None else gid + self._known_host_file = setting('SFTP_KNOWN_HOST_FILE') \ + if known_host_file is None else known_host_file - self._root_path = settings.SFTP_STORAGE_ROOT - self._base_url = settings.MEDIA_URL + self._root_path = setting('SFTP_STORAGE_ROOT', '') \ + if root_path is None else root_path + self._base_url = setting('MEDIA_URL') if base_url is None else base_url # for now it's all posix paths. Maybe someday we'll support figuring # out if the remote host is windows. @@ -263,5 +223,5 @@ def write(self, content): def close(self): if self._is_dirty: - self._storage._save(self._name, self.file.getvalue()) + self._storage._save(self._name, self) self.file.close() diff --git a/tests/test_sftp.py b/tests/test_sftp.py new file mode 100644 index 000000000..e31ef445e --- /dev/null +++ b/tests/test_sftp.py @@ -0,0 +1,146 @@ +import stat +from datetime import datetime +try: + from unittest.mock import patch, MagicMock +except ImportError: # Python 3.2 and below + from mock import patch, MagicMock +from django.test import TestCase +from django.core.files.base import File +from django.utils.six import BytesIO +from storages.backends import sftpstorage + + +class SFTPStorageTest(TestCase): + def setUp(self): + self.storage = sftpstorage.SFTPStorage('foo') + + def test_init(self): + pass + + @patch('paramiko.SSHClient') + def test_connect(self, mock_ssh): + self.storage._connect() + self.assertEqual('foo', mock_ssh.return_value.connect.call_args[0][0]) + + def test_open(self): + file_ = self.storage._open('foo') + self.assertIsInstance(file_, sftpstorage.SFTPStorageFile) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_read(self, mock_sftp): + file_ = self.storage._read('foo') + self.assertTrue(mock_sftp.open.called) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_chown(self, mock_sftp): + self.storage._chown('foo', 1, 1) + self.assertEqual(mock_sftp.chown.call_args[0], ('foo', 1, 1)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_mkdir(self, mock_sftp): + self.storage._mkdir('foo') + self.assertEqual(mock_sftp.mkdir.call_args[0], ('foo',)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.side_effect': (IOError(), True) + }) + def test_mkdir_parent(self, mock_sftp): + self.storage._mkdir('bar/foo') + self.assertEqual(mock_sftp.mkdir.call_args_list[0][0], ('bar',)) + self.assertEqual(mock_sftp.mkdir.call_args_list[1][0], ('bar/foo',)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_save(self, mock_sftp): + self.storage._save('foo', File(BytesIO(b'foo'), 'foo')) + self.assertTrue(mock_sftp.open.return_value.write.called) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.side_effect': (IOError(), True) + }) + def test_save_in_subdir(self, mock_sftp): + self.storage._save('bar/foo', File(BytesIO(b'foo'), 'foo')) + self.assertEqual(mock_sftp.mkdir.call_args_list[0][0], ('bar',)) + self.assertTrue(mock_sftp.open.return_value.write.called) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_delete(self, mock_sftp): + self.storage.delete('foo') + self.assertEqual(mock_sftp.remove.call_args_list[0][0], ('foo',)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_exists(self, mock_sftp): + self.assertTrue(self.storage.exists('foo')) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.side_effect': IOError() + }) + def test_not_exists(self, mock_sftp): + self.assertFalse(self.storage.exists('foo')) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'listdir_attr.return_value': + [MagicMock(filename='foo', st_mode=stat.S_IFDIR), + MagicMock(filename='bar', st_mode=None)]}) + def test_listdir(self, mock_sftp): + dirs, files = self.storage.listdir('/') + self.assertTrue(dirs) + self.assertTrue(files) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_size': 42, + }) + def test_size(self, mock_sftp): + self.assertEqual(self.storage.size('foo'), 42) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_atime': 1469674684.000000, + }) + def test_accessed_time(self, mock_sftp): + self.assertEqual(self.storage.accessed_time('foo'), + datetime(2016, 7, 27, 21, 58, 4)) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_mtime': 1469674684.000000, + }) + def test_modified_time(self, mock_sftp): + self.assertEqual(self.storage.modified_time('foo'), + datetime(2016, 7, 27, 21, 58, 4)) + + def test_url(self): + self.assertEqual(self.storage.url('foo'), '/media/foo') + # Test custom + self.storage._base_url = 'http://bar.pt/' + self.assertEqual(self.storage.url('foo'), 'http://bar.pt/foo') + # Test error + with self.assertRaises(ValueError): + self.storage._base_url = None + self.storage.url('foo') + + +class SFTPStorageFileTest(TestCase): + def setUp(self): + self.storage = sftpstorage.SFTPStorage('foo') + self.file = sftpstorage.SFTPStorageFile('bar', self.storage, 'wb') + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'stat.return_value.st_size': 42, + }) + def test_size(self, mock_sftp): + self.assertEqual(self.file.size, 42) + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{ + 'open.return_value.read.return_value': b'foo', + }) + def test_read(self, mock_sftp): + self.assertEqual(self.file.read(), b'foo') + self.assertTrue(mock_sftp.open.called) + + def test_write(self): + self.file.write(b'foo') + self.assertEqual(self.file.file.read(), b'foo') + + @patch('storages.backends.sftpstorage.SFTPStorage.sftp') + def test_close(self, mock_sftp): + self.file.write(b'foo') + self.file.close() + self.assertTrue(mock_sftp.open.return_value.write.called) diff --git a/tox.ini b/tox.ini index 9cadef9a4..2f0464084 100644 --- a/tox.ini +++ b/tox.ini @@ -18,3 +18,4 @@ deps = boto>=2.32.0 pytest-cov==2.2.1 dropbox>=3.24 + paramiko