From 42152e8f60f1bba1747581b00de6862053dcfe7c Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 14 May 2020 14:27:09 +0200 Subject: [PATCH 1/7] Use flock for the feed lock file instead of just looking for existence. --- ospd_openvas/lock.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/ospd_openvas/lock.py b/ospd_openvas/lock.py index b6bedc1f..9aa23547 100644 --- a/ospd_openvas/lock.py +++ b/ospd_openvas/lock.py @@ -19,6 +19,7 @@ import logging import time +import fcntl from pathlib import Path @@ -29,9 +30,7 @@ class LockFile: def __init__(self, path: Path): self._lock_file_path = path self._has_lock = False - - def is_locked(self) -> bool: - return self._lock_file_path.exists() + self._fd = None def has_lock(self) -> bool: return self._has_lock @@ -39,7 +38,7 @@ def has_lock(self) -> bool: def acquire_lock(self) -> "LockFile": """ Acquite a lock by creating a lock file. """ - if self.has_lock() or self.is_locked(): + if self.has_lock(): return self try: @@ -47,17 +46,25 @@ def acquire_lock(self) -> "LockFile": parent_dir = self._lock_file_path.parent parent_dir.mkdir(parents=True, exist_ok=True) - self._lock_file_path.touch(exist_ok=False) - self._has_lock = True - - logger.debug("Created lock file %s.", str(self._lock_file_path)) - - except FileExistsError as e: + self._fd = open(self._lock_file_path, 'w') + except PermissionError as e: logger.error( "Failed to create lock file %s. %s", str(self._lock_file_path), e, ) + return self + + # Try to adquire the lock. + try: + fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + self._has_lock = True + logger.debug("Created lock file %s.", str(self._lock_file_path)) + except BlockingIOError as e: + logger.error( + "Failed to lock the file %s. %s", str(self._lock_file_path), e, + ) + self._fd.close() return self @@ -71,10 +78,13 @@ def wait_for_lock(self): def release_lock(self) -> None: """ Release the lock by deleting the lock file """ - if self.has_lock() and self.is_locked(): - self._lock_file_path.unlink() + if self.has_lock() and self._fd: + fcntl.flock(self._fd, fcntl.LOCK_UN) + self._fd.close() self._has_lock = False - logger.debug("Removed lock file %s.", str(self._lock_file_path)) + logger.debug( + "Removed lock from file %s.", str(self._lock_file_path) + ) def __enter__(self): self.acquire_lock() From e3d7eacc1537f0fd931a0b93d752e68411d5fc3f Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 14 May 2020 14:28:26 +0200 Subject: [PATCH 2/7] Adjuste tests --- tests/test_lock.py | 55 +++++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/tests/test_lock.py b/tests/test_lock.py index 4aada65f..aef9da37 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -19,9 +19,11 @@ import unittest import shutil import tempfile +import fcntl from pathlib import Path +from unittest.mock import patch, MagicMock from ospd_openvas.lock import LockFile @@ -32,17 +34,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(str(self.temp_dir)) - def test_is_locked(self): - lock_file_path = self.temp_dir / 'test.lock' - - lock_file = LockFile(lock_file_path) - - self.assertFalse(lock_file.is_locked()) - - lock_file_path.touch() - - self.assertTrue(lock_file.is_locked()) - def test_acquire_lock(self): lock_file_path = self.temp_dir / 'test.lock' @@ -50,21 +41,23 @@ def test_acquire_lock(self): lock_file.acquire_lock() self.assertTrue(lock_file.has_lock()) - self.assertTrue(lock_file.is_locked()) - self.assertTrue(lock_file_path.exists()) + lock_file.release_lock() - def test_already_locked(self): + @patch('ospd_openvas.lock.logger') + def test_already_locked(self, mock_logger): lock_file_path = self.temp_dir / 'test.lock' - lock_file_path.touch() + + lock_file_aux = LockFile(lock_file_path) + lock_file_aux.acquire_lock() + self.assertTrue(lock_file_aux.has_lock()) lock_file = LockFile(lock_file_path) lock_file.acquire_lock() - self.assertFalse(lock_file.has_lock()) - self.assertTrue(lock_file.is_locked()) + mock_logger.error.assert_called_once() - self.assertTrue(lock_file_path.exists()) + lock_file_aux.release_lock() def test_create_parent_dirs(self): lock_file_path = self.temp_dir / 'foo' / 'bar' / 'test.lock' @@ -73,37 +66,33 @@ def test_create_parent_dirs(self): lock_file.acquire_lock() self.assertTrue(lock_file.has_lock()) - self.assertTrue(lock_file.is_locked()) self.assertTrue(lock_file_path.exists()) self.assertTrue(lock_file_path.parent.is_dir()) self.assertTrue(lock_file_path.parent.parent.is_dir()) - def test_context_manager(self): - lock_file_path = self.temp_dir / 'test.lock' + lock_file.release_lock() + @patch('ospd_openvas.lock.logger') + def test_create_paren_dirs_fail(self, mock_logger): + lock_file_path = Path('/root/lock/file/test.lock') lock_file = LockFile(lock_file_path) - with lock_file: - self.assertTrue(lock_file.is_locked()) - self.assertTrue(lock_file.has_lock()) - self.assertTrue(lock_file_path.is_file()) - - self.assertFalse(lock_file.is_locked()) + lock_file.acquire_lock() self.assertFalse(lock_file.has_lock()) - self.assertFalse(lock_file_path.is_file()) - def test_context_manager_lock_exists(self): + mock_logger.error.assert_called_once() + + def test_context_manager(self): lock_file_path = self.temp_dir / 'test.lock' - lock_file_path.touch() lock_file = LockFile(lock_file_path) with lock_file: - self.assertTrue(lock_file.is_locked()) + self.assertTrue(lock_file.has_lock()) self.assertTrue(lock_file_path.is_file()) - self.assertFalse(lock_file.has_lock()) + lock_file.release_lock() - self.assertTrue(lock_file.is_locked()) + # The file is not removed self.assertFalse(lock_file.has_lock()) self.assertTrue(lock_file_path.is_file()) From 0eba8c1f0d9418fa4b7c47163cfc3738102295bf Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 14 May 2020 14:31:43 +0200 Subject: [PATCH 3/7] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fdd1329..1868aed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Don't send host details and log messages to the client when Boreas is enabled. [#252](https://github.com/greenbone/ospd-openvas/pull/252) - Progress bar calculation do not takes in accound dead hosts. [#252](https://github.com/greenbone/ospd-openvas/pull/252) - Host progress is stored as integer. [#256](https://github.com/greenbone/ospd-openvas/pull/256) +- Use flock for the feed lock file. [#257](https://github.com/greenbone/ospd-openvas/pull/257) ### Fixed - Check vt_aux for None before trying to access it. [#177](https://github.com/greenbone/ospd-openvas/pull/177) From 47b907d30f861c48b2ed91d856695aa9ac0dc7d0 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 14 May 2020 16:02:51 +0200 Subject: [PATCH 4/7] Update lock file --- poetry.lock | 131 ++++++++++++++++++++++++++-------------------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8adc313b..7f79071d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5,15 +5,15 @@ marker = "python_version >= \"3.6\" and python_version < \"4.0\"" name = "appdirs" optional = false python-versions = "*" -version = "1.4.3" +version = "1.4.4" [[package]] category = "dev" description = "An abstract syntax tree for Python with inference support." name = "astroid" optional = false -python-versions = ">=3.5.*" -version = "2.4.0" +python-versions = ">=3.5" +version = "2.4.1" [package.dependencies] lazy-object-proxy = ">=1.4.0,<1.5.0" @@ -131,12 +131,12 @@ marker = "python_version >= \"3.6\" and python_version < \"4.0\"" name = "click" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "7.1.1" +version = "7.1.2" [[package]] category = "dev" description = "Cross-platform colored terminal text." -marker = "platform_system == \"Windows\" or sys_platform == \"win32\" or python_version >= \"3.6\" and python_version < \"4.0\" and platform_system == \"Windows\"" +marker = "python_version >= \"3.6\" and python_version < \"4.0\" and platform_system == \"Windows\" or sys_platform == \"win32\"" name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -145,6 +145,7 @@ version = "0.4.3" [[package]] category = "dev" description = "Terminal string styling done right, in Python." +marker = "python_version >= \"3.6\" and python_version < \"4.0\"" name = "colorful" optional = false python-versions = "*" @@ -159,7 +160,7 @@ description = "cryptography is a package which provides cryptographic recipes an name = "cryptography" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" -version = "2.9" +version = "2.9.2" [package.dependencies] cffi = ">=1.8,<1.11.3 || >1.11.3" @@ -186,7 +187,7 @@ description = "Python @deprecated decorator to deprecate old python classes, fun name = "deprecated" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.2.9" +version = "1.2.10" [package.dependencies] wrapt = ">=1.10,<2" @@ -244,20 +245,19 @@ description = "" name = "ospd" optional = false python-versions = "^3.5" -version = "20.4a1" +version = "20.8a1" [package.dependencies] defusedxml = "^0.6.0" -deprecated = "^1.2.9" +deprecated = "^1.2.10" lxml = "^4.5.0" paramiko = "^2.7.1" psutil = "^5.7.0" [package.source] -reference = "d13c9674cc472228536966e62a28bb55c4490cf6" +reference = "aab0be3ffd09ce46b2aa0d25ace773d331d1cbd9" type = "git" url = "https://github.com/greenbone/ospd.git" - [[package]] category = "main" description = "Core utilities for Python packages" @@ -386,7 +386,7 @@ marker = "python_version >= \"3.6\" and python_version < \"4.0\"" name = "regex" optional = false python-versions = "*" -version = "2020.4.4" +version = "2020.5.14" [[package]] category = "dev" @@ -413,7 +413,7 @@ description = "Python Library for Tom's Obvious, Minimal Language" name = "toml" optional = false python-versions = "*" -version = "0.10.0" +version = "0.10.1" [[package]] category = "dev" @@ -438,7 +438,7 @@ description = "Module for decorators, wrappers and monkey patching." name = "wrapt" optional = false python-versions = "*" -version = "1.11.2" +version = "1.12.1" [metadata] content-hash = "947b260be490cc2794ce4e32c6802e921c8709a3da1c4ce86483fa5658db789e" @@ -446,12 +446,12 @@ python-versions = "^3.5" [metadata.files] appdirs = [ - {file = "appdirs-1.4.3-py2.py3-none-any.whl", hash = "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"}, - {file = "appdirs-1.4.3.tar.gz", hash = "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92"}, + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] astroid = [ - {file = "astroid-2.4.0-py3-none-any.whl", hash = "sha256:2fecea42b20abb1922ed65c7b5be27edfba97211b04b2b6abc6a43549a024ea6"}, - {file = "astroid-2.4.0.tar.gz", hash = "sha256:29fa5d46a2404d01c834fcb802a3943685f1fc538eb2a02a161349f5505ac196"}, + {file = "astroid-2.4.1-py3-none-any.whl", hash = "sha256:d8506842a3faf734b81599c8b98dcc423de863adcc1999248480b18bd31a0f38"}, + {file = "astroid-2.4.1.tar.gz", hash = "sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1"}, ] attrs = [ {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, @@ -524,8 +524,8 @@ cffi = [ {file = "cffi-1.14.0.tar.gz", hash = "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6"}, ] click = [ - {file = "click-7.1.1-py2.py3-none-any.whl", hash = "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"}, - {file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"}, + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, @@ -536,33 +536,33 @@ colorful = [ {file = "colorful-0.5.4.tar.gz", hash = "sha256:86848ad4e2eda60cd2519d8698945d22f6f6551e23e95f3f14dfbb60997807ea"}, ] cryptography = [ - {file = "cryptography-2.9-cp27-cp27m-macosx_10_9_intel.whl", hash = "sha256:ef9a55013676907df6c9d7dd943eb1770d014f68beaa7e73250fb43c759f4585"}, - {file = "cryptography-2.9-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:2a2ad24d43398d89f92209289f15265107928f22a8d10385f70def7a698d6a02"}, - {file = "cryptography-2.9-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:95e1296e0157361fe2f5f0ed307fd31f94b0ca13372e3673fa95095a627636a1"}, - {file = "cryptography-2.9-cp27-cp27m-win32.whl", hash = "sha256:192ca04a36852a994ef21df13cca4d822adbbdc9d5009c0f96f1d2929e375d4f"}, - {file = "cryptography-2.9-cp27-cp27m-win_amd64.whl", hash = "sha256:ed1d0760c7e46436ec90834d6f10477ff09475c692ed1695329d324b2c5cd547"}, - {file = "cryptography-2.9-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:19ae795137682a9778892fb4390c07811828b173741bce91e30f899424b3934d"}, - {file = "cryptography-2.9-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d1bf5a1a0d60c7f9a78e448adcb99aa101f3f9588b16708044638881be15d6bc"}, - {file = "cryptography-2.9-cp35-abi3-macosx_10_9_intel.whl", hash = "sha256:1b9b535d6b55936a79dbe4990b64bb16048f48747c76c29713fea8c50eca2acf"}, - {file = "cryptography-2.9-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:7a279f33a081d436e90e91d1a7c338553c04e464de1c9302311a5e7e4b746088"}, - {file = "cryptography-2.9-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:49870684da168b90110bbaf86140d4681032c5e6a2461adc7afdd93be5634216"}, - {file = "cryptography-2.9-cp35-cp35m-win32.whl", hash = "sha256:6b91cab3841b4c7cb70e4db1697c69f036c8bc0a253edc0baa6783154f1301e4"}, - {file = "cryptography-2.9-cp35-cp35m-win_amd64.whl", hash = "sha256:587f98ce27ac4547177a0c6fe0986b8736058daffe9160dcf5f1bd411b7fbaa1"}, - {file = "cryptography-2.9-cp36-cp36m-win32.whl", hash = "sha256:cc20316e3f5a6b582fc3b029d8dc03aabeb645acfcb7fc1d9848841a33265748"}, - {file = "cryptography-2.9-cp36-cp36m-win_amd64.whl", hash = "sha256:3be7a5722d5bfe69894d3f7bbed15547b17619f3a88a318aab2e37f457524164"}, - {file = "cryptography-2.9-cp37-cp37m-win32.whl", hash = "sha256:7598974f6879a338c785c513e7c5a4329fbc58b9f6b9a6305035fca5b1076552"}, - {file = "cryptography-2.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5aca6f00b2f42546b9bdf11a69f248d1881212ce5b9e2618b04935b87f6f82a1"}, - {file = "cryptography-2.9-cp38-cp38-win32.whl", hash = "sha256:9fc9da390e98cb6975eadf251b6e5fa088820141061bf041cd5c72deba1dc526"}, - {file = "cryptography-2.9-cp38-cp38-win_amd64.whl", hash = "sha256:6b744039b55988519cc183149cceb573189b3e46e16ccf6f8c46798bb767c9dc"}, - {file = "cryptography-2.9.tar.gz", hash = "sha256:0cacd3ef5c604b8e5f59bf2582c076c98a37fe206b31430d0cd08138aff0986e"}, + {file = "cryptography-2.9.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:daf54a4b07d67ad437ff239c8a4080cfd1cc7213df57d33c97de7b4738048d5e"}, + {file = "cryptography-2.9.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:3b3eba865ea2754738616f87292b7f29448aec342a7c720956f8083d252bf28b"}, + {file = "cryptography-2.9.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:c447cf087cf2dbddc1add6987bbe2f767ed5317adb2d08af940db517dd704365"}, + {file = "cryptography-2.9.2-cp27-cp27m-win32.whl", hash = "sha256:f118a95c7480f5be0df8afeb9a11bd199aa20afab7a96bcf20409b411a3a85f0"}, + {file = "cryptography-2.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:c4fd17d92e9d55b84707f4fd09992081ba872d1a0c610c109c18e062e06a2e55"}, + {file = "cryptography-2.9.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d0d5aeaedd29be304848f1c5059074a740fa9f6f26b84c5b63e8b29e73dfc270"}, + {file = "cryptography-2.9.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:1e4014639d3d73fbc5ceff206049c5a9a849cefd106a49fa7aaaa25cc0ce35cf"}, + {file = "cryptography-2.9.2-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:96c080ae7118c10fcbe6229ab43eb8b090fccd31a09ef55f83f690d1ef619a1d"}, + {file = "cryptography-2.9.2-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:e993468c859d084d5579e2ebee101de8f5a27ce8e2159959b6673b418fd8c785"}, + {file = "cryptography-2.9.2-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:88c881dd5a147e08d1bdcf2315c04972381d026cdb803325c03fe2b4a8ed858b"}, + {file = "cryptography-2.9.2-cp35-cp35m-win32.whl", hash = "sha256:651448cd2e3a6bc2bb76c3663785133c40d5e1a8c1a9c5429e4354201c6024ae"}, + {file = "cryptography-2.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:726086c17f94747cedbee6efa77e99ae170caebeb1116353c6cf0ab67ea6829b"}, + {file = "cryptography-2.9.2-cp36-cp36m-win32.whl", hash = "sha256:091d31c42f444c6f519485ed528d8b451d1a0c7bf30e8ca583a0cac44b8a0df6"}, + {file = "cryptography-2.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:bb1f0281887d89617b4c68e8db9a2c42b9efebf2702a3c5bf70599421a8623e3"}, + {file = "cryptography-2.9.2-cp37-cp37m-win32.whl", hash = "sha256:18452582a3c85b96014b45686af264563e3e5d99d226589f057ace56196ec78b"}, + {file = "cryptography-2.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:22e91636a51170df0ae4dcbd250d318fd28c9f491c4e50b625a49964b24fe46e"}, + {file = "cryptography-2.9.2-cp38-cp38-win32.whl", hash = "sha256:844a76bc04472e5135b909da6aed84360f522ff5dfa47f93e3dd2a0b84a89fa0"}, + {file = "cryptography-2.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:1dfa985f62b137909496e7fc182dac687206d8d089dd03eaeb28ae16eec8e7d5"}, + {file = "cryptography-2.9.2.tar.gz", hash = "sha256:a0c30272fb4ddda5f5ffc1089d7405b7a71b0b0f51993cb4e5dbb4590b2fc229"}, ] defusedxml = [ {file = "defusedxml-0.6.0-py2.py3-none-any.whl", hash = "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93"}, {file = "defusedxml-0.6.0.tar.gz", hash = "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5"}, ] deprecated = [ - {file = "Deprecated-1.2.9-py2.py3-none-any.whl", hash = "sha256:55b41a15bda04c6a2c0d27dd4c2b7b81ffa6348c9cad8f077ac1978c59927ab9"}, - {file = "Deprecated-1.2.9.tar.gz", hash = "sha256:0cf37d293a96805c6afd8b5fc525cb40f23a2cac9b2d066ac3bd4b04e72ceccc"}, + {file = "Deprecated-1.2.10-py2.py3-none-any.whl", hash = "sha256:a766c1dccb30c5f6eb2b203f87edd1d8588847709c78589e1521d769addc8218"}, + {file = "Deprecated-1.2.10.tar.gz", hash = "sha256:525ba66fb5f90b07169fdd48b6373c18f1ee12728ca277ca44567a367d9d7f74"}, ] isort = [ {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"}, @@ -694,27 +694,27 @@ redis = [ {file = "redis-3.5.1.tar.gz", hash = "sha256:6e9d2722a95d10ddf854596e66516d316d99c6a483e5db3b35c34e1158b2bfa1"}, ] regex = [ - {file = "regex-2020.4.4-cp27-cp27m-win32.whl", hash = "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f"}, - {file = "regex-2020.4.4-cp27-cp27m-win_amd64.whl", hash = "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3"}, - {file = "regex-2020.4.4-cp36-cp36m-win32.whl", hash = "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8"}, - {file = "regex-2020.4.4-cp36-cp36m-win_amd64.whl", hash = "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948"}, - {file = "regex-2020.4.4-cp37-cp37m-win32.whl", hash = "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e"}, - {file = "regex-2020.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89"}, - {file = "regex-2020.4.4-cp38-cp38-win32.whl", hash = "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3"}, - {file = "regex-2020.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3"}, - {file = "regex-2020.4.4.tar.gz", hash = "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142"}, + {file = "regex-2020.5.14-cp27-cp27m-win32.whl", hash = "sha256:e565569fc28e3ba3e475ec344d87ed3cd8ba2d575335359749298a0899fe122e"}, + {file = "regex-2020.5.14-cp27-cp27m-win_amd64.whl", hash = "sha256:d466967ac8e45244b9dfe302bbe5e3337f8dc4dec8d7d10f5e950d83b140d33a"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:27ff7325b297fb6e5ebb70d10437592433601c423f5acf86e5bc1ee2919b9561"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ea55b80eb0d1c3f1d8d784264a6764f931e172480a2f1868f2536444c5f01e01"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:c9bce6e006fbe771a02bda468ec40ffccbf954803b470a0345ad39c603402577"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:d881c2e657c51d89f02ae4c21d9adbef76b8325fe4d5cf0e9ad62f850f3a98fd"}, + {file = "regex-2020.5.14-cp36-cp36m-win32.whl", hash = "sha256:99568f00f7bf820c620f01721485cad230f3fb28f57d8fbf4a7967ec2e446994"}, + {file = "regex-2020.5.14-cp36-cp36m-win_amd64.whl", hash = "sha256:70c14743320a68c5dac7fc5a0f685be63bc2024b062fe2aaccc4acc3d01b14a1"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a7c37f048ec3920783abab99f8f4036561a174f1314302ccfa4e9ad31cb00eb4"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:89d76ce33d3266173f5be80bd4efcbd5196cafc34100fdab814f9b228dee0fa4"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:51f17abbe973c7673a61863516bdc9c0ef467407a940f39501e786a07406699c"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ce5cc53aa9fbbf6712e92c7cf268274eaff30f6bd12a0754e8133d85a8fb0f5f"}, + {file = "regex-2020.5.14-cp37-cp37m-win32.whl", hash = "sha256:8044d1c085d49673aadb3d7dc20ef5cb5b030c7a4fa253a593dda2eab3059929"}, + {file = "regex-2020.5.14-cp37-cp37m-win_amd64.whl", hash = "sha256:c2062c7d470751b648f1cacc3f54460aebfc261285f14bc6da49c6943bd48bdd"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux1_i686.whl", hash = "sha256:329ba35d711e3428db6b45a53b1b13a0a8ba07cbbcf10bbed291a7da45f106c3"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:579ea215c81d18da550b62ff97ee187b99f1b135fd894a13451e00986a080cad"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:3a9394197664e35566242686d84dfd264c07b20f93514e2e09d3c2b3ffdf78fe"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ce367d21f33e23a84fb83a641b3834dd7dd8e9318ad8ff677fbfae5915a239f7"}, + {file = "regex-2020.5.14-cp38-cp38-win32.whl", hash = "sha256:1386e75c9d1574f6aa2e4eb5355374c8e55f9aac97e224a8a5a6abded0f9c927"}, + {file = "regex-2020.5.14-cp38-cp38-win_amd64.whl", hash = "sha256:7e61be8a2900897803c293247ef87366d5df86bf701083b6c43119c7c6c99108"}, + {file = "regex-2020.5.14.tar.gz", hash = "sha256:ce450ffbfec93821ab1fea94779a8440e10cf63819be6e176eb1973a6017aff5"}, ] rope = [ {file = "rope-0.17.0.tar.gz", hash = "sha256:658ad6705f43dcf3d6df379da9486529cf30e02d9ea14c5682aa80eb33b649e1"}, @@ -724,9 +724,8 @@ six = [ {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, ] toml = [ - {file = "toml-0.10.0-py2.7.egg", hash = "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3"}, - {file = "toml-0.10.0-py2.py3-none-any.whl", hash = "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"}, - {file = "toml-0.10.0.tar.gz", hash = "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c"}, + {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, + {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, ] tomlkit = [ {file = "tomlkit-0.5.11-py2.py3-none-any.whl", hash = "sha256:4e1bd6c9197d984528f9ff0cc9db667c317d8881288db50db20eeeb0f6b0380b"}, @@ -756,5 +755,5 @@ typed-ast = [ {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] wrapt = [ - {file = "wrapt-1.11.2.tar.gz", hash = "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"}, + {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, ] From de5860e269d29ca7c26d6dbdb0e53ab3bbd01bf3 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 14 May 2020 16:19:05 +0200 Subject: [PATCH 5/7] Add support for python 3.5 Use Path.open() instead of open() Update test to use the mocked assert_called_once() --- ospd_openvas/lock.py | 2 +- tests/test_lock.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ospd_openvas/lock.py b/ospd_openvas/lock.py index 9aa23547..a11a5fdb 100644 --- a/ospd_openvas/lock.py +++ b/ospd_openvas/lock.py @@ -46,7 +46,7 @@ def acquire_lock(self) -> "LockFile": parent_dir = self._lock_file_path.parent parent_dir.mkdir(parents=True, exist_ok=True) - self._fd = open(self._lock_file_path, 'w') + self._fd = self._lock_file_path.open('w') except PermissionError as e: logger.error( "Failed to create lock file %s. %s", diff --git a/tests/test_lock.py b/tests/test_lock.py index aef9da37..3a4d101f 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -25,6 +25,7 @@ from unittest.mock import patch, MagicMock from ospd_openvas.lock import LockFile +from .helper import assert_called_once class LockFileTestCase(unittest.TestCase): @@ -55,7 +56,7 @@ def test_already_locked(self, mock_logger): lock_file = LockFile(lock_file_path) lock_file.acquire_lock() self.assertFalse(lock_file.has_lock()) - mock_logger.error.assert_called_once() + assert_called_once(mock_logger.error) lock_file_aux.release_lock() @@ -81,7 +82,7 @@ def test_create_paren_dirs_fail(self, mock_logger): lock_file.acquire_lock() self.assertFalse(lock_file.has_lock()) - mock_logger.error.assert_called_once() + assert_called_once(mock_logger.error) def test_context_manager(self): lock_file_path = self.temp_dir / 'test.lock' From 16fc0da7a318d0edabd58524c3b07f675c04998b Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 14 May 2020 17:04:26 +0200 Subject: [PATCH 6/7] Improve expection handling and make methods private --- ospd_openvas/lock.py | 24 +++++++++++++++++------- tests/test_lock.py | 18 +++++++++--------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/ospd_openvas/lock.py b/ospd_openvas/lock.py index a11a5fdb..e3ef0130 100644 --- a/ospd_openvas/lock.py +++ b/ospd_openvas/lock.py @@ -35,7 +35,7 @@ def __init__(self, path: Path): def has_lock(self) -> bool: return self._has_lock - def acquire_lock(self) -> "LockFile": + def _acquire_lock(self) -> "LockFile": """ Acquite a lock by creating a lock file. """ if self.has_lock(): @@ -47,12 +47,17 @@ def acquire_lock(self) -> "LockFile": parent_dir.mkdir(parents=True, exist_ok=True) self._fd = self._lock_file_path.open('w') - except PermissionError as e: + except Exception as e: # pylint: disable=broad-except logger.error( "Failed to create lock file %s. %s", str(self._lock_file_path), e, ) + try: + self._fd.close() + self._fd = None + except (AttributeError, TypeError): + pass return self # Try to adquire the lock. @@ -64,31 +69,36 @@ def acquire_lock(self) -> "LockFile": logger.error( "Failed to lock the file %s. %s", str(self._lock_file_path), e, ) - self._fd.close() + try: + self._fd.close() + self._fd = None + except (AttributeError, TypeError): + pass return self def wait_for_lock(self): while not self.has_lock(): - self.acquire_lock() + self._acquire_lock() time.sleep(10) return self - def release_lock(self) -> None: + def _release_lock(self) -> None: """ Release the lock by deleting the lock file """ if self.has_lock() and self._fd: fcntl.flock(self._fd, fcntl.LOCK_UN) self._fd.close() + self._fd = None self._has_lock = False logger.debug( "Removed lock from file %s.", str(self._lock_file_path) ) def __enter__(self): - self.acquire_lock() + self._acquire_lock() return self def __exit__(self, exc_type, exc_value, exc_tb): - self.release_lock() + self._release_lock() diff --git a/tests/test_lock.py b/tests/test_lock.py index 3a4d101f..0674c40b 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -39,32 +39,32 @@ def test_acquire_lock(self): lock_file_path = self.temp_dir / 'test.lock' lock_file = LockFile(lock_file_path) - lock_file.acquire_lock() + lock_file._acquire_lock() self.assertTrue(lock_file.has_lock()) self.assertTrue(lock_file_path.exists()) - lock_file.release_lock() + lock_file._release_lock() @patch('ospd_openvas.lock.logger') def test_already_locked(self, mock_logger): lock_file_path = self.temp_dir / 'test.lock' lock_file_aux = LockFile(lock_file_path) - lock_file_aux.acquire_lock() + lock_file_aux._acquire_lock() self.assertTrue(lock_file_aux.has_lock()) lock_file = LockFile(lock_file_path) - lock_file.acquire_lock() + lock_file._acquire_lock() self.assertFalse(lock_file.has_lock()) assert_called_once(mock_logger.error) - lock_file_aux.release_lock() + lock_file_aux._release_lock() def test_create_parent_dirs(self): lock_file_path = self.temp_dir / 'foo' / 'bar' / 'test.lock' lock_file = LockFile(lock_file_path) - lock_file.acquire_lock() + lock_file._acquire_lock() self.assertTrue(lock_file.has_lock()) @@ -72,14 +72,14 @@ def test_create_parent_dirs(self): self.assertTrue(lock_file_path.parent.is_dir()) self.assertTrue(lock_file_path.parent.parent.is_dir()) - lock_file.release_lock() + lock_file._release_lock() @patch('ospd_openvas.lock.logger') def test_create_paren_dirs_fail(self, mock_logger): lock_file_path = Path('/root/lock/file/test.lock') lock_file = LockFile(lock_file_path) - lock_file.acquire_lock() + lock_file._acquire_lock() self.assertFalse(lock_file.has_lock()) assert_called_once(mock_logger.error) @@ -92,7 +92,7 @@ def test_context_manager(self): with lock_file: self.assertTrue(lock_file.has_lock()) self.assertTrue(lock_file_path.is_file()) - lock_file.release_lock() + lock_file._release_lock() # The file is not removed self.assertFalse(lock_file.has_lock()) From 9b89d7c094990936d07f01361eef1baf1462ab70 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Fri, 15 May 2020 08:50:47 +0200 Subject: [PATCH 7/7] Make catch of exceptions broader --- ospd_openvas/lock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ospd_openvas/lock.py b/ospd_openvas/lock.py index e3ef0130..ec4b7105 100644 --- a/ospd_openvas/lock.py +++ b/ospd_openvas/lock.py @@ -56,7 +56,7 @@ def _acquire_lock(self) -> "LockFile": try: self._fd.close() self._fd = None - except (AttributeError, TypeError): + except Exception: # pylint: disable=broad-except pass return self @@ -72,7 +72,7 @@ def _acquire_lock(self) -> "LockFile": try: self._fd.close() self._fd = None - except (AttributeError, TypeError): + except Exception: # pylint: disable=broad-except pass return self