From db43d3d93b51e8629f3ca55468fc29cdd34f0294 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 28 Sep 2020 22:19:57 +0000 Subject: [PATCH 001/109] #150: Work out how the existing tests will work with the new test structure - Just a few comments to help me work out what to do with the existing tests for DB backend. These will be moved to work with Pytest once the new test structure has been defined --- test/test_base.py | 2 ++ test/test_database_helpers.py | 3 ++- test/test_entityHelper.py | 1 + test/test_helpers.py | 7 +++++-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/test_base.py b/test/test_base.py index 9c8f1d9b..185cc376 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -3,6 +3,8 @@ from src.main import app +# Move this into the test defintions and let it be inherited in test classes that need +# it class FlaskAppTest(TestCase): """ The FlaskAppTest Base class sets up a test client to be used to mock requests diff --git a/test/test_database_helpers.py b/test/test_database_helpers.py index d7ee2173..bdbfd1be 100644 --- a/test/test_database_helpers.py +++ b/test/test_database_helpers.py @@ -30,7 +30,8 @@ " has a valid backend type" ) - +# Common across both, needs parameterized tests for different cased names, need multiple +# tests for each filter class TestQueryFilterFactory(TestCase): def test_order_filter(self): self.assertIs( diff --git a/test/test_entityHelper.py b/test/test_entityHelper.py index 780bc2d7..a2e9c224 100644 --- a/test/test_entityHelper.py +++ b/test/test_entityHelper.py @@ -4,6 +4,7 @@ from common.models.db_models import DATAFILE, DATASET, DATAFILEFORMAT, INVESTIGATION +# DB only tests class TestEntityHelper(TestCase): def setUp(self): self.dataset = DATASET() diff --git a/test/test_helpers.py b/test/test_helpers.py index ce4d517b..04cc653e 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -29,6 +29,7 @@ from test.test_base import FlaskAppTest +# Put this in test definitions, but do tests there, don't abstract out class TestIs_valid_json(TestCase): def test_array(self): self.assertTrue(is_valid_json("[]")) @@ -55,6 +56,7 @@ def test_list(self): self.assertFalse(is_valid_json([])) +# Common for both backends, setup and teardown will be different class TestRequires_session_id(FlaskAppTest): def setUp(self): super().setUp() @@ -93,7 +95,7 @@ def test_good_credentials(self): ).status_code, ) - +# Common across both, no need to abstract out class TestQueries_records(TestCase): def test_missing_record_error(self): @queries_records @@ -161,6 +163,7 @@ def raise_bad_request_error(): self.assertEqual(400, ctx.exception.status_code) +# Common across both, no need to abstract out class TestGet_session_id_from_auth_header(FlaskAppTest): def test_no_session_in_header(self): with self.app: @@ -177,7 +180,7 @@ def test_with_good_header(self): self.app.get("/", headers={"Authorization": "Bearer test"}) self.assertEqual("test", get_session_id_from_auth_header()) - +# Common across both, needs abstracting out, class per filter, multiple tests per filter class TestGet_filters_from_query_string(FlaskAppTest): def test_no_filters(self): with self.app: From 2dee8f0b3881830853ef794f608f9e523cfc8e0b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 29 Sep 2020 12:48:38 +0000 Subject: [PATCH 002/109] #150: Allow backends to be created for testing purposes - Currently getting a 403 on these changes but I'm not sure the code is at fault... --- common/backends.py | 25 ++++++++++++------- src/resources/entities/entity_endpoint.py | 6 ++++- .../non_entities/sessions_endpoints.py | 4 ++- .../table_endpoints/table_endpoints.py | 5 +++- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/common/backends.py b/common/backends.py index cc65e915..bcc2bf46 100644 --- a/common/backends.py +++ b/common/backends.py @@ -4,12 +4,19 @@ from common.config import config import sys -backend_type = config.get_backend_type() - -if backend_type == "db": - backend = DatabaseBackend() -elif backend_type == "python_icat": - backend = PythonICATBackend() -else: - sys.exit(f"Invalid config value '{backend_type}' for config option backend") - backend = Backend() + +def create_backend(backend_type): + """ + TODO - Add docstring + """ + + if backend_type == "db": + backend = DatabaseBackend() + elif backend_type == "python_icat": + backend = PythonICATBackend() + else: + # Might turn to a warning so the abstract class can be tested? + sys.exit(f"Invalid config value '{backend_type}' for config option backend") + backend = Backend() + + return backend diff --git a/src/resources/entities/entity_endpoint.py b/src/resources/entities/entity_endpoint.py index cd742d49..35eb4307 100644 --- a/src/resources/entities/entity_endpoint.py +++ b/src/resources/entities/entity_endpoint.py @@ -5,7 +5,11 @@ get_session_id_from_auth_header, get_filters_from_query_string, ) -from common.backends import backend +from common.backends import create_backend +from common.config import config + + +backend = create_backend(config.get_backend_type()) def get_endpoint(name, table): diff --git a/src/resources/non_entities/sessions_endpoints.py b/src/resources/non_entities/sessions_endpoints.py index 492700e1..e709c87a 100644 --- a/src/resources/non_entities/sessions_endpoints.py +++ b/src/resources/non_entities/sessions_endpoints.py @@ -11,11 +11,13 @@ ) from common.helpers import get_session_id_from_auth_header from common.models.db_models import SESSION -from common.backends import backend +from common.backends import create_backend from common.exceptions import AuthenticationError +from common.config import config log = logging.getLogger() +backend = create_backend(config.get_backend_type()) class Sessions(Resource): def post(self): diff --git a/src/resources/table_endpoints/table_endpoints.py b/src/resources/table_endpoints/table_endpoints.py index 4de8a945..7d843463 100644 --- a/src/resources/table_endpoints/table_endpoints.py +++ b/src/resources/table_endpoints/table_endpoints.py @@ -10,7 +10,10 @@ get_session_id_from_auth_header, get_filters_from_query_string, ) -from common.backends import backend +from common.backends import create_backend +from common.config import config + +backend = create_backend(config.get_backend_type()) class InstrumentsFacilityCycles(Resource): From 96b37283cc1ecdb2f4e652477b85256f89a30e86 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 9 Nov 2020 12:48:22 +0000 Subject: [PATCH 003/109] #150: Add pytest to dev dependencies - This also updates a couple of dependencies as marked by safety, which didn't get merged in from a recent git merge --- poetry.lock | 225 +++++++++++++++++++++++++++++++++++-------------- pyproject.toml | 5 +- 2 files changed, 163 insertions(+), 67 deletions(-) diff --git a/poetry.lock b/poetry.lock index 26b07beb..4ff84b3b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -30,17 +30,25 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "attrs" -version = "20.2.0" +version = "20.3.0" description = "Classes Without Boilerplate" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] @@ -81,7 +89,7 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "certifi" -version = "2020.6.20" +version = "2020.11.8" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false @@ -291,7 +299,7 @@ dotenv = ["python-dotenv"] [[package]] name = "flask-cors" -version = "3.0.8" +version = "3.0.9" description = "A Flask extension adding a decorator for CORS support" category = "main" optional = false @@ -374,6 +382,14 @@ zipp = ">=0.5" docs = ["sphinx", "rst.linker"] testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "itsdangerous" version = "1.1.0" @@ -426,7 +442,7 @@ six = "*" [[package]] name = "pathspec" -version = "0.8.0" +version = "0.8.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false @@ -467,6 +483,28 @@ six = "*" coverage = ["pytest-cov"] testing = ["mock", "pytest", "pytest-rerunfailures"] +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "py" +version = "1.9.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "pycodestyle" version = "2.6.0" @@ -502,6 +540,29 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "pytest" +version = "6.1.2" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=17.4.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +checkqa_mypy = ["mypy (==0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.8.1" @@ -523,7 +584,7 @@ python-versions = "*" [[package]] name = "pytz" -version = "2020.1" +version = "2020.4" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -531,15 +592,15 @@ python-versions = "*" [[package]] name = "pyyaml" -version = "5.1.2" +version = "5.3.1" description = "YAML parser and emitter for Python" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "regex" -version = "2020.10.23" +version = "2020.10.28" description = "Alternative regular expression module, to replace re." category = "dev" optional = false @@ -643,11 +704,11 @@ python-versions = "*" [[package]] name = "toml" -version = "0.10.1" +version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" optional = false -python-versions = "*" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typed-ast" @@ -697,7 +758,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "bfbe333aab10d4b666d71e1e0845c58f0b74456f93a11be47b0a16d94b3a791d" +content-hash = "9966041f735c46523968264b2e770b5cf039f9e3ad6b31d96f2398e9cb4283fa" [metadata.files] aniso8601 = [ @@ -712,9 +773,13 @@ appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] attrs = [ - {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, - {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] bandit = [ {file = "bandit-1.6.2-py2.py3-none-any.whl", hash = "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952"}, @@ -725,8 +790,8 @@ black = [ {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, ] certifi = [ - {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, - {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, + {file = "certifi-2020.11.8-py2.py3-none-any.whl", hash = "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd"}, + {file = "certifi-2020.11.8.tar.gz", hash = "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"}, ] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, @@ -794,8 +859,8 @@ flask = [ {file = "Flask-1.1.2.tar.gz", hash = "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060"}, ] flask-cors = [ - {file = "Flask-Cors-3.0.8.tar.gz", hash = "sha256:72170423eb4612f0847318afff8c247b38bd516b7737adfc10d1c2cdbb382d16"}, - {file = "Flask_Cors-3.0.8-py2.py3-none-any.whl", hash = "sha256:f4d97201660e6bbcff2d89d082b5b6d31abee04b1b3003ee073a6fd25ad1d69a"}, + {file = "Flask-Cors-3.0.9.tar.gz", hash = "sha256:6bcfc100288c5d1bcb1dbb854babd59beee622ffd321e444b05f24d6d58466b8"}, + {file = "Flask_Cors-3.0.9-py2.py3-none-any.whl", hash = "sha256:cee4480aaee421ed029eaa788f4049e3e26d15b5affb6a880dade6bafad38324"}, ] flask-restful = [ {file = "Flask-RESTful-0.3.7.tar.gz", hash = "sha256:f8240ec12349afe8df1db168ea7c336c4e5b0271a36982bff7394f93275f2ca9"}, @@ -820,6 +885,10 @@ importlib-metadata = [ {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, ] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] itsdangerous = [ {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, @@ -872,8 +941,8 @@ packaging = [ {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, ] pathspec = [ - {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, - {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, + {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, + {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, ] pbr = [ {file = "pbr-5.5.1-py2.py3-none-any.whl", hash = "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00"}, @@ -887,6 +956,14 @@ pip-tools = [ {file = "pip-tools-5.3.1.tar.gz", hash = "sha256:5672c2b6ca0f1fd803f3b45568c2cf7fadf135b4971e7d665232b2075544c0ef"}, {file = "pip_tools-5.3.1-py2.py3-none-any.whl", hash = "sha256:73787e23269bf8a9230f376c351297b9037ed0d32ab0f9bef4a187d976acc054"}, ] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, +] pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, @@ -903,6 +980,10 @@ pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] +pytest = [ + {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, + {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, +] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, @@ -911,52 +992,66 @@ python-icat = [ {file = "python-icat-0.17.0.tar.gz", hash = "sha256:92942ce5e4b4c7b7db8179b78c07c58b56091a1d275385f69dd99d19a58a9396"}, ] pytz = [ - {file = "pytz-2020.1-py2.py3-none-any.whl", hash = "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed"}, - {file = "pytz-2020.1.tar.gz", hash = "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"}, + {file = "pytz-2020.4-py2.py3-none-any.whl", hash = "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"}, + {file = "pytz-2020.4.tar.gz", hash = "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268"}, ] pyyaml = [ - {file = "PyYAML-5.1.2-cp27-cp27m-win32.whl", hash = "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8"}, - {file = "PyYAML-5.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"}, - {file = "PyYAML-5.1.2-cp34-cp34m-win32.whl", hash = "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9"}, - {file = "PyYAML-5.1.2-cp34-cp34m-win_amd64.whl", hash = "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696"}, - {file = "PyYAML-5.1.2-cp35-cp35m-win32.whl", hash = "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41"}, - {file = "PyYAML-5.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73"}, - {file = "PyYAML-5.1.2-cp36-cp36m-win32.whl", hash = "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299"}, - {file = "PyYAML-5.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b"}, - {file = "PyYAML-5.1.2-cp37-cp37m-win32.whl", hash = "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae"}, - {file = "PyYAML-5.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34"}, - {file = "PyYAML-5.1.2-cp38-cp38m-win32.whl", hash = "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9"}, - {file = "PyYAML-5.1.2-cp38-cp38m-win_amd64.whl", hash = "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681"}, - {file = "PyYAML-5.1.2.tar.gz", hash = "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, + {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, + {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ - {file = "regex-2020.10.23-cp27-cp27m-win32.whl", hash = "sha256:781906e45ef1d10a0ed9ec8ab83a09b5e0d742de70e627b20d61ccb1b1d3964d"}, - {file = "regex-2020.10.23-cp27-cp27m-win_amd64.whl", hash = "sha256:8cd0d587aaac74194ad3e68029124c06245acaeddaae14cb45844e5c9bebeea4"}, - {file = "regex-2020.10.23-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:af360e62a9790e0a96bc9ac845d87bfa0e4ee0ee68547ae8b5a9c1030517dbef"}, - {file = "regex-2020.10.23-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e21340c07090ddc8c16deebfd82eb9c9e1ec5e62f57bb86194a2595fd7b46e0"}, - {file = "regex-2020.10.23-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:e5f6aa56dda92472e9d6f7b1e6331f4e2d51a67caafff4d4c5121cadac03941e"}, - {file = "regex-2020.10.23-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:c30d8766a055c22e39dd7e1a4f98f6266169f2de05db737efe509c2fb9c8a3c8"}, - {file = "regex-2020.10.23-cp36-cp36m-win32.whl", hash = "sha256:1a065e7a6a1b4aa851a0efa1a2579eabc765246b8b3a5fd74000aaa3134b8b4e"}, - {file = "regex-2020.10.23-cp36-cp36m-win_amd64.whl", hash = "sha256:c95d514093b80e5309bdca5dd99e51bcf82c44043b57c34594d9d7556bd04d05"}, - {file = "regex-2020.10.23-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f4b1c65ee86bfbf7d0c3dfd90592a9e3d6e9ecd36c367c884094c050d4c35d04"}, - {file = "regex-2020.10.23-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d62205f00f461fe8b24ade07499454a3b7adf3def1225e258b994e2215fd15c5"}, - {file = "regex-2020.10.23-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:b706c70070eea03411b1761fff3a2675da28d042a1ab7d0863b3efe1faa125c9"}, - {file = "regex-2020.10.23-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d43cf21df524283daa80ecad551c306b7f52881c8d0fe4e3e76a96b626b6d8d8"}, - {file = "regex-2020.10.23-cp37-cp37m-win32.whl", hash = "sha256:570e916a44a361d4e85f355aacd90e9113319c78ce3c2d098d2ddf9631b34505"}, - {file = "regex-2020.10.23-cp37-cp37m-win_amd64.whl", hash = "sha256:1c447b0d108cddc69036b1b3910fac159f2b51fdeec7f13872e059b7bc932be1"}, - {file = "regex-2020.10.23-cp38-cp38-manylinux1_i686.whl", hash = "sha256:8469377a437dbc31e480993399fd1fd15fe26f382dc04c51c9cb73e42965cc06"}, - {file = "regex-2020.10.23-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:59d5c6302d22c16d59611a9fd53556554010db1d47e9df5df37be05007bebe75"}, - {file = "regex-2020.10.23-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a973d5a7a324e2a5230ad7c43f5e1383cac51ef4903bf274936a5634b724b531"}, - {file = "regex-2020.10.23-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:97a023f97cddf00831ba04886d1596ef10f59b93df7f855856f037190936e868"}, - {file = "regex-2020.10.23-cp38-cp38-win32.whl", hash = "sha256:e289a857dca3b35d3615c3a6a438622e20d1bf0abcb82c57d866c8d0be3f44c4"}, - {file = "regex-2020.10.23-cp38-cp38-win_amd64.whl", hash = "sha256:0cb23ed0e327c18fb7eac61ebbb3180ebafed5b9b86ca2e15438201e5903b5dd"}, - {file = "regex-2020.10.23-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c53dc8ee3bb7b7e28ee9feb996a0c999137be6c1d3b02cb6b3c4cba4f9e5ed09"}, - {file = "regex-2020.10.23-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6a46eba253cedcbe8a6469f881f014f0a98819d99d341461630885139850e281"}, - {file = "regex-2020.10.23-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:240509721a663836b611fa13ca1843079fc52d0b91ef3f92d9bba8da12e768a0"}, - {file = "regex-2020.10.23-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:6f567df0601e9c7434958143aebea47a9c4b45434ea0ae0286a4ec19e9877169"}, - {file = "regex-2020.10.23-cp39-cp39-win32.whl", hash = "sha256:bfd7a9fddd11d116a58b62ee6c502fd24cfe22a4792261f258f886aa41c2a899"}, - {file = "regex-2020.10.23-cp39-cp39-win_amd64.whl", hash = "sha256:1a511470db3aa97432ac8c1bf014fcc6c9fbfd0f4b1313024d342549cf86bcd6"}, - {file = "regex-2020.10.23.tar.gz", hash = "sha256:2278453c6a76280b38855a263198961938108ea2333ee145c5168c36b8e2b376"}, + {file = "regex-2020.10.28-cp27-cp27m-win32.whl", hash = "sha256:4b5a9bcb56cc146c3932c648603b24514447eafa6ce9295234767bf92f69b504"}, + {file = "regex-2020.10.28-cp27-cp27m-win_amd64.whl", hash = "sha256:c13d311a4c4a8d671f5860317eb5f09591fbe8259676b86a85769423b544451e"}, + {file = "regex-2020.10.28-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c454ad88e56e80e44f824ef8366bb7e4c3def12999151fd5c0ea76a18fe9aa3e"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c8a2b7ccff330ae4c460aff36626f911f918555660cc28163417cb84ffb25789"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4afa350f162551cf402bfa3cd8302165c8e03e689c897d185f16a167328cc6dd"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:b88fa3b8a3469f22b4f13d045d9bd3eda797aa4e406fde0a2644bc92bbdd4bdd"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:f43109822df2d3faac7aad79613f5f02e4eab0fc8ad7932d2e70e2a83bd49c26"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:de7fd57765398d141949946c84f3590a68cf5887dac3fc52388df0639b01eda4"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:9b6305295b6591e45f069d3553c54d50cc47629eb5c218aac99e0f7fafbf90a1"}, + {file = "regex-2020.10.28-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:bd904c0dec29bbd0769887a816657491721d5f545c29e30fd9d7a1a275dc80ab"}, + {file = "regex-2020.10.28-cp36-cp36m-win32.whl", hash = "sha256:8092a5a06ad9a7a247f2a76ace121183dc4e1a84c259cf9c2ce3bbb69fac3582"}, + {file = "regex-2020.10.28-cp36-cp36m-win_amd64.whl", hash = "sha256:49461446b783945597c4076aea3f49aee4b4ce922bd241e4fcf62a3e7c61794c"}, + {file = "regex-2020.10.28-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:297116e79074ec2a2f885d22db00ce6e88b15f75162c5e8b38f66ea734e73c64"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:8ca9dca965bd86ea3631b975d63b0693566d3cc347e55786d5514988b6f5b84c"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ea37320877d56a7f0a1e6a625d892cf963aa7f570013499f5b8d5ab8402b5625"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:3a5f08039eee9ea195a89e180c5762bfb55258bfb9abb61a20d3abee3b37fd12"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:cb905f3d2e290a8b8f1579d3984f2cfa7c3a29cc7cba608540ceeed18513f520"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:96f99219dddb33e235a37283306834700b63170d7bb2a1ee17e41c6d589c8eb9"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:227a8d2e5282c2b8346e7f68aa759e0331a0b4a890b55a5cfbb28bd0261b84c0"}, + {file = "regex-2020.10.28-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:2564def9ce0710d510b1fc7e5178ce2d20f75571f788b5197b3c8134c366f50c"}, + {file = "regex-2020.10.28-cp37-cp37m-win32.whl", hash = "sha256:a62162be05edf64f819925ea88d09d18b09bebf20971b363ce0c24e8b4aa14c0"}, + {file = "regex-2020.10.28-cp37-cp37m-win_amd64.whl", hash = "sha256:03855ee22980c3e4863dc84c42d6d2901133362db5daf4c36b710dd895d78f0a"}, + {file = "regex-2020.10.28-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf4f896c42c63d1f22039ad57de2644c72587756c0cfb3cc3b7530cfe228277f"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux1_i686.whl", hash = "sha256:625116aca6c4b57c56ea3d70369cacc4d62fead4930f8329d242e4fe7a58ce4b"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2dc522e25e57e88b4980d2bdd334825dbf6fa55f28a922fc3bfa60cc09e5ef53"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:119e0355dbdd4cf593b17f2fc5dbd4aec2b8899d0057e4957ba92f941f704bf5"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:cfcf28ed4ce9ced47b9b9670a4f0d3d3c0e4d4779ad4dadb1ad468b097f808aa"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:b45bab9f224de276b7bc916f6306b86283f6aa8afe7ed4133423efb42015a898"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:52e83a5f28acd621ba8e71c2b816f6541af7144b69cc5859d17da76c436a5427"}, + {file = "regex-2020.10.28-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:aacc8623ffe7999a97935eeabbd24b1ae701d08ea8f874a6ff050e93c3e658cf"}, + {file = "regex-2020.10.28-cp38-cp38-win32.whl", hash = "sha256:06b52815d4ad38d6524666e0d50fe9173533c9cc145a5779b89733284e6f688f"}, + {file = "regex-2020.10.28-cp38-cp38-win_amd64.whl", hash = "sha256:c3466a84fce42c2016113101018a9981804097bacbab029c2d5b4fcb224b89de"}, + {file = "regex-2020.10.28-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:127a9e0c0d91af572fbb9e56d00a504dbd4c65e574ddda3d45b55722462210de"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c2c6c56ee97485a127555c9595c069201b5161de9d05495fbe2132b5ac104786"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1ec66700a10e3c75f1f92cbde36cca0d3aaee4c73dfa26699495a3a30b09093c"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:11116d424734fe356d8777f89d625f0df783251ada95d6261b4c36ad27a394bb"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f1fce1e4929157b2afeb4bb7069204d4370bab9f4fc03ca1fbec8bd601f8c87d"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:3dfca201fa6b326239e1bccb00b915e058707028809b8ecc0cf6819ad233a740"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:b8a686a6c98872007aa41fdbb2e86dc03b287d951ff4a7f1da77fb7f14113e4d"}, + {file = "regex-2020.10.28-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:c32c91a0f1ac779cbd73e62430de3d3502bbc45ffe5bb6c376015acfa848144b"}, + {file = "regex-2020.10.28-cp39-cp39-win32.whl", hash = "sha256:832339223b9ce56b7b15168e691ae654d345ac1635eeb367ade9ecfe0e66bee0"}, + {file = "regex-2020.10.28-cp39-cp39-win_amd64.whl", hash = "sha256:654c1635f2313d0843028487db2191530bca45af61ca85d0b16555c399625b0e"}, + {file = "regex-2020.10.28.tar.gz", hash = "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b"}, ] requests = [ {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, @@ -991,8 +1086,8 @@ text-unidecode = [ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] toml = [ - {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, - {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] typed-ast = [ {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"}, diff --git a/pyproject.toml b/pyproject.toml index c06de98f..8356dbdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,10 @@ python = "^3.6" Flask-RESTful = "0.3.7" SQLAlchemy = "1.3.8" PyMySQL = "0.9.3" -Flask-Cors = "3.0.8" +Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" -PyYAML = "5.1.2" +PyYAML = "5.3.1" python-icat = "0.17.0" suds-community = "^0.8.4" @@ -36,6 +36,7 @@ flake8-commas = "^2.0.0" flake8-comprehensions = "^3.3.0" flake8-logging-format = "^0.6.0" pep8-naming = "^0.11.1" +pytest = "^6.1.2" [tool.poetry.scripts] datagateway-api = "datagateway_api.src.main:run_api" From 29f811127ad4885b115214da61e985e829b01508 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 9 Nov 2020 13:26:21 +0000 Subject: [PATCH 004/109] #150: Add nox 'tests' session - This will run the discovered unit tests in multiple versions of Python (unless nox is specified with -p [version_num] which is probably what I'll do for the majority of the time, just useful to test multi-version compatability) --- noxfile.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/noxfile.py b/noxfile.py index 5abf3111..fa36940c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -61,3 +61,10 @@ def safety(session): external=True, ) session.run("safety", "check", f"--file={requirements.name}", "--full-report") + + +@nox.session(python=["3.6", "3.7", "3.8"], reuse_venv=True) +def tests(session): + args = session.posargs + session.run("poetry", "install", external=True) + session.run("pytest", *args) From 343f8d4fe609417bc51245cde9fce99126bf7b9c Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 9 Nov 2020 15:39:08 +0000 Subject: [PATCH 005/109] #150: Add tests for DateHandler --- test/icat/test_date_handler.py | 77 ++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 test/icat/test_date_handler.py diff --git a/test/icat/test_date_handler.py b/test/icat/test_date_handler.py new file mode 100644 index 00000000..aff4ba6f --- /dev/null +++ b/test/icat/test_date_handler.py @@ -0,0 +1,77 @@ +from datetime import datetime + +import pytest + +from datagateway_api.common.date_handler import DateHandler +from datagateway_api.common.exceptions import BadRequestError + + +class TestIsStrADate: + def test_valid_date(self): + date_output = DateHandler.is_str_a_date("2008-10-15") + assert date_output is True + + def test_valid_boundary_date(self): + date_output = DateHandler.is_str_a_date("29/2/2020") + assert date_output is True + + def test_invalid_boundary_date(self): + date_output = DateHandler.is_str_a_date("29/2/2019") + # There was no leap year in 2019 + assert date_output is False + + def test_invalid_date(self): + date_output = DateHandler.is_str_a_date("25/25/2020") + assert date_output is False + + +class TestStrToDatetime: + def test_valid_str(self): + datetime_output = DateHandler.str_to_datetime_object("2008-10-15 12:05:09") + assert datetime_output == datetime( + year=2008, month=10, day=15, hour=12, minute=5, second=9, + ) + + def test_valid_boundary_str(self): + datetime_output = DateHandler.str_to_datetime_object("2020-02-29 20:20:20") + assert datetime_output == datetime( + year=2020, month=2, day=29, hour=20, minute=20, second=20, + ) + + def test_invalid_boundary_str(self): + with pytest.raises(BadRequestError): + DateHandler.str_to_datetime_object("2019-02-29 12:05:09") + + def test_invalid_str_format_symbols(self): + with pytest.raises(BadRequestError): + DateHandler.str_to_datetime_object("2019/10/05 12:05:09") + + def test_invalid_str_format_order(self): + with pytest.raises(BadRequestError): + DateHandler.str_to_datetime_object("12:05:09 2019-10-05") + + +class TestDatetimeToStr: + def test_valid_datetime(self): + example_date = datetime( + year=2008, month=10, day=15, hour=12, minute=5, second=9, + ) + str_date_output = DateHandler.datetime_object_to_str(example_date) + assert str_date_output == "2008-10-15 12:05:09" + + def test_valid_datetime_no_time(self): + example_date = datetime(year=2008, month=10, day=15) + str_date_output = DateHandler.datetime_object_to_str(example_date) + assert str_date_output == "2008-10-15 00:00:00" + + def test_valid_boundary_datetime(self): + # Can't test invalid leap years as invalid datetime objects can't be created + example_date = datetime( + year=2020, month=2, day=29, hour=23, minute=59, second=59, + ) + str_date_output = DateHandler.datetime_object_to_str(example_date) + assert str_date_output == "2020-02-29 23:59:59" + + def test_invalid_datetime(self): + # TODO - Not sure how to create an invalid datetime object + pass From 230eaca266bb124f44c26f01151f2bf5a6c47dbc Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 10 Nov 2020 09:33:22 +0000 Subject: [PATCH 006/109] #150: Make config file path configurable - This will make testing the configuration easier since you cannot guarantee what the contents of config.json will be - Default is what it was before --- datagateway_api/common/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index 298fcadf..da0c8859 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -10,9 +10,9 @@ class Config(object): - def __init__(self): - config_path = Path(__file__).parent.parent.parent / "config.json" - with open(config_path) as target: + def __init__(self, path=Path(__file__).parent.parent.parent / "config.json"): + self.path = path + with open(self.path) as target: self.config = json.load(target) target.close() From d1b045fbeb9734169c92c9cdd39d6d846fe71590 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 10 Nov 2020 09:37:19 +0000 Subject: [PATCH 007/109] #150: Add tests for config --- test/icat/test_config.py | 140 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 test/icat/test_config.py diff --git a/test/icat/test_config.py b/test/icat/test_config.py new file mode 100644 index 00000000..273c6fcf --- /dev/null +++ b/test/icat/test_config.py @@ -0,0 +1,140 @@ +from pathlib import Path +import tempfile + +import pytest + +from datagateway_api.common.config import Config + + +@pytest.fixture() +def valid_config(): + return Config(path=Path(__file__).parent.parent.parent / "config.json.example") + + +@pytest.fixture() +def invalid_config(): + blank_config_file = tempfile.NamedTemporaryFile(mode="w+", suffix=".json") + blank_config_file.write("{}") + blank_config_file.seek(0) + + return Config(path=blank_config_file.name) + + +class TestGetBackendType: + def test_valid_backend_type(self, valid_config): + backend_type = valid_config.get_backend_type() + assert backend_type == "db" + + def test_invalid_backend_type(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_backend_type() + + +class TestGetDBURL: + def test_valid_db_url(self, valid_config): + db_url = valid_config.get_db_url() + assert db_url == "mysql+pymysql://root:rootpw@localhost:13306/icatdb" + + def test_invalid_db_url(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_db_url() + + +class TestICATURL: + def test_valid_icat_url(self, valid_config): + icat_url = valid_config.get_icat_url() + assert icat_url == "https://localhost.localdomain:8181" + + def test_invalid_icat_url(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_icat_url() + + +class TestICATCheckCert: + def test_valid_icat_check_cert(self, valid_config): + icat_check_cert = valid_config.get_icat_check_cert() + assert icat_check_cert is False + + def test_invalid_icat_check_cert(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_icat_check_cert() + + +class TestGetLogLevel: + def test_valid_log_level(self, valid_config): + log_level = valid_config.get_log_level() + assert log_level == "WARN" + + def test_invalid_log_level(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_log_level() + + +class TestGetLogLocation: + def test_valid_log_location(self, valid_config): + log_location = valid_config.get_log_location() + assert log_location == "/home/user1/datagateway-api/logs.log" + + def test_invalid_log_location(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_log_location() + + +class TestIsDebugMode: + def test_valid_debug_mode(self, valid_config): + debug_mode = valid_config.is_debug_mode() + assert debug_mode is False + + def test_invalid_debug_mode(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.is_debug_mode() + + +class TestIsGenerateSwagger: + def test_valid_generate_swagger(self, valid_config): + generate_swagger = valid_config.is_generate_swagger() + assert generate_swagger is False + + def test_invalid_generate_swagger(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.is_generate_swagger() + + +class TestGetHost: + def test_valid_host(self, valid_config): + host = valid_config.get_host() + assert host == "127.0.0.1" + + def test_invalid_host(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_host() + + +class TestGetPort: + def test_valid_port(self, valid_config): + port = valid_config.get_port() + assert port == "5000" + + def test_invalid_port(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_icat_url() + + +class TestGetICATProperties: + def test_valid_icat_properties(self, valid_config): + example_icat_properties = { + "maxEntities": 10000, + "lifetimeMinutes": 120, + "authenticators": [ + { + "mnemonic": "simple", + "keys": [{"name": "username"}, {"name": "password", "hide": True}], + "friendly": "Simple", + }, + ], + "containerType": "Glassfish", + } + + icat_properties = valid_config.get_icat_properties() + # Values could vary across versions, less likely that keys will + assert icat_properties.keys() == example_icat_properties.keys() From 257082e3d517d58c468022bf66ad7f834a8430a6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 10 Nov 2020 11:42:21 +0000 Subject: [PATCH 008/109] #150: Add pytest-cov to the repo - nox -s tests -- --cov is a good starting point for usage - I'm not sure how much I trust the output - there's around half coverage for both DB & ICAT backend's helper files, even though I've currently got the ICAT backend configured... --- .gitignore | 1 + poetry.lock | 71 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 12 +++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 99e13355..bfcfa809 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ config.json .vscode/ .nox/ .python-version +.coverage diff --git a/poetry.lock b/poetry.lock index 4ff84b3b..9feb72a6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -119,6 +119,20 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "coverage" +version = "5.3" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.dependencies] +toml = {version = "*", optional = true, markers = "extra == \"toml\""} + +[package.extras] +toml = ["toml"] + [[package]] name = "dparse" version = "0.5.1" @@ -563,6 +577,21 @@ toml = "*" checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-cov" +version = "2.10.1" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +coverage = ">=4.4" +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] + [[package]] name = "python-dateutil" version = "2.8.1" @@ -758,7 +787,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "9966041f735c46523968264b2e770b5cf039f9e3ad6b31d96f2398e9cb4283fa" +content-hash = "d3190c53f5cec339b2b4559998fb054cf6c3c9c5dd61caacde0dcb8b7a887863" [metadata.files] aniso8601 = [ @@ -805,6 +834,42 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +coverage = [ + {file = "coverage-5.3-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270"}, + {file = "coverage-5.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4"}, + {file = "coverage-5.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9"}, + {file = "coverage-5.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729"}, + {file = "coverage-5.3-cp27-cp27m-win32.whl", hash = "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d"}, + {file = "coverage-5.3-cp27-cp27m-win_amd64.whl", hash = "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418"}, + {file = "coverage-5.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9"}, + {file = "coverage-5.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5"}, + {file = "coverage-5.3-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822"}, + {file = "coverage-5.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097"}, + {file = "coverage-5.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9"}, + {file = "coverage-5.3-cp35-cp35m-win32.whl", hash = "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636"}, + {file = "coverage-5.3-cp35-cp35m-win_amd64.whl", hash = "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f"}, + {file = "coverage-5.3-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237"}, + {file = "coverage-5.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54"}, + {file = "coverage-5.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7"}, + {file = "coverage-5.3-cp36-cp36m-win32.whl", hash = "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a"}, + {file = "coverage-5.3-cp36-cp36m-win_amd64.whl", hash = "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d"}, + {file = "coverage-5.3-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8"}, + {file = "coverage-5.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f"}, + {file = "coverage-5.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c"}, + {file = "coverage-5.3-cp37-cp37m-win32.whl", hash = "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751"}, + {file = "coverage-5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709"}, + {file = "coverage-5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516"}, + {file = "coverage-5.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f"}, + {file = "coverage-5.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259"}, + {file = "coverage-5.3-cp38-cp38-win32.whl", hash = "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82"}, + {file = "coverage-5.3-cp38-cp38-win_amd64.whl", hash = "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221"}, + {file = "coverage-5.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978"}, + {file = "coverage-5.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21"}, + {file = "coverage-5.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24"}, + {file = "coverage-5.3-cp39-cp39-win32.whl", hash = "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7"}, + {file = "coverage-5.3-cp39-cp39-win_amd64.whl", hash = "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7"}, + {file = "coverage-5.3.tar.gz", hash = "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0"}, +] dparse = [ {file = "dparse-0.5.1-py3-none-any.whl", hash = "sha256:e953a25e44ebb60a5c6efc2add4420c177f1d8404509da88da9729202f306994"}, {file = "dparse-0.5.1.tar.gz", hash = "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367"}, @@ -984,6 +1049,10 @@ pytest = [ {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, ] +pytest-cov = [ + {file = "pytest-cov-2.10.1.tar.gz", hash = "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e"}, + {file = "pytest_cov-2.10.1-py2.py3-none-any.whl", hash = "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191"}, +] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, diff --git a/pyproject.toml b/pyproject.toml index 8356dbdd..b26eac75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,8 @@ flake8-comprehensions = "^3.3.0" flake8-logging-format = "^0.6.0" pep8-naming = "^0.11.1" pytest = "^6.1.2" +coverage = {extras = ["toml"], version = "^5.3"} +pytest-cov = "^2.10.1" [tool.poetry.scripts] datagateway-api = "datagateway_api.src.main:run_api" @@ -44,3 +46,13 @@ datagateway-api = "datagateway_api.src.main:run_api" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.coverage.paths] +source = ["datagateway_api"] + +[tool.coverage.run] +branch = true +source = ["datagateway_api"] + +[tool.coverage.report] +show_missing = true From 57fcd682dbe17532f37b3850cb7937b9c3a0e4fb Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 10 Nov 2020 12:43:07 +0000 Subject: [PATCH 009/109] #150: Add configuration options required for repo's tests - This commit also adds getters in config.py --- config.json.example | 5 ++++- datagateway_api/common/config.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index c977e9b0..0ae4df26 100644 --- a/config.json.example +++ b/config.json.example @@ -8,5 +8,8 @@ "debug_mode": false, "generate_swagger": false, "host": "127.0.0.1", - "port": "5000" + "port": "5000", + "test_username": "root", + "test_password": "pw", + "test_mechanism": "simple" } diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index da0c8859..7ae31b9e 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -76,6 +76,24 @@ def get_port(self): except KeyError: sys.exit("Missing config value, port") + def get_test_username(self): + try: + return self.config["test_username"] + except KeyError: + sys.exit("Missing config value, test_username") + + def get_test_password(self): + try: + return self.config["test_password"] + except KeyError: + sys.exit("Missing config value, test_password") + + def get_test_mechanism(self): + try: + return self.config["test_mechanism"] + except KeyError: + sys.exit("Missing config value, test_mechanism") + def get_icat_properties(self): """ ICAT properties can be retrieved using Python ICAT's client object, however this From 7455c480f88aa255322cff29258ca4562c101b1f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 10 Nov 2020 12:53:04 +0000 Subject: [PATCH 010/109] #150: Add tests for test configuration options --- test/icat/test_config.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/icat/test_config.py b/test/icat/test_config.py index 273c6fcf..57ac2c20 100644 --- a/test/icat/test_config.py +++ b/test/icat/test_config.py @@ -120,6 +120,36 @@ def test_invalid_port(self, invalid_config): invalid_config.get_icat_url() +class TestGetTestUsername: + def test_valid_test_username(self, valid_config): + test_username = valid_config.get_test_username() + assert test_username == "root" + + def test_invalid_test_username(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_test_username() + + +class TestGetTestPassword: + def test_valid_test_password(self, valid_config): + test_password = valid_config.get_test_password() + assert test_password == "pw" + + def test_invalid_test_password(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_test_password() + + +class TestGetTestMechanism: + def test_valid_test_mechanism(self, valid_config): + test_mechanism = valid_config.get_test_mechanism() + assert test_mechanism == "simple" + + def test_invalid_test_mechanism(self, invalid_config): + with pytest.raises(SystemExit): + invalid_config.get_test_mechanism() + + class TestGetICATProperties: def test_valid_icat_properties(self, valid_config): example_icat_properties = { From 572e89cd36db1cfe15857c7e40da0c7e3df3c402 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 10 Nov 2020 13:02:19 +0000 Subject: [PATCH 011/109] #150: Combine config test username and password - This will make it more convenient to use through the tests in the repo --- config.json.example | 3 +-- datagateway_api/common/config.py | 12 +++--------- test/icat/test_config.py | 22 ++++++---------------- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/config.json.example b/config.json.example index 0ae4df26..8aab7943 100644 --- a/config.json.example +++ b/config.json.example @@ -9,7 +9,6 @@ "generate_swagger": false, "host": "127.0.0.1", "port": "5000", - "test_username": "root", - "test_password": "pw", + "test_user_credentials": {"username": "root", "password": "pw"}, "test_mechanism": "simple" } diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index 7ae31b9e..cdc26b97 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -76,17 +76,11 @@ def get_port(self): except KeyError: sys.exit("Missing config value, port") - def get_test_username(self): + def get_test_user_credentials(self): try: - return self.config["test_username"] + return self.config["test_user_credentials"] except KeyError: - sys.exit("Missing config value, test_username") - - def get_test_password(self): - try: - return self.config["test_password"] - except KeyError: - sys.exit("Missing config value, test_password") + sys.exit("Missing config value, test_user_credentials") def get_test_mechanism(self): try: diff --git a/test/icat/test_config.py b/test/icat/test_config.py index 57ac2c20..dee771c4 100644 --- a/test/icat/test_config.py +++ b/test/icat/test_config.py @@ -120,24 +120,14 @@ def test_invalid_port(self, invalid_config): invalid_config.get_icat_url() -class TestGetTestUsername: - def test_valid_test_username(self, valid_config): - test_username = valid_config.get_test_username() - assert test_username == "root" +class TestGetTestUserCredentials: + def test_valid_test_user_credentials(self, valid_config): + test_user_credentials = valid_config.get_test_user_credentials() + assert test_user_credentials == {"username": "root", "password": "pw"} - def test_invalid_test_username(self, invalid_config): + def test_invalid_test_user_credentials(self, invalid_config): with pytest.raises(SystemExit): - invalid_config.get_test_username() - - -class TestGetTestPassword: - def test_valid_test_password(self, valid_config): - test_password = valid_config.get_test_password() - assert test_password == "pw" - - def test_invalid_test_password(self, invalid_config): - with pytest.raises(SystemExit): - invalid_config.get_test_password() + invalid_config.get_test_user_credentials() class TestGetTestMechanism: From 71720d0e657703550701041192fcd3ff3442f543 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:15:03 +0000 Subject: [PATCH 012/109] #150: Add pytest fixtures for filter testing - Since they're placed in conftest.py, pytest will automatically pick these up, they don't need to be imported into the places I use them in --- test/conftest.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..845ad913 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,20 @@ +from icat.client import Client +from icat.query import Query +import pytest + +from datagateway_api.common.config import config + + +@pytest.fixture(scope="package") +def icat_client(): + client = Client(config.get_icat_url(), checkCert=config.get_icat_check_cert()) + client.login(config.get_test_mechanism(), config.get_test_user_credentials()) + print(f"ID: {client.sessionId}") + return client + + +@pytest.fixture() +def icat_query(icat_client): + query = Query(icat_client, "Investigation") + + return query From d5b189b08b85b47aa8504c775ea938878b074fa6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:16:20 +0000 Subject: [PATCH 013/109] #150: Add tests for ICAT where filters --- test/icat/filters/test_where_filter.py | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/icat/filters/test_where_filter.py diff --git a/test/icat/filters/test_where_filter.py b/test/icat/filters/test_where_filter.py new file mode 100644 index 00000000..750eadff --- /dev/null +++ b/test/icat/filters/test_where_filter.py @@ -0,0 +1,67 @@ +import pytest + +from datagateway_api.common.exceptions import BadRequestError, FilterError +from datagateway_api.common.filter_order_handler import FilterOrderHandler +from datagateway_api.common.icat.filters import PythonICATWhereFilter + + +class TestICATWhereFilter: + @pytest.mark.parametrize( + "operation, value, expected_condition_value", + [ + pytest.param("eq", 5, "= '5'", id="equal"), + pytest.param("ne", 5, "!= 5", id="not equal"), + pytest.param("like", 5, "like '%5%'", id="like"), + pytest.param("lt", 5, "< '5'", id="less than"), + pytest.param("lte", 5, "<= '5'", id="less than or equal"), + pytest.param("gt", 5, "> '5'", id="greater than"), + pytest.param("gte", 5, ">= '5'", id="greater than or equal"), + pytest.param("in", [1, 2, 3, 4], "in (1, 2, 3, 4)", id="in a list"), + ], + ) + def test_valid_operations( + self, icat_query, operation, value, expected_condition_value, + ): + test_filter = PythonICATWhereFilter("id", value, operation) + test_filter.apply_filter(icat_query) + + assert icat_query.conditions == {"id": expected_condition_value} + + def test_invalid_in_operation(self, icat_query): + with pytest.raises(BadRequestError): + PythonICATWhereFilter("id", "1, 2, 3, 4, 5", "in") + + def test_invalid_operation(self, icat_query): + test_filter = PythonICATWhereFilter("id", 10, "non") + + with pytest.raises(FilterError): + test_filter.apply_filter(icat_query) + + def test_valid_internal_icat_value(self, icat_query): + """Check that values that point to other values in the schema are applied""" + test_filter = PythonICATWhereFilter("startDate", "o.endDate", "lt") + test_filter.apply_filter(icat_query) + + assert icat_query.conditions == {"startDate": "< o.endDate"} + + def test_valid_field(self, icat_query): + test_filter = PythonICATWhereFilter("title", "Investigation Title", "eq") + test_filter.apply_filter(icat_query) + + assert icat_query.conditions == {"title": "= 'Investigation Title'"} + + def test_invalid_field(self, icat_query): + test_filter = PythonICATWhereFilter("random_field", "my_value", "eq") + + with pytest.raises(FilterError): + test_filter.apply_filter(icat_query) + + def test_multiple_conditions_per_field(self, icat_query): + lt_filter = PythonICATWhereFilter("id", 10, "lt") + gt_filter = PythonICATWhereFilter("id", 5, "gt") + + filter_handler = FilterOrderHandler() + filter_handler.add_filters([lt_filter, gt_filter]) + filter_handler.apply_filters(icat_query) + + assert icat_query.conditions == {"id": ["< '10'", "> '5'"]} From 3c350ee1a2a6e772f54c9ac4c9ee7e6725888065 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:16:52 +0000 Subject: [PATCH 014/109] #150: Add tests for ICAT order filters --- test/icat/filters/test_order_filter.py | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 test/icat/filters/test_order_filter.py diff --git a/test/icat/filters/test_order_filter.py b/test/icat/filters/test_order_filter.py new file mode 100644 index 00000000..6f2ba8c8 --- /dev/null +++ b/test/icat/filters/test_order_filter.py @@ -0,0 +1,55 @@ +import pytest + + +from datagateway_api.common.exceptions import FilterError +from datagateway_api.common.filter_order_handler import FilterOrderHandler +from datagateway_api.common.icat.filters import PythonICATOrderFilter + + +class TestICATOrderFilter: + def test_direction_is_uppercase(self, icat_query): + """Direction must be uppercase for Python ICAT to see the input as valid""" + test_filter = PythonICATOrderFilter("id", "asc") + + assert test_filter.direction == "ASC" + + def test_result_order_appended(self, icat_query): + id_filter = PythonICATOrderFilter("id", "ASC") + title_filter = PythonICATOrderFilter("title", "DESC") + + filter_handler = FilterOrderHandler() + filter_handler.add_filters([id_filter, title_filter]) + filter_handler.apply_filters(icat_query) + + assert PythonICATOrderFilter.result_order == [("id", "ASC"), ("title", "DESC")] + + filter_handler.clear_python_icat_order_filters() + + def test_filter_applied_to_query(self, icat_query): + test_filter = PythonICATOrderFilter("id", "DESC") + + filter_handler = FilterOrderHandler() + filter_handler.add_filter(test_filter) + filter_handler.apply_filters(icat_query) + + assert icat_query.order == [("id", "DESC")] + + filter_handler.clear_python_icat_order_filters() + + def test_invalid_field(self, icat_query): + test_filter = PythonICATOrderFilter("unknown_field", "DESC") + + filter_handler = FilterOrderHandler() + filter_handler.add_filter(test_filter) + with pytest.raises(FilterError): + filter_handler.apply_filters(icat_query) + + filter_handler.clear_python_icat_order_filters() + + def test_invalid_direction(self, icat_query): + test_filter = PythonICATOrderFilter("id", "up") + + filter_handler = FilterOrderHandler() + filter_handler.add_filter(test_filter) + with pytest.raises(FilterError): + filter_handler.apply_filters(icat_query) From 90f3b4b684607c9b0bc012ee9f47681b467ccd18 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:17:15 +0000 Subject: [PATCH 015/109] #150: Add tests for ICAT distinct filters --- test/icat/filters/test_distinct_filter.py | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/icat/filters/test_distinct_filter.py diff --git a/test/icat/filters/test_distinct_filter.py b/test/icat/filters/test_distinct_filter.py new file mode 100644 index 00000000..66255b3a --- /dev/null +++ b/test/icat/filters/test_distinct_filter.py @@ -0,0 +1,45 @@ +import pytest + +from datagateway_api.common.exceptions import FilterError +from datagateway_api.common.icat.filters import PythonICATDistinctFieldFilter + + +class TestICATDistinctFilter: + def test_valid_str_field_input(self, icat_query): + test_filter = PythonICATDistinctFieldFilter("name") + test_filter.apply_filter(icat_query) + + assert ( + icat_query.conditions == {"name": "!= null"} + and icat_query.aggregate == "DISTINCT" + ) + + def test_valid_list_fields_input(self, icat_query): + test_filter = PythonICATDistinctFieldFilter(["doi", "name", "title"]) + test_filter.apply_filter(icat_query) + + assert ( + icat_query.conditions + == {"doi": "!= null", "name": "!= null", "title": "!= null"} + and icat_query.aggregate == "DISTINCT" + ) + + def test_invalid_field(self, icat_query): + test_filter = PythonICATDistinctFieldFilter("my_new_field") + with pytest.raises(FilterError): + test_filter.apply_filter(icat_query) + + def test_distinct_aggregate_added(self, icat_query): + test_filter = PythonICATDistinctFieldFilter("id") + test_filter.apply_filter(icat_query) + + assert icat_query.aggregate == "DISTINCT" + + @pytest.mark.parametrize("existing_aggregate", ["COUNT", "AVG", "SUM"]) + def test_existing_aggregate_appended(self, icat_query, existing_aggregate): + icat_query.setAggregate(existing_aggregate) + + test_filter = PythonICATDistinctFieldFilter("name") + test_filter.apply_filter(icat_query) + + assert icat_query.aggregate == f"{existing_aggregate}:DISTINCT" From 7fe62ff135c710626fe6c5915635fc1fe30c9a7e Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:17:37 +0000 Subject: [PATCH 016/109] #150: Add tests for ICAT include filters --- test/icat/filters/test_include_filter.py | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test/icat/filters/test_include_filter.py diff --git a/test/icat/filters/test_include_filter.py b/test/icat/filters/test_include_filter.py new file mode 100644 index 00000000..505c61dc --- /dev/null +++ b/test/icat/filters/test_include_filter.py @@ -0,0 +1,47 @@ +import pytest + +from datagateway_api.common.icat.filters import PythonICATIncludeFilter + + +class TestICATIncludeFilter: + @pytest.mark.parametrize( + "filter_input, expected_output", + [ + pytest.param("investigationUsers", {"investigationUsers"}, id="string"), + pytest.param( + {"investigationUsers": "user"}, + {"investigationUsers.user"}, + id="dictionary", + ), + pytest.param( + {"datasets": ["datafiles", "sample"]}, + {"datasets.datafiles", "datasets.sample"}, + id="dictionary with list", + ), + pytest.param( + {"datasets": {"datafiles": "datafileFormat"}}, + {"datasets.datafiles.datafileFormat"}, + id="nested dictionary", + ), + pytest.param( + ["studyInvestigations", "datasets", "facility"], + {"studyInvestigations", "datasets", "facility"}, + id="list of strings", + ), + pytest.param( + [{"investigationUsers": "user"}, {"datasets": "datafiles"}], + {"investigationUsers.user", "datasets.datafiles"}, + id="list of dictionaries", + ), + pytest.param( + ["investigationUsers", ["datasets", "facility"]], + {"investigationUsers", "datasets", "facility"}, + id="nested list", + ), + ], + ) + def test_valid_input(self, icat_query, filter_input, expected_output): + test_filter = PythonICATIncludeFilter(filter_input) + test_filter.apply_filter(icat_query) + + assert icat_query.includes == expected_output From 34d46bacc9c7736c3f654ff63eedc4c1ee0716e6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:18:04 +0000 Subject: [PATCH 017/109] #150: Add tests for ICAT skip filters --- test/icat/filters/test_skip_filter.py | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test/icat/filters/test_skip_filter.py diff --git a/test/icat/filters/test_skip_filter.py b/test/icat/filters/test_skip_filter.py new file mode 100644 index 00000000..9d07e690 --- /dev/null +++ b/test/icat/filters/test_skip_filter.py @@ -0,0 +1,29 @@ +import pytest + +from datagateway_api.common.config import config +from datagateway_api.common.exceptions import FilterError +from datagateway_api.common.icat.filters import PythonICATSkipFilter + + +class TestICATSkipFilter: + @pytest.mark.parametrize( + "skip_value", [pytest.param(10, id="typical"), pytest.param(0, id="boundary")], + ) + def test_valid_skip_value(self, icat_query, skip_value): + test_filter = PythonICATSkipFilter(skip_value) + test_filter.apply_filter(icat_query) + + assert icat_query.limit == ( + skip_value, + config.get_icat_properties()["maxEntities"], + ) + + @pytest.mark.parametrize( + "skip_value", + [pytest.param(-375, id="extreme invalid"), pytest.param(-1, id="boundary")], + ) + def test_invalid_skip_value(self, icat_query, skip_value): + test_filter = PythonICATSkipFilter(skip_value) + + with pytest.raises(FilterError): + test_filter.apply_filter(icat_query) From f4ef19f4e898406515f58b24d252393d744787c6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:18:24 +0000 Subject: [PATCH 018/109] #150: Add tests for ICAT limit filters --- test/icat/filters/test_limit_filter.py | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test/icat/filters/test_limit_filter.py diff --git a/test/icat/filters/test_limit_filter.py b/test/icat/filters/test_limit_filter.py new file mode 100644 index 00000000..61af6942 --- /dev/null +++ b/test/icat/filters/test_limit_filter.py @@ -0,0 +1,58 @@ +import pytest + +from datagateway_api.common.exceptions import FilterError +from datagateway_api.common.filter_order_handler import FilterOrderHandler +from datagateway_api.common.icat.filters import ( + PythonICATLimitFilter, + PythonICATSkipFilter, +) + + +class TestICATLimitFilter: + @pytest.mark.parametrize( + "limit_value", + [ + pytest.param(10, id="typical"), + pytest.param(0, id="low boundary"), + pytest.param(9999, id="high boundary"), + ], + ) + def test_valid_limit_value(self, icat_query, limit_value): + test_filter = PythonICATLimitFilter(limit_value) + test_filter.apply_filter(icat_query) + + assert icat_query.limit == (0, limit_value) + + @pytest.mark.parametrize( + "limit_value", + [pytest.param(-50, id="extreme invalid"), pytest.param(-1, id="boundary")], + ) + def test_invalid_limit_value(self, icat_query, limit_value): + test_filter = PythonICATLimitFilter(limit_value) + + with pytest.raises(FilterError): + test_filter.apply_filter(icat_query) + + @pytest.mark.parametrize( + "skip_value, limit_value", + [ + pytest.param(10, 10, id="identical typical values"), + pytest.param(0, 0, id="identical low boundary values"), + pytest.param(15, 25, id="different typical values"), + pytest.param(0, 9999, id="different boundary values"), + ], + ) + def test_limit_and_skip_merge_correctly(self, icat_query, skip_value, limit_value): + """ + Skip and limit values are set together in Python ICAT, limit value should match + max entities allowed in one transaction in ICAT as defined in ICAT properties + """ + skip_filter = PythonICATSkipFilter(skip_value) + limit_filter = PythonICATLimitFilter(limit_value) + + filter_handler = FilterOrderHandler() + filter_handler.add_filters([skip_filter, limit_filter]) + filter_handler.merge_python_icat_limit_skip_filters() + filter_handler.apply_filters(icat_query) + + assert icat_query.limit == (skip_value, limit_value) From 55397e56a1090ee35ac9fcb81b5724326cfbbe91 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 11 Nov 2020 20:18:58 +0000 Subject: [PATCH 019/109] #150: Add test for backend creation --- test/icat/test_backends.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/icat/test_backends.py diff --git a/test/icat/test_backends.py b/test/icat/test_backends.py new file mode 100644 index 00000000..30b2f48d --- /dev/null +++ b/test/icat/test_backends.py @@ -0,0 +1,19 @@ +import pytest + +from datagateway_api.common.backends import create_backend +from datagateway_api.common.database.backend import DatabaseBackend +from datagateway_api.common.icat.backend import PythonICATBackend + + +class TestBackends: + @pytest.mark.parametrize( + "backend_name, backend_type", + [ + pytest.param("db", DatabaseBackend, id="Database Backend"), + pytest.param("python_icat", PythonICATBackend, id="Python ICAT Backend"), + ], + ) + def test_backend_creation(self, backend_name, backend_type): + test_backend = create_backend(backend_name) + + assert type(test_backend) == backend_type From e621f423e38a0538ae51cd9fa66ab84be6b299e9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 16 Nov 2020 14:24:22 +0000 Subject: [PATCH 020/109] #150: Add tests for filter handler --- test/icat/test_filter_order_handler.py | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 test/icat/test_filter_order_handler.py diff --git a/test/icat/test_filter_order_handler.py b/test/icat/test_filter_order_handler.py new file mode 100644 index 00000000..b31fe13d --- /dev/null +++ b/test/icat/test_filter_order_handler.py @@ -0,0 +1,69 @@ +import pytest + +from datagateway_api.common.filter_order_handler import FilterOrderHandler +from datagateway_api.common.icat.filters import ( + PythonICATLimitFilter, + PythonICATWhereFilter, +) + + +class TestFilterOrderHandler: + """ + `merge_python_icat_limit_skip_filters` and`clear_python_icat_order_filters()` are + tested while testing the ICAT backend filters, so tests of these functions won't be + found here + """ + + def test_add_filter(self, icat_query): + test_handler = FilterOrderHandler() + test_filter = PythonICATWhereFilter("id", 2, "eq") + + test_handler.add_filter(test_filter) + + assert test_handler.filters == [test_filter] + + def test_add_filters(self): + test_handler = FilterOrderHandler() + id_filter = PythonICATWhereFilter("id", 2, "eq") + name_filter = PythonICATWhereFilter("name", "New Name", "like") + filter_list = [id_filter, name_filter] + + test_handler.add_filters(filter_list) + + assert test_handler.filters == filter_list + + def test_remove_filter(self): + test_filter = PythonICATWhereFilter("id", 2, "eq") + + test_handler = FilterOrderHandler() + test_handler.add_filter(test_filter) + test_handler.remove_filter(test_filter) + + assert test_handler.filters == [] + + def test_remove_not_added_filter(self): + test_handler = FilterOrderHandler() + test_filter = PythonICATWhereFilter("id", 2, "eq") + + with pytest.raises(ValueError): + test_handler.remove_filter(test_filter) + + def test_sort_filters(self): + limit_filter = PythonICATLimitFilter(10) + where_filter = PythonICATWhereFilter("id", 2, "eq") + + test_handler = FilterOrderHandler() + test_handler.add_filters([limit_filter, where_filter]) + test_handler.sort_filters() + + assert test_handler.filters == [where_filter, limit_filter] + + def test_apply_filters(self, icat_query): + where_filter = PythonICATWhereFilter("id", 2, "eq") + limit_filter = PythonICATLimitFilter(10) + + test_handler = FilterOrderHandler() + test_handler.add_filters([where_filter, limit_filter]) + test_handler.apply_filters(icat_query) + + assert icat_query.conditions == {"id": "= '2'"} and icat_query.limit == (0, 10) From 7bf11413cabbb3560b826828b0eb51d287323c62 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 19 Nov 2020 16:46:51 +0000 Subject: [PATCH 021/109] #150: Ensure list flatten always returns the list in the same order - That function would return the elements of a list in different order each time (despite identical inputs each time) so this could prove more difficult to test --- datagateway_api/common/icat/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/common/icat/query.py b/datagateway_api/common/icat/query.py index fe46c1fe..1a8090ee 100644 --- a/datagateway_api/common/icat/query.py +++ b/datagateway_api/common/icat/query.py @@ -329,4 +329,4 @@ def flatten_query_included_fields(self, includes): ICAT query """ - return [m for n in (field.split(".") for field in includes) for m in n] + return [m for n in (field.split(".") for field in sorted(includes)) for m in n] From 5444bd59d0fac9e185bb470f8939daaeacb4e7b6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 19 Nov 2020 17:04:20 +0000 Subject: [PATCH 022/109] #150: Add way of injecting data into ICAT for testing endpoint and ICATQuery - This fixture also removes the data from ICAT at the end of the test --- test/conftest.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 845ad913..bbf99107 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,15 +1,17 @@ +import uuid + from icat.client import Client from icat.query import Query import pytest from datagateway_api.common.config import config +from test.icat.test_query import remove_meta_attributes @pytest.fixture(scope="package") def icat_client(): client = Client(config.get_icat_url(), checkCert=config.get_icat_check_cert()) client.login(config.get_test_mechanism(), config.get_test_user_credentials()) - print(f"ID: {client.sessionId}") return client @@ -18,3 +20,29 @@ def icat_query(icat_client): query = Query(icat_client, "Investigation") return query + + +@pytest.fixture() +def add_single_investigation_test_data(icat_client): + # Inject data + investigation = icat_client.new("investigation") + investigation.name = "Test Data for DataGateway API Testing" + investigation.title = "Test data for the Python ICAT Backend on DataGateway API" + # UUID visit ID means uniquesness constraint should always be met + investigation.visitId = str(uuid.uuid1()) + investigation.facility = icat_client.get("Facility", 1) + investigation.type = icat_client.get("InvestigationType", 1) + investigation.create() + + investigation_dict = investigation.as_dict() + remove_meta_attributes(investigation_dict) + + yield [investigation_dict] + + # Remove data from ICAT + icat_client.delete(investigation) + + +@pytest.fixture() +def remove_single_investiation_result(): + pass From d7b88dd049726b76aed87d4c276b8a5d81c3bcbb Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 19 Nov 2020 17:08:18 +0000 Subject: [PATCH 023/109] #150: Add beginning of tests for ICATQuery - This commit also includes a skeleton for the remaining tests for this class --- test/icat/test_query.py | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 test/icat/test_query.py diff --git a/test/icat/test_query.py b/test/icat/test_query.py new file mode 100644 index 00000000..e79fbf9f --- /dev/null +++ b/test/icat/test_query.py @@ -0,0 +1,92 @@ +from icat.entity import Entity +import pytest + +from datagateway_api.common.exceptions import PythonICATError +from datagateway_api.common.icat.filters import PythonICATWhereFilter +from datagateway_api.common.icat.query import ICATQuery + + +def remove_meta_attributes(entity_dict): + meta_attributes = Entity.MetaAttr + for attr in meta_attributes: + entity_dict.pop(attr) + + +class TestICATQuery: + def test_valid_query_creation(self, icat_client): + # Paramitise and add inputs for conditions, aggregate and includes + test_query = ICATQuery(icat_client, "User") + + assert test_query.query.entity == icat_client.getEntityClass("User") + + def test_invalid_query_creation(self, icat_client): + with pytest.raises(PythonICATError): + ICATQuery(icat_client, "User", conditions={"invalid": "invalid"}) + + # @pytest.mark.usefixtures("") + def test_valid_query_exeuction( + self, icat_client, add_single_investigation_test_data, + ): + test_query = ICATQuery(icat_client, "Investigation") + test_data_filter = PythonICATWhereFilter( + "title", "Test data for the Python ICAT Backend on DataGateway API", "eq", + ) + test_data_filter.apply_filter(test_query.query) + query_data = test_query.execute_query(icat_client) + + query_output_dicts = [] + for entity in query_data: + entity_dict = entity.as_dict() + remove_meta_attributes(entity_dict) + query_output_dicts.append(entity_dict) + + assert query_output_dicts == add_single_investigation_test_data + + def test_invalid_query_execution(self, icat_client): + # Try to get ICATValidationError raised + pass + + def test_valid_count_query_execution(self, icat_client): + pass + + def test_valid_distinct_query_execution(self, icat_client): + pass + + def test_json_format_execution_output(self, icat_client): + pass + + def test_icat_execution_output(self, icat_client): + pass + + # gap in function testing + + def test_valid_entity_to_dict_conversion(self, icat_client): + # Want just a typical entity and an entity with an entity list in it + pass + + def test_valid_distinct_attribute_mapping(self): + pass + + # another gap + + def test_include_fields_list_flatten(self, icat_client): + included_field_set = { + "investigationUsers.investigation.datasets", + "userGroups", + "instrumentScientists", + "studies", + } + + test_query = ICATQuery(icat_client, "User") + + flat_list = test_query.flatten_query_included_fields(included_field_set) + print(flat_list) + + assert flat_list == [ + "instrumentScientists", + "investigationUsers", + "investigation", + "datasets", + "studies", + "userGroups", + ] From 07e42ec8527b3db951e9ca92a4cdfa9ec63a4f4c Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 20 Nov 2020 12:42:24 +0000 Subject: [PATCH 024/109] #150: Add skeleton classes for remaining ICAT backend tests --- test/icat/test_isis_endpoints.py | 12 +++++ test/icat/test_session_handling.py | 24 ++++++++++ test/icat/test_standard_endpoints.py | 70 ++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 test/icat/test_isis_endpoints.py create mode 100644 test/icat/test_session_handling.py create mode 100644 test/icat/test_standard_endpoints.py diff --git a/test/icat/test_isis_endpoints.py b/test/icat/test_isis_endpoints.py new file mode 100644 index 00000000..c0dffec4 --- /dev/null +++ b/test/icat/test_isis_endpoints.py @@ -0,0 +1,12 @@ +class TestISISEndpoints: + def test_valid_get_facility_cycles_with_filters(self): + pass + + def test_valid_get_facility_cycles_count_with_filters(self): + pass + + def test_valid_get_investigations_with_filters(self): + pass + + def test_valid_get_investigations_count_with_filters(seLf): + pass diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py new file mode 100644 index 00000000..53ae6d83 --- /dev/null +++ b/test/icat/test_session_handling.py @@ -0,0 +1,24 @@ +class TestSessionHandling: + def test_session_id_decorator(self): + pass + + def test_get_valid_session_details(self): + pass + + def test_get_invalid_session_details(self): + pass + + def test_refresh_session(self): + pass + + def test_valid_login(self): + pass + + def test_invalid_login(self): + pass + + def test_valid_logout(self): + pass + + def test_invalid_logout(self): + pass diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py new file mode 100644 index 00000000..5240072d --- /dev/null +++ b/test/icat/test_standard_endpoints.py @@ -0,0 +1,70 @@ +class TestStandardEndpoints: + def test_all_endpoints_exist(self): + pass + + def test_valid_get_with_filters(self): + pass + + def test_invalid_get_with_filters(self): + # Invalid data? + pass + + def test_filters_applied_get_with_filters(self): + pass + + def test_valid_create_data(self): + pass + + def test_invalid_create_data(self): + # Invalid request body + pass + + def test_invalid_create_data_1(self): + # TODO - Rename function + # Target ICATObjectExistsError + pass + + def test_valid_update_data(self): + pass + + def test_valid_boundary_update_data(self): + """ Request body is a dictionary, not a list of dictionaries""" + pass + + def test_invalid_update_data(self): + # Exclude an ID at in one of the data pieces + pass + + def test_valid_get_one_with_filters(self): + pass + + def test_filters_applied_get_one_with_filters(self): + # Can't be a limit filter, maybe order desc would be better + pass + + def test_valid_count_with_filters(self): + pass + + def test_filters_applied_count_with_filters(self): + pass + + def test_valid_get_with_id(self): + pass + + def test_invalid_get_with_id(self): + # Do a get one with filters (order desc), extract the id of that, add 5 and do a + # request for that + pass + + def test_valid_delete_with_id(self): + pass + + def test_invalid_delete_with_id(self): + # like invalid get, but try to delete + pass + + def test_valid_update_with_id(self): + pass + + def test_invalid_update_with_id(self): + pass From ebf8b2e0e6055a7c70adb6c1ba1ebb720def7c96 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 11:37:58 +0000 Subject: [PATCH 025/109] #150: Fix failing filter tests - This adds validation on skip and limit values on the respective filters --- datagateway_api/common/filters.py | 12 +++++++++--- test/icat/filters/test_limit_filter.py | 4 +--- test/icat/filters/test_skip_filter.py | 4 +--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/datagateway_api/common/filters.py b/datagateway_api/common/filters.py index 74d06dcc..a7844d46 100644 --- a/datagateway_api/common/filters.py +++ b/datagateway_api/common/filters.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod import logging -from datagateway_api.common.exceptions import BadRequestError +from datagateway_api.common.exceptions import BadRequestError, FilterError log = logging.getLogger() @@ -55,14 +55,20 @@ class SkipFilter(QueryFilter): precedence = 3 def __init__(self, skip_value): - self.skip_value = skip_value + if skip_value >= 0: + self.skip_value = skip_value + else: + raise FilterError("The value of the skip filter must be positive") class LimitFilter(QueryFilter): precedence = 4 def __init__(self, limit_value): - self.limit_value = limit_value + if limit_value >= 0: + self.limit_value = limit_value + else: + raise FilterError("The value of the limit filter must be positive") class IncludeFilter(QueryFilter): diff --git a/test/icat/filters/test_limit_filter.py b/test/icat/filters/test_limit_filter.py index 61af6942..5599e22c 100644 --- a/test/icat/filters/test_limit_filter.py +++ b/test/icat/filters/test_limit_filter.py @@ -28,10 +28,8 @@ def test_valid_limit_value(self, icat_query, limit_value): [pytest.param(-50, id="extreme invalid"), pytest.param(-1, id="boundary")], ) def test_invalid_limit_value(self, icat_query, limit_value): - test_filter = PythonICATLimitFilter(limit_value) - with pytest.raises(FilterError): - test_filter.apply_filter(icat_query) + PythonICATLimitFilter(limit_value) @pytest.mark.parametrize( "skip_value, limit_value", diff --git a/test/icat/filters/test_skip_filter.py b/test/icat/filters/test_skip_filter.py index 9d07e690..995b19bd 100644 --- a/test/icat/filters/test_skip_filter.py +++ b/test/icat/filters/test_skip_filter.py @@ -23,7 +23,5 @@ def test_valid_skip_value(self, icat_query, skip_value): [pytest.param(-375, id="extreme invalid"), pytest.param(-1, id="boundary")], ) def test_invalid_skip_value(self, icat_query, skip_value): - test_filter = PythonICATSkipFilter(skip_value) - with pytest.raises(FilterError): - test_filter.apply_filter(icat_query) + PythonICATSkipFilter(skip_value) From a3c16e14117b1ac91c2da5ee29e59ce6b4ce6cd1 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 11:43:05 +0000 Subject: [PATCH 026/109] #150: Rename pytest fixture for ICAT data injection --- test/conftest.py | 2 +- test/icat/test_query.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index bbf99107..1cbc5b0b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -23,7 +23,7 @@ def icat_query(icat_client): @pytest.fixture() -def add_single_investigation_test_data(icat_client): +def single_investigation_test_data(icat_client): # Inject data investigation = icat_client.new("investigation") investigation.name = "Test Data for DataGateway API Testing" diff --git a/test/icat/test_query.py b/test/icat/test_query.py index e79fbf9f..7475d6fb 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -23,9 +23,8 @@ def test_invalid_query_creation(self, icat_client): with pytest.raises(PythonICATError): ICATQuery(icat_client, "User", conditions={"invalid": "invalid"}) - # @pytest.mark.usefixtures("") def test_valid_query_exeuction( - self, icat_client, add_single_investigation_test_data, + self, icat_client, single_investigation_test_data, ): test_query = ICATQuery(icat_client, "Investigation") test_data_filter = PythonICATWhereFilter( @@ -40,7 +39,7 @@ def test_valid_query_exeuction( remove_meta_attributes(entity_dict) query_output_dicts.append(entity_dict) - assert query_output_dicts == add_single_investigation_test_data + assert query_output_dicts == single_investigation_test_data def test_invalid_query_execution(self, icat_client): # Try to get ICATValidationError raised From df48005ebefbc3efa12ec1592085ff85ec0e1b55 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 16:32:00 +0000 Subject: [PATCH 027/109] #150: Correct argument case --- test/icat/test_isis_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/icat/test_isis_endpoints.py b/test/icat/test_isis_endpoints.py index c0dffec4..db39e2f1 100644 --- a/test/icat/test_isis_endpoints.py +++ b/test/icat/test_isis_endpoints.py @@ -8,5 +8,5 @@ def test_valid_get_facility_cycles_count_with_filters(self): def test_valid_get_investigations_with_filters(self): pass - def test_valid_get_investigations_count_with_filters(seLf): + def test_valid_get_investigations_count_with_filters(self): pass From 367eebb246f37ac65fb656a17abcfe7aead7a6e9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 16:38:14 +0000 Subject: [PATCH 028/109] #150: Add function to remove ICAT meta attributes and new test --- test/conftest.py | 8 ++++--- test/icat/test_query.py | 46 ++++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 1cbc5b0b..ea054dd1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,11 +1,11 @@ import uuid from icat.client import Client +from icat.entity import Entity from icat.query import Query import pytest from datagateway_api.common.config import config -from test.icat.test_query import remove_meta_attributes @pytest.fixture(scope="package") @@ -33,9 +33,11 @@ def single_investigation_test_data(icat_client): investigation.facility = icat_client.get("Facility", 1) investigation.type = icat_client.get("InvestigationType", 1) investigation.create() - investigation_dict = investigation.as_dict() - remove_meta_attributes(investigation_dict) + + meta_attributes = Entity.MetaAttr + for attr in meta_attributes: + investigation_dict.pop(attr) yield [investigation_dict] diff --git a/test/icat/test_query.py b/test/icat/test_query.py index 7475d6fb..dc2244e6 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -6,10 +6,28 @@ from datagateway_api.common.icat.query import ICATQuery -def remove_meta_attributes(entity_dict): +def prepare_icat_data_for_assertion(data): + """ + Remove meta attributes from ICAT data. Meta attributes contain data about data + creation/modification, and should be removed to ensure correct assertion values + + :param data: ICAT data containing meta attributes such as modTime + :type data: :class:`dict` or an inherited version of :class:`icat.entity.Entity` + """ + assertable_data = [] meta_attributes = Entity.MetaAttr - for attr in meta_attributes: - entity_dict.pop(attr) + + for entity in data: + # Convert to dictionary if an ICAT entity object + if isinstance(entity, Entity): + entity = entity.as_dict() + + for attr in meta_attributes: + entity.pop(attr) + + assertable_data.append(entity) + + return assertable_data class TestICATQuery: @@ -33,11 +51,7 @@ def test_valid_query_exeuction( test_data_filter.apply_filter(test_query.query) query_data = test_query.execute_query(icat_client) - query_output_dicts = [] - for entity in query_data: - entity_dict = entity.as_dict() - remove_meta_attributes(entity_dict) - query_output_dicts.append(entity_dict) + query_output_dicts = prepare_icat_data_for_assertion(query_data) assert query_output_dicts == single_investigation_test_data @@ -51,11 +65,19 @@ def test_valid_count_query_execution(self, icat_client): def test_valid_distinct_query_execution(self, icat_client): pass - def test_json_format_execution_output(self, icat_client): - pass + def test_json_format_execution_output( + self, icat_client, single_investigation_test_data, + ): + test_query = ICATQuery(icat_client, "Investigation") + test_data_filter = PythonICATWhereFilter( + "title", "Test data for the Python ICAT Backend on DataGateway API", "eq", + ) + test_data_filter.apply_filter(test_query.query) + query_data = test_query.execute_query(icat_client, True) - def test_icat_execution_output(self, icat_client): - pass + query_output_json = prepare_icat_data_for_assertion(query_data) + + assert query_output_json == single_investigation_test_data # gap in function testing From 6de60c71c7c5bd4bccd7341128c3e20c49dca481 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 16:39:18 +0000 Subject: [PATCH 029/109] #150: Add test to break a query --- test/icat/test_query.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/icat/test_query.py b/test/icat/test_query.py index dc2244e6..a0e98ffa 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -2,7 +2,10 @@ import pytest from datagateway_api.common.exceptions import PythonICATError -from datagateway_api.common.icat.filters import PythonICATWhereFilter +from datagateway_api.common.icat.filters import ( + PythonICATSkipFilter, + PythonICATWhereFilter, +) from datagateway_api.common.icat.query import ICATQuery @@ -56,8 +59,15 @@ def test_valid_query_exeuction( assert query_output_dicts == single_investigation_test_data def test_invalid_query_execution(self, icat_client): - # Try to get ICATValidationError raised - pass + test_query = ICATQuery(icat_client, "Investigation") + + # Create filter with valid value, then change to invalid value that'll cause 500 + test_skip_filter = PythonICATSkipFilter(1) + test_skip_filter.skip_value = -1 + test_skip_filter.apply_filter(test_query.query) + + with pytest.raises(PythonICATError): + test_query.execute_query(icat_client) def test_valid_count_query_execution(self, icat_client): pass From 08b8fb6ecf9ee4316af408a99d9c8fc0c75d2b68 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 16:39:55 +0000 Subject: [PATCH 030/109] #150: Add fixture to return auth header --- test/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index ea054dd1..2555793b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -15,6 +15,11 @@ def icat_client(): return client +@pytest.fixture() +def valid_credentials_header(icat_client): + return {"Authorization": f"Bearer {icat_client.sessionId}"} + + @pytest.fixture() def icat_query(icat_client): query = Query(icat_client, "Investigation") From 00923bd59227a981b18f546e897b8257cb8cf726 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 16:46:08 +0000 Subject: [PATCH 031/109] #150: Add fixture for flask test app --- test/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 2555793b..cd59e43a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,6 +6,7 @@ import pytest from datagateway_api.common.config import config +from datagateway_api.src.main import app @pytest.fixture(scope="package") @@ -51,5 +52,6 @@ def single_investigation_test_data(icat_client): @pytest.fixture() -def remove_single_investiation_result(): - pass +def flask_test_app(): + app.config["TESTING"] = True + return app.test_client() From 833f8e9787a752d7d37b4d4f13ff510ffaab0628 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 23 Nov 2020 17:25:56 +0000 Subject: [PATCH 032/109] #150: Add tests for valid GET requests - Test to see if all endpoints have been added to the API failed, hence the block comment around it --- test/icat/test_standard_endpoints.py | 102 +++++++++++++++++++++------ 1 file changed, 82 insertions(+), 20 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 5240072d..8e8a607a 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -1,17 +1,53 @@ +import pytest + +from datagateway_api.src.main import api +from datagateway_api.src.main import app +from datagateway_api.src.resources.entities.entity_map import endpoints +from test.icat.test_query import prepare_icat_data_for_assertion +from test.test_base import FlaskAppTest + + class TestStandardEndpoints: def test_all_endpoints_exist(self): - pass + """ + session_endpoint_exist = api.owns_endpoint("sessions") + assert session_endpoint_exist + + for endpoint_entity in endpoints.keys(): + get_endpoint_exist = api.owns_endpoint(endpoint_entity.lower()) + assert get_endpoint_exist - def test_valid_get_with_filters(self): + id_endpoint_exist = api.owns_endpoint( + f"{endpoint_entity.lower()}/", + ) + assert id_endpoint_exist + + count_endpoint_exist = api.owns_endpoint(f"{endpoint_entity.lower()}/count") + assert count_endpoint_exist + + findone_endpoint_exist = api.owns_endpoint( + f"{endpoint_entity.lower()}/findone", + ) + assert findone_endpoint_exist + """ pass + def test_valid_get_with_filters( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + test_response = flask_test_app.get( + '/investigations?where={"title": {"eq": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == single_investigation_test_data + def test_invalid_get_with_filters(self): # Invalid data? pass - def test_filters_applied_get_with_filters(self): - pass - def test_valid_create_data(self): pass @@ -35,21 +71,47 @@ def test_invalid_update_data(self): # Exclude an ID at in one of the data pieces pass - def test_valid_get_one_with_filters(self): - pass - - def test_filters_applied_get_one_with_filters(self): - # Can't be a limit filter, maybe order desc would be better - pass - - def test_valid_count_with_filters(self): - pass - - def test_filters_applied_count_with_filters(self): - pass - - def test_valid_get_with_id(self): - pass + def test_valid_get_one_with_filters( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + test_response = flask_test_app.get( + '/investigations/findone?where={"title": {"eq": "Test data for the Python' + ' ICAT Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + response_json = prepare_icat_data_for_assertion([test_response.json]) + + assert response_json == single_investigation_test_data + + @pytest.mark.usefixtures("single_investigation_test_data") + def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header): + test_response = flask_test_app.get( + '/investigations/count?where={"title": {"eq": "Test data for the Python' + ' ICAT Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + + assert test_response.json == 1 + + def test_valid_get_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + # Need to identify the ID given to the test data + investigation_data = flask_test_app.get( + '/investigations?where={"title": {"eq": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + test_data_id = investigation_data.json[0]["id"] + + test_response = flask_test_app.get( + f"/investigations/{test_data_id}", headers=valid_credentials_header, + ) + # Get with ID gives a dictionary response (only ever one result from that kind + # of request), so list around json is required for the call + response_json = prepare_icat_data_for_assertion([test_response.json]) + + assert response_json == single_investigation_test_data def test_invalid_get_with_id(self): # Do a get one with filters (order desc), extract the id of that, add 5 and do a From 9e0c56b93e458b89bc366130c0d3dc7d0813b72b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 11:44:32 +0000 Subject: [PATCH 033/109] #150: Add update test - Added some logging to the API to make logging a bit better --- datagateway_api/common/icat/helpers.py | 1 + test/icat/test_standard_endpoints.py | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/datagateway_api/common/icat/helpers.py b/datagateway_api/common/icat/helpers.py index b3297fb7..e94e85b7 100644 --- a/datagateway_api/common/icat/helpers.py +++ b/datagateway_api/common/icat/helpers.py @@ -171,6 +171,7 @@ def update_attributes(old_entity, new_entity): - typically if Python ICAT doesn't allow an attribute to be edited (e.g. modId & modTime) """ + log.debug("Updating entity attributes: %s", list(new_entity.keys())) for key in new_entity: try: original_data_attribute = getattr(old_entity, key) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 8e8a607a..a48d177d 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -125,8 +125,26 @@ def test_invalid_delete_with_id(self): # like invalid get, but try to delete pass - def test_valid_update_with_id(self): - pass + def test_valid_update_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" + + update_data_json = { + "id": single_investigation_test_data[0]["id"], + "doi": expected_doi, + "summary": expected_summary, + } + single_investigation_test_data[0]["doi"] = expected_doi + single_investigation_test_data[0]["summary"] = expected_summary + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_json, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == single_investigation_test_data def test_invalid_update_with_id(self): pass From 9c3ece577a9610792e58180fadaff6e7b6ccd9d9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 11:45:25 +0000 Subject: [PATCH 034/109] #150: Add fixture to inject multiple investigation results - Add test to utilise said fixture - Add invalid test --- test/conftest.py | 30 ++++++++++++++++++++++++ test/icat/test_standard_endpoints.py | 35 ++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index cd59e43a..6d7950d0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -51,6 +51,36 @@ def single_investigation_test_data(icat_client): icat_client.delete(investigation) +@pytest.fixture() +def multiple_investigation_test_data(icat_client): + investigation_test_data = [] + investigation_dicts = [] + meta_attributes = Entity.MetaAttr + + for i in range(5): + investigation = icat_client.new("investigation") + investigation.name = f"Test Data for DataGateway API Testing {i}" + investigation.title = ( + f"Test data for the Python ICAT Backend on DataGateway API {i}" + ) + investigation.visitId = str(uuid.uuid1()) + investigation.facility = icat_client.get("Facility", 1) + investigation.type = icat_client.get("InvestigationType", 1) + investigation.create() + investigation_test_data.append(investigation) + investigation_dict = investigation.as_dict() + + for attr in meta_attributes: + investigation_dict.pop(attr) + + investigation_dicts.append(investigation_dict) + + yield investigation_dicts + + for entity in investigation_test_data: + icat_client.delete(entity) + + @pytest.fixture() def flask_test_app(): app.config["TESTING"] = True diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index a48d177d..5a76945d 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -44,6 +44,26 @@ def test_valid_get_with_filters( assert response_json == single_investigation_test_data + @pytest.mark.usefixtures("multiple_investigation_test_data") + def test_valid_get_with_filters_distinct( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations?where={"title": {"like": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}&distinct="title"', + headers=valid_credentials_header, + ) + + expected = [ + { + "title": f"Test data for the Python ICAT Backend on DataGateway API {i}" + for i in range(5) + }, + ] + + for title in expected: + assert title in test_response.json + def test_invalid_get_with_filters(self): # Invalid data? pass @@ -113,10 +133,21 @@ def test_valid_get_with_id( assert response_json == single_investigation_test_data - def test_invalid_get_with_id(self): + def test_invalid_get_with_id(self, flask_test_app, valid_credentials_header): # Do a get one with filters (order desc), extract the id of that, add 5 and do a # request for that - pass + # Need to identify the ID given to the test data + final_investigation_result = flask_test_app.get( + '/investigations/findone?order="id DESC"', headers=valid_credentials_header, + ) + test_data_id = final_investigation_result.json["id"] + + # Adding 100 onto the ID to the most recent result should ensure a 404 + test_response = flask_test_app.get( + f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 def test_valid_delete_with_id(self): pass From fdafb7a79764919286a959cdc83fd9168158c326 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 16:43:57 +0000 Subject: [PATCH 035/109] #150: Add test for skip & limit filter merge --- test/icat/test_standard_endpoints.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 5a76945d..2fc6be56 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -64,6 +64,33 @@ def test_valid_get_with_filters_distinct( for title in expected: assert title in test_response.json + def test_limit_skip_merge_get_with_filters( + self, + flask_test_app, + valid_credentials_header, + multiple_investigation_test_data, + ): + skip_value = 1 + limit_value = 2 + + test_response = flask_test_app.get( + '/investigations?where={"title": {"like": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}' + f'&skip={skip_value}&limit={limit_value}&order="id ASC"', + headers=valid_credentials_header, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + filtered_investigation_data = [] + filter_count = 0 + while filter_count < limit_value: + filtered_investigation_data.append( + multiple_investigation_test_data.pop(skip_value), + ) + filter_count += 1 + + assert response_json == filtered_investigation_data + def test_invalid_get_with_filters(self): # Invalid data? pass From fc7f3a15063bedd11cffd9366104c85e60ac13aa Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 17:01:23 +0000 Subject: [PATCH 036/109] #150: Add tests for PATCH endpoints - Single and multiple update tests --- test/icat/test_standard_endpoints.py | 57 ++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 2fc6be56..6381ea8e 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -91,10 +91,6 @@ def test_limit_skip_merge_get_with_filters( assert response_json == filtered_investigation_data - def test_invalid_get_with_filters(self): - # Invalid data? - pass - def test_valid_create_data(self): pass @@ -107,12 +103,57 @@ def test_invalid_create_data_1(self): # Target ICATObjectExistsError pass - def test_valid_update_data(self): - pass + def test_valid_multiple_update_data( + self, + flask_test_app, + valid_credentials_header, + multiple_investigation_test_data, + ): + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" - def test_valid_boundary_update_data(self): + update_data_list = [] + + for investigation in multiple_investigation_test_data: + investigation["doi"] = expected_doi + investigation["summary"] = expected_summary + + update_entity = { + "id": investigation["id"], + "doi": expected_doi, + "summary": expected_summary, + } + update_data_list.append(update_entity) + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_list, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == multiple_investigation_test_data + + def test_valid_boundary_update_data( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): """ Request body is a dictionary, not a list of dictionaries""" - pass + + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" + + update_data_json = { + "id": single_investigation_test_data[0]["id"], + "doi": expected_doi, + "summary": expected_summary, + } + single_investigation_test_data[0]["doi"] = expected_doi + single_investigation_test_data[0]["summary"] = expected_summary + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_json, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == single_investigation_test_data def test_invalid_update_data(self): # Exclude an ID at in one of the data pieces From 510943e94f9dfa9eae5aabdc61a7221064db054a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 17:10:18 +0000 Subject: [PATCH 037/109] #150: Add invalid test for data updates --- test/icat/test_standard_endpoints.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 6381ea8e..b39c1dee 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -1,5 +1,6 @@ import pytest +from datagateway_api.common.exceptions import BadRequestError from datagateway_api.src.main import api from datagateway_api.src.main import app from datagateway_api.src.resources.entities.entity_map import endpoints @@ -155,9 +156,26 @@ def test_valid_boundary_update_data( assert response_json == single_investigation_test_data - def test_invalid_update_data(self): - # Exclude an ID at in one of the data pieces - pass + def test_invalid_update_data( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """There should be an ID in the request body to know which entity to update""" + + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" + + update_data_json = { + "doi": expected_doi, + "summary": expected_summary, + } + single_investigation_test_data[0]["doi"] = expected_doi + single_investigation_test_data[0]["summary"] = expected_summary + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_json, + ) + + assert test_response.status_code == 400 def test_valid_get_one_with_filters( self, flask_test_app, valid_credentials_header, single_investigation_test_data, From 6477491be0f8f3d90cb4620c7f8b9764a9ec6eda Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 17:32:22 +0000 Subject: [PATCH 038/109] #150: Modify existing test to actually point to an update by ID endpoint - I got the function names mixed up when originally doing that function... --- test/icat/test_standard_endpoints.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index b39c1dee..c90b3cce 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -1,6 +1,5 @@ import pytest -from datagateway_api.common.exceptions import BadRequestError from datagateway_api.src.main import api from datagateway_api.src.main import app from datagateway_api.src.resources.entities.entity_map import endpoints @@ -249,7 +248,6 @@ def test_valid_update_with_id( expected_summary = "Test Summary" update_data_json = { - "id": single_investigation_test_data[0]["id"], "doi": expected_doi, "summary": expected_summary, } @@ -257,9 +255,11 @@ def test_valid_update_with_id( single_investigation_test_data[0]["summary"] = expected_summary test_response = flask_test_app.patch( - "/investigations", headers=valid_credentials_header, json=update_data_json, + f"/investigations/{single_investigation_test_data[0]['id']}", + headers=valid_credentials_header, + json=update_data_json, ) - response_json = prepare_icat_data_for_assertion(test_response.json) + response_json = prepare_icat_data_for_assertion([test_response.json]) assert response_json == single_investigation_test_data From 451e4d713db29f3b3a160fc827c2526fa70f3f1c Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 18:15:43 +0000 Subject: [PATCH 039/109] #150: Add invalid update by ID test - Currently fails, request gets a 500, not a 400 despite the user input actually being the issue --- test/icat/test_standard_endpoints.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index c90b3cce..ec604b63 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -263,5 +263,23 @@ def test_valid_update_with_id( assert response_json == single_investigation_test_data - def test_invalid_update_with_id(self): - pass + def test_invalid_update_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """This test will attempt to put `icatdb` into an invalid state""" + + # DOI cannot be over 255 characters, which this string is + invalid_update_json = { + "doi": "__________________________________________________________________" + "_________________________________________________________________________" + "_________________________________________________________________________" + "_________________________________________________________________________", + } + + test_response = flask_test_app.patch( + f"/investigations/{single_investigation_test_data[0]['id']}", + headers=valid_credentials_header, + json=invalid_update_json, + ) + + assert test_response.status_code == 400 From ba62e445df879fa7b55eb3c5e1cd7c3487b7fb8a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 24 Nov 2020 18:16:43 +0000 Subject: [PATCH 040/109] #150: Change ICATValidationError to raise a BadRequestError - An exception raised as an alternative to the database going into an invalid state will usually be due to invalid user input - This fixes the failing test created in the previous commit --- datagateway_api/common/icat/helpers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/datagateway_api/common/icat/helpers.py b/datagateway_api/common/icat/helpers.py index e94e85b7..def49b29 100644 --- a/datagateway_api/common/icat/helpers.py +++ b/datagateway_api/common/icat/helpers.py @@ -193,8 +193,10 @@ def update_attributes(old_entity, new_entity): try: old_entity.update() - except (ICATValidationError, ICATInternalError) as e: + except ICATInternalError as e: raise PythonICATError(e) + except ICATValidationError as e: + raise BadRequestError(e) def get_entity_by_id( @@ -474,9 +476,9 @@ def create_entities(client, entity_type, data): try: new_entity.create() - except (ICATValidationError, ICATInternalError) as e: + except ICATInternalError as e: raise PythonICATError(e) - except (ICATObjectExistsError, ICATParameterError) as e: + except (ICATObjectExistsError, ICATParameterError, ICATValidationError) as e: raise BadRequestError(e) created_data.append(get_entity_by_id(client, entity_type, new_entity.id, True)) From bead8cfd5a10d2214601a57cb25ae7f8fc983c45 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 09:18:02 +0000 Subject: [PATCH 041/109] #150: Add more invalid tests for update endpoint --- test/icat/test_standard_endpoints.py | 39 ++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index ec604b63..8f0eceae 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -155,20 +155,15 @@ def test_valid_boundary_update_data( assert response_json == single_investigation_test_data - def test_invalid_update_data( + def test_invalid_missing_update_data( self, flask_test_app, valid_credentials_header, single_investigation_test_data, ): """There should be an ID in the request body to know which entity to update""" - expected_doi = "Test Data Identifier" - expected_summary = "Test Summary" - update_data_json = { - "doi": expected_doi, - "summary": expected_summary, + "doi": "Test Data Identifier", + "summary": "Test Summary", } - single_investigation_test_data[0]["doi"] = expected_doi - single_investigation_test_data[0]["summary"] = expected_summary test_response = flask_test_app.patch( "/investigations", headers=valid_credentials_header, json=update_data_json, @@ -176,6 +171,34 @@ def test_invalid_update_data( assert test_response.status_code == 400 + @pytest.mark.parametrize( + "update_key, update_value", + [ + pytest.param("invalidAttr", "Some Value", id="invalid attribute"), + pytest.param("modId", "simple/root", id="meta attribute update"), + ], + ) + def test_invalid_attribute_update( + self, + flask_test_app, + valid_credentials_header, + single_investigation_test_data, + update_key, + update_value, + ): + invalid_update_data_json = { + "id": single_investigation_test_data[0]["id"], + update_key: update_value, + } + + test_response = flask_test_app.patch( + "/investigations", + headers=valid_credentials_header, + json=invalid_update_data_json, + ) + + assert test_response.status_code == 400 + def test_valid_get_one_with_filters( self, flask_test_app, valid_credentials_header, single_investigation_test_data, ): From a2fd64413e3057a60ad6df4e5cbfa3304bd78c6f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 11:12:39 +0000 Subject: [PATCH 042/109] #150: Add tests for no result requests - This was trying to cover 404s but I'm not sure how I'd get a count query to 404.. --- test/icat/test_standard_endpoints.py | 35 +++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 8f0eceae..9c4979c8 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -44,6 +44,17 @@ def test_valid_get_with_filters( assert response_json == single_investigation_test_data + def test_valid_no_results_get_with_filters( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations?where={"title": {"eq": "This filter should cause a 404 for' + 'testing purposes..."}}', + headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 + @pytest.mark.usefixtures("multiple_investigation_test_data") def test_valid_get_with_filters_distinct( self, flask_test_app, valid_credentials_header, @@ -199,7 +210,7 @@ def test_invalid_attribute_update( assert test_response.status_code == 400 - def test_valid_get_one_with_filters( + def test_valid_findone_with_filters( self, flask_test_app, valid_credentials_header, single_investigation_test_data, ): test_response = flask_test_app.get( @@ -211,6 +222,17 @@ def test_valid_get_one_with_filters( assert response_json == single_investigation_test_data + def test_valid_no_results_findone_with_filters( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations/findone?where={"title": {"eq": "This filter should cause a' + '404 for testing purposes..."}}', + headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 + @pytest.mark.usefixtures("single_investigation_test_data") def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header): test_response = flask_test_app.get( @@ -221,6 +243,17 @@ def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header assert test_response.json == 1 + def test_valid_no_results_count_with_filters( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations/count?where={"title": {"eq": "This filter should cause a' + '404 for testing purposes..."}}', + headers=valid_credentials_header, + ) + + assert test_response.json == 0 + def test_valid_get_with_id( self, flask_test_app, valid_credentials_header, single_investigation_test_data, ): From 03c7c8fd8230b2e15e03e723d3c39630ff45970e Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 11:52:45 +0000 Subject: [PATCH 043/109] #150: Add tests for DELETE endpoints --- test/conftest.py | 7 +++++- test/icat/test_standard_endpoints.py | 33 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 6d7950d0..9b89af78 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,7 @@ from icat.client import Client from icat.entity import Entity +from icat.exception import ICATNoObjectError from icat.query import Query import pytest @@ -48,7 +49,11 @@ def single_investigation_test_data(icat_client): yield [investigation_dict] # Remove data from ICAT - icat_client.delete(investigation) + try: + icat_client.delete(investigation) + except ICATNoObjectError as e: + # This should occur on DELETE endpoints, normal behaviour for those tests + print(e) @pytest.fixture() diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 9c4979c8..6c048486 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -275,9 +275,8 @@ def test_valid_get_with_id( assert response_json == single_investigation_test_data def test_invalid_get_with_id(self, flask_test_app, valid_credentials_header): - # Do a get one with filters (order desc), extract the id of that, add 5 and do a - # request for that - # Need to identify the ID given to the test data + """Request with a non-existent ID""" + final_investigation_result = flask_test_app.get( '/investigations/findone?order="id DESC"', headers=valid_credentials_header, ) @@ -290,12 +289,30 @@ def test_invalid_get_with_id(self, flask_test_app, valid_credentials_header): assert test_response.status_code == 404 - def test_valid_delete_with_id(self): - pass + def test_valid_delete_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + test_response = flask_test_app.delete( + f'/investigations/{single_investigation_test_data[0]["id"]}', + headers=valid_credentials_header, + ) - def test_invalid_delete_with_id(self): - # like invalid get, but try to delete - pass + assert test_response.status_code == 204 + + def test_invalid_delete_with_id(self, flask_test_app, valid_credentials_header): + """Request with a non-existent ID""" + + final_investigation_result = flask_test_app.get( + '/investigations/findone?order="id DESC"', headers=valid_credentials_header, + ) + test_data_id = final_investigation_result.json["id"] + + # Adding 100 onto the ID to the most recent result should ensure a 404 + test_response = flask_test_app.delete( + f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 def test_valid_update_with_id( self, flask_test_app, valid_credentials_header, single_investigation_test_data, From 9edd0e92e2b833d25fb9afd24e08a31091d5b195 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 14:05:56 +0000 Subject: [PATCH 044/109] #150: Add pytest plugin to improve assertion diffs --- poetry.lock | 41 ++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 9feb72a6..037b6641 100644 --- a/poetry.lock +++ b/poetry.lock @@ -373,6 +373,14 @@ python-versions = ">=3.4" [package.dependencies] gitdb = ">=4.0.1,<5" +[[package]] +name = "icdiff" +version = "1.9.1" +description = "improved colored diff" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "idna" version = "2.10" @@ -511,6 +519,14 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] +[[package]] +name = "pprintpp" +version = "0.4.0" +description = "A drop-in replacement for pprint that's actually pretty" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "py" version = "1.9.0" @@ -592,6 +608,19 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-icdiff" +version = "0.5" +description = "use icdiff for better error messages in pytest assertions" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +icdiff = "*" +pprintpp = "*" +pytest = "*" + [[package]] name = "python-dateutil" version = "2.8.1" @@ -787,7 +816,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "d3190c53f5cec339b2b4559998fb054cf6c3c9c5dd61caacde0dcb8b7a887863" +content-hash = "8ce6140731b2c2e9d2ba4f591ee85ca6d805851acba95169562ba1311a252ac5" [metadata.files] aniso8601 = [ @@ -942,6 +971,9 @@ gitpython = [ {file = "GitPython-3.1.11-py3-none-any.whl", hash = "sha256:6eea89b655917b500437e9668e4a12eabdcf00229a0df1762aabd692ef9b746b"}, {file = "GitPython-3.1.11.tar.gz", hash = "sha256:befa4d101f91bad1b632df4308ec64555db684c360bd7d2130b4807d49ce86b8"}, ] +icdiff = [ + {file = "icdiff-1.9.1.tar.gz", hash = "sha256:66972dd03318da55280991db375d3ef6b66d948c67af96c1ebdb21587e86655e"}, +] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, @@ -1025,6 +1057,10 @@ pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] +pprintpp = [ + {file = "pprintpp-0.4.0-py2.py3-none-any.whl", hash = "sha256:b6b4dcdd0c0c0d75e4d7b2f21a9e933e5b2ce62b26e1a54537f9651ae5a5c01d"}, + {file = "pprintpp-0.4.0.tar.gz", hash = "sha256:ea826108e2c7f49dc6d66c752973c3fc9749142a798d6b254e1e301cfdbc6403"}, +] py = [ {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, @@ -1053,6 +1089,9 @@ pytest-cov = [ {file = "pytest-cov-2.10.1.tar.gz", hash = "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e"}, {file = "pytest_cov-2.10.1-py2.py3-none-any.whl", hash = "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191"}, ] +pytest-icdiff = [ + {file = "pytest-icdiff-0.5.tar.gz", hash = "sha256:3a14097f4385665cb04330e6ae09a3dd430375f717e94482af6944470ad5f100"}, +] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, diff --git a/pyproject.toml b/pyproject.toml index b26eac75..5fea5a14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ pep8-naming = "^0.11.1" pytest = "^6.1.2" coverage = {extras = ["toml"], version = "^5.3"} pytest-cov = "^2.10.1" +pytest-icdiff = "^0.5" [tool.poetry.scripts] datagateway-api = "datagateway_api.src.main:run_api" From ce048da69af5012775cfffbc00563e4e062ffa41 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 14:06:44 +0000 Subject: [PATCH 045/109] #150: Fix data creation --- datagateway_api/common/icat/helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datagateway_api/common/icat/helpers.py b/datagateway_api/common/icat/helpers.py index def49b29..2ac6742d 100644 --- a/datagateway_api/common/icat/helpers.py +++ b/datagateway_api/common/icat/helpers.py @@ -445,9 +445,7 @@ def create_entities(client, entity_type, data): for result in data: new_entity = client.new( - get_icat_entity_name_as_camel_case( - client, entity_type, camel_case_output=True, - ), + get_icat_entity_name_as_camel_case(client, entity_type), ) for attribute_name, value in result.items(): From e358ec7d8b37498e9c422ba7bdf46a1dd7e38b10 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 14:07:32 +0000 Subject: [PATCH 046/109] #150: Add valid test for data creation --- test/icat/test_query.py | 6 ++- test/icat/test_standard_endpoints.py | 60 +++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/test/icat/test_query.py b/test/icat/test_query.py index a0e98ffa..8e0717c0 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -9,7 +9,7 @@ from datagateway_api.common.icat.query import ICATQuery -def prepare_icat_data_for_assertion(data): +def prepare_icat_data_for_assertion(data, remove_id=False): """ Remove meta attributes from ICAT data. Meta attributes contain data about data creation/modification, and should be removed to ensure correct assertion values @@ -28,6 +28,10 @@ def prepare_icat_data_for_assertion(data): for attr in meta_attributes: entity.pop(attr) + # meta_attributes is immutable + if remove_id: + entity.pop("id") + assertable_data.append(entity) return assertable_data diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 6c048486..296c0a69 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -102,8 +102,64 @@ def test_limit_skip_merge_get_with_filters( assert response_json == filtered_investigation_data - def test_valid_create_data(self): - pass + def test_valid_create_data(self, flask_test_app, valid_credentials_header): + create_investigation_json = [ + { + "name": "Test Data for API Testing, Data Creation 1", + "title": "Test data for the Python ICAT Backend on DataGateway API", + "summary": "Test data for DataGateway API testing", + "releaseDate": "2020-03-03 08:00:08", + "startDate": "2020-02-02 09:00:09", + "endDate": "2020-02-03 10:00:10", + "visitId": "Data Creation Visit", + "doi": "DataGateway API Test DOI", + "facility": 1, + "type": 1, + }, + { + "name": "Test Data for API Testing, Data Creation 2", + "title": "Test data for the Python ICAT Backend on DataGateway API", + "summary": "Test data for DataGateway API testing", + "releaseDate": "2020-03-03 08:00:08", + "startDate": "2020-02-02 09:00:09", + "endDate": "2020-02-03 10:00:10", + "visitId": "Data Creation Visit", + "doi": "DataGateway API Test DOI", + "facility": 1, + "type": 1, + }, + ] + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=create_investigation_json, + ) + + test_data_ids = [] + for investigation_request, investigation_response in zip( + create_investigation_json, test_response.json, + ): + investigation_request.pop("facility") + investigation_request.pop("type") + test_data_ids.append(investigation_response["id"]) + + response_json = prepare_icat_data_for_assertion( + test_response.json, remove_id=True, + ) + + assert create_investigation_json == response_json + + # Delete the entities created by this test + for investigation_id in test_data_ids: + delete_test_data = flask_test_app.delete( + f"/investigations/{investigation_id}", headers=valid_credentials_header, + ) + + # This test isn't testing DELETE requests but this will easily signpost if + # there's an issue here (which will inevitably impact tests in the same + # session) + assert delete_test_data.status_code == 204 def test_invalid_create_data(self): # Invalid request body From e1f07b7362161ff5300068d95cbff9b4dcac7c68 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 14:46:19 +0000 Subject: [PATCH 047/109] #150: Add invalid tests for data creation --- test/icat/test_standard_endpoints.py | 41 +++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index 296c0a69..ba02d39b 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -165,10 +165,43 @@ def test_invalid_create_data(self): # Invalid request body pass - def test_invalid_create_data_1(self): - # TODO - Rename function - # Target ICATObjectExistsError - pass + def test_invalid_create_data(self, flask_test_app, valid_credentials_header): + """An investigation requires a minimum of: name, visitId, facility, type""" + + invalid_request_body = { + "title": "Test Title for DataGateway API Backend testing", + } + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=invalid_request_body, + ) + + assert test_response.status_code == 400 + + def test_invalid_existing_data_create( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """This test targets raising ICATObjectExistsError, causing a 400""" + + # entity.as_dict() removes details about facility and type, hence they're + # hardcoded here instead of using sinle_investigation_test_data + existing_object_json = { + "name": single_investigation_test_data[0]["name"], + "title": single_investigation_test_data[0]["title"], + "visitId": single_investigation_test_data[0]["visitId"], + "facility": 1, + "type": 1, + } + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=existing_object_json, + ) + + assert test_response.status_code == 400 def test_valid_multiple_update_data( self, From b16098e4c4f2663d0beac1202d0763863397359b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 15:13:15 +0000 Subject: [PATCH 048/109] #150: Add test to create single investigation - Also remove assertion for data deletion in these tests - I've changed the name of the data used in these tests so they don't have an impact on later tests, so there's no need to highlight any failures in data deletion --- test/icat/test_standard_endpoints.py | 51 +++++++++++++++++++++------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index ba02d39b..ccb8f274 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -103,7 +103,7 @@ def test_limit_skip_merge_get_with_filters( assert response_json == filtered_investigation_data def test_valid_create_data(self, flask_test_app, valid_credentials_header): - create_investigation_json = [ + create_investigations_json = [ { "name": "Test Data for API Testing, Data Creation 1", "title": "Test data for the Python ICAT Backend on DataGateway API", @@ -133,12 +133,12 @@ def test_valid_create_data(self, flask_test_app, valid_credentials_header): test_response = flask_test_app.post( "/investigations", headers=valid_credentials_header, - json=create_investigation_json, + json=create_investigations_json, ) test_data_ids = [] for investigation_request, investigation_response in zip( - create_investigation_json, test_response.json, + create_investigations_json, test_response.json, ): investigation_request.pop("facility") investigation_request.pop("type") @@ -148,22 +148,49 @@ def test_valid_create_data(self, flask_test_app, valid_credentials_header): test_response.json, remove_id=True, ) - assert create_investigation_json == response_json + assert create_investigations_json == response_json # Delete the entities created by this test for investigation_id in test_data_ids: - delete_test_data = flask_test_app.delete( + flask_test_app.delete( f"/investigations/{investigation_id}", headers=valid_credentials_header, ) - # This test isn't testing DELETE requests but this will easily signpost if - # there's an issue here (which will inevitably impact tests in the same - # session) - assert delete_test_data.status_code == 204 + def test_valid_boundary_create_data(self, flask_test_app, valid_credentials_header): + """Create a single investigation, as opposed to multiple""" + + create_investigation_json = { + "name": "Test Data for API Testing, Data Creation 0", + "title": "Test data for the Python ICAT Backend on the API", + "summary": "Test data for DataGateway API testing", + "releaseDate": "2020-03-03 08:00:08", + "startDate": "2020-02-02 09:00:09", + "endDate": "2020-02-03 10:00:10", + "visitId": "Data Creation Visit", + "doi": "DataGateway API Test DOI", + "facility": 1, + "type": 1, + } - def test_invalid_create_data(self): - # Invalid request body - pass + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=create_investigation_json, + ) + + create_investigation_json.pop("facility") + create_investigation_json.pop("type") + created_test_data_id = test_response.json[0]["id"] + + response_json = prepare_icat_data_for_assertion( + test_response.json, remove_id=True, + ) + + assert [create_investigation_json] == response_json + + flask_test_app.delete( + f"/investigations/{created_test_data_id}", headers=valid_credentials_header, + ) def test_invalid_create_data(self, flask_test_app, valid_credentials_header): """An investigation requires a minimum of: name, visitId, facility, type""" From ba5992645411fd16a12d4432c22e083dc3d2256b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 25 Nov 2020 17:08:30 +0000 Subject: [PATCH 049/109] #150: Move endpoint tests to their own files/classes --- .../icat/endpoints/test_count_with_filters.py | 24 + test/icat/endpoints/test_create.py | 131 ++++++ test/icat/endpoints/test_delete_by_id.py | 25 + test/icat/endpoints/test_findone.py | 26 + test/icat/endpoints/test_get_by_id.py | 38 ++ test/icat/endpoints/test_get_with_filters.py | 75 +++ test/icat/endpoints/test_update_by_id.py | 46 ++ test/icat/endpoints/test_update_multiple.py | 101 ++++ test/icat/test_standard_endpoints.py | 444 ------------------ 9 files changed, 466 insertions(+), 444 deletions(-) create mode 100644 test/icat/endpoints/test_count_with_filters.py create mode 100644 test/icat/endpoints/test_create.py create mode 100644 test/icat/endpoints/test_delete_by_id.py create mode 100644 test/icat/endpoints/test_findone.py create mode 100644 test/icat/endpoints/test_get_by_id.py create mode 100644 test/icat/endpoints/test_get_with_filters.py create mode 100644 test/icat/endpoints/test_update_by_id.py create mode 100644 test/icat/endpoints/test_update_multiple.py diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters.py new file mode 100644 index 00000000..3c634af3 --- /dev/null +++ b/test/icat/endpoints/test_count_with_filters.py @@ -0,0 +1,24 @@ +import pytest + + +class TestCountWithFilters: + @pytest.mark.usefixtures("single_investigation_test_data") + def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header): + test_response = flask_test_app.get( + '/investigations/count?where={"title": {"eq": "Test data for the Python' + ' ICAT Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + + assert test_response.json == 1 + + def test_valid_no_results_count_with_filters( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations/count?where={"title": {"eq": "This filter should cause a' + '404 for testing purposes..."}}', + headers=valid_credentials_header, + ) + + assert test_response.json == 0 diff --git a/test/icat/endpoints/test_create.py b/test/icat/endpoints/test_create.py new file mode 100644 index 00000000..04f8c39e --- /dev/null +++ b/test/icat/endpoints/test_create.py @@ -0,0 +1,131 @@ +from test.icat.test_query import prepare_icat_data_for_assertion + + +class TestCreateData: + def test_valid_create_data(self, flask_test_app, valid_credentials_header): + create_investigations_json = [ + { + "name": "Test Data for API Testing, Data Creation 1", + "title": "Test data for the Python ICAT Backend on DataGateway API", + "summary": "Test data for DataGateway API testing", + "releaseDate": "2020-03-03 08:00:08", + "startDate": "2020-02-02 09:00:09", + "endDate": "2020-02-03 10:00:10", + "visitId": "Data Creation Visit", + "doi": "DataGateway API Test DOI", + "facility": 1, + "type": 1, + }, + { + "name": "Test Data for API Testing, Data Creation 2", + "title": "Test data for the Python ICAT Backend on DataGateway API", + "summary": "Test data for DataGateway API testing", + "releaseDate": "2020-03-03 08:00:08", + "startDate": "2020-02-02 09:00:09", + "endDate": "2020-02-03 10:00:10", + "visitId": "Data Creation Visit", + "doi": "DataGateway API Test DOI", + "facility": 1, + "type": 1, + }, + ] + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=create_investigations_json, + ) + + test_data_ids = [] + for investigation_request, investigation_response in zip( + create_investigations_json, test_response.json, + ): + investigation_request.pop("facility") + investigation_request.pop("type") + test_data_ids.append(investigation_response["id"]) + + response_json = prepare_icat_data_for_assertion( + test_response.json, remove_id=True, + ) + + assert create_investigations_json == response_json + + # Delete the entities created by this test + for investigation_id in test_data_ids: + flask_test_app.delete( + f"/investigations/{investigation_id}", headers=valid_credentials_header, + ) + + def test_valid_boundary_create_data(self, flask_test_app, valid_credentials_header): + """Create a single investigation, as opposed to multiple""" + + create_investigation_json = { + "name": "Test Data for API Testing, Data Creation 0", + "title": "Test data for the Python ICAT Backend on the API", + "summary": "Test data for DataGateway API testing", + "releaseDate": "2020-03-03 08:00:08", + "startDate": "2020-02-02 09:00:09", + "endDate": "2020-02-03 10:00:10", + "visitId": "Data Creation Visit", + "doi": "DataGateway API Test DOI", + "facility": 1, + "type": 1, + } + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=create_investigation_json, + ) + + create_investigation_json.pop("facility") + create_investigation_json.pop("type") + created_test_data_id = test_response.json[0]["id"] + + response_json = prepare_icat_data_for_assertion( + test_response.json, remove_id=True, + ) + + assert [create_investigation_json] == response_json + + flask_test_app.delete( + f"/investigations/{created_test_data_id}", headers=valid_credentials_header, + ) + + def test_invalid_create_data(self, flask_test_app, valid_credentials_header): + """An investigation requires a minimum of: name, visitId, facility, type""" + + invalid_request_body = { + "title": "Test Title for DataGateway API Backend testing", + } + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=invalid_request_body, + ) + + assert test_response.status_code == 400 + + def test_invalid_existing_data_create( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """This test targets raising ICATObjectExistsError, causing a 400""" + + # entity.as_dict() removes details about facility and type, hence they're + # hardcoded here instead of using sinle_investigation_test_data + existing_object_json = { + "name": single_investigation_test_data[0]["name"], + "title": single_investigation_test_data[0]["title"], + "visitId": single_investigation_test_data[0]["visitId"], + "facility": 1, + "type": 1, + } + + test_response = flask_test_app.post( + "/investigations", + headers=valid_credentials_header, + json=existing_object_json, + ) + + assert test_response.status_code == 400 diff --git a/test/icat/endpoints/test_delete_by_id.py b/test/icat/endpoints/test_delete_by_id.py new file mode 100644 index 00000000..07d8ff0d --- /dev/null +++ b/test/icat/endpoints/test_delete_by_id.py @@ -0,0 +1,25 @@ +class TestDeleteByID: + def test_valid_delete_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + test_response = flask_test_app.delete( + f'/investigations/{single_investigation_test_data[0]["id"]}', + headers=valid_credentials_header, + ) + + assert test_response.status_code == 204 + + def test_invalid_delete_with_id(self, flask_test_app, valid_credentials_header): + """Request with a non-existent ID""" + + final_investigation_result = flask_test_app.get( + '/investigations/findone?order="id DESC"', headers=valid_credentials_header, + ) + test_data_id = final_investigation_result.json["id"] + + # Adding 100 onto the ID to the most recent result should ensure a 404 + test_response = flask_test_app.delete( + f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_findone.py b/test/icat/endpoints/test_findone.py new file mode 100644 index 00000000..84d3f830 --- /dev/null +++ b/test/icat/endpoints/test_findone.py @@ -0,0 +1,26 @@ +from test.icat.test_query import prepare_icat_data_for_assertion + + +class TestFindone: + def test_valid_findone_with_filters( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + test_response = flask_test_app.get( + '/investigations/findone?where={"title": {"eq": "Test data for the Python' + ' ICAT Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + response_json = prepare_icat_data_for_assertion([test_response.json]) + + assert response_json == single_investigation_test_data + + def test_valid_no_results_findone_with_filters( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations/findone?where={"title": {"eq": "This filter should cause a' + '404 for testing purposes..."}}', + headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id.py new file mode 100644 index 00000000..24b962c1 --- /dev/null +++ b/test/icat/endpoints/test_get_by_id.py @@ -0,0 +1,38 @@ +from test.icat.test_query import prepare_icat_data_for_assertion + + +class TestGetByID: + def test_valid_get_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + # Need to identify the ID given to the test data + investigation_data = flask_test_app.get( + '/investigations?where={"title": {"eq": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + test_data_id = investigation_data.json[0]["id"] + + test_response = flask_test_app.get( + f"/investigations/{test_data_id}", headers=valid_credentials_header, + ) + # Get with ID gives a dictionary response (only ever one result from that kind + # of request), so list around json is required for the call + response_json = prepare_icat_data_for_assertion([test_response.json]) + + assert response_json == single_investigation_test_data + + def test_invalid_get_with_id(self, flask_test_app, valid_credentials_header): + """Request with a non-existent ID""" + + final_investigation_result = flask_test_app.get( + '/investigations/findone?order="id DESC"', headers=valid_credentials_header, + ) + test_data_id = final_investigation_result.json["id"] + + # Adding 100 onto the ID to the most recent result should ensure a 404 + test_response = flask_test_app.get( + f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_get_with_filters.py b/test/icat/endpoints/test_get_with_filters.py new file mode 100644 index 00000000..46fccf4c --- /dev/null +++ b/test/icat/endpoints/test_get_with_filters.py @@ -0,0 +1,75 @@ +import pytest + +from test.icat.test_query import prepare_icat_data_for_assertion + + +class TestGetWithFilters: + def test_valid_get_with_filters( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + test_response = flask_test_app.get( + '/investigations?where={"title": {"eq": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}', + headers=valid_credentials_header, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == single_investigation_test_data + + def test_valid_no_results_get_with_filters( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations?where={"title": {"eq": "This filter should cause a 404 for' + 'testing purposes..."}}', + headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 + + @pytest.mark.usefixtures("multiple_investigation_test_data") + def test_valid_get_with_filters_distinct( + self, flask_test_app, valid_credentials_header, + ): + test_response = flask_test_app.get( + '/investigations?where={"title": {"like": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}&distinct="title"', + headers=valid_credentials_header, + ) + + expected = [ + { + "title": f"Test data for the Python ICAT Backend on DataGateway API {i}" + for i in range(5) + }, + ] + + for title in expected: + assert title in test_response.json + + def test_limit_skip_merge_get_with_filters( + self, + flask_test_app, + valid_credentials_header, + multiple_investigation_test_data, + ): + skip_value = 1 + limit_value = 2 + + test_response = flask_test_app.get( + '/investigations?where={"title": {"like": "Test data for the Python ICAT' + ' Backend on DataGateway API"}}' + f'&skip={skip_value}&limit={limit_value}&order="id ASC"', + headers=valid_credentials_header, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + filtered_investigation_data = [] + filter_count = 0 + while filter_count < limit_value: + filtered_investigation_data.append( + multiple_investigation_test_data.pop(skip_value), + ) + filter_count += 1 + + assert response_json == filtered_investigation_data diff --git a/test/icat/endpoints/test_update_by_id.py b/test/icat/endpoints/test_update_by_id.py new file mode 100644 index 00000000..aad64e37 --- /dev/null +++ b/test/icat/endpoints/test_update_by_id.py @@ -0,0 +1,46 @@ +from test.icat.test_query import prepare_icat_data_for_assertion + + +class TestUpdateByID: + def test_valid_update_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" + + update_data_json = { + "doi": expected_doi, + "summary": expected_summary, + } + single_investigation_test_data[0]["doi"] = expected_doi + single_investigation_test_data[0]["summary"] = expected_summary + + test_response = flask_test_app.patch( + f"/investigations/{single_investigation_test_data[0]['id']}", + headers=valid_credentials_header, + json=update_data_json, + ) + response_json = prepare_icat_data_for_assertion([test_response.json]) + + assert response_json == single_investigation_test_data + + def test_invalid_update_with_id( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """This test will attempt to put `icatdb` into an invalid state""" + + # DOI cannot be over 255 characters, which this string is + invalid_update_json = { + "doi": "__________________________________________________________________" + "_________________________________________________________________________" + "_________________________________________________________________________" + "_________________________________________________________________________", + } + + test_response = flask_test_app.patch( + f"/investigations/{single_investigation_test_data[0]['id']}", + headers=valid_credentials_header, + json=invalid_update_json, + ) + + assert test_response.status_code == 400 diff --git a/test/icat/endpoints/test_update_multiple.py b/test/icat/endpoints/test_update_multiple.py new file mode 100644 index 00000000..3b25c79b --- /dev/null +++ b/test/icat/endpoints/test_update_multiple.py @@ -0,0 +1,101 @@ +import pytest + +from test.icat.test_query import prepare_icat_data_for_assertion + + +class TestUpdateMultipleEntities: + def test_valid_multiple_update_data( + self, + flask_test_app, + valid_credentials_header, + multiple_investigation_test_data, + ): + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" + + update_data_list = [] + + for investigation in multiple_investigation_test_data: + investigation["doi"] = expected_doi + investigation["summary"] = expected_summary + + update_entity = { + "id": investigation["id"], + "doi": expected_doi, + "summary": expected_summary, + } + update_data_list.append(update_entity) + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_list, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == multiple_investigation_test_data + + def test_valid_boundary_update_data( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """ Request body is a dictionary, not a list of dictionaries""" + + expected_doi = "Test Data Identifier" + expected_summary = "Test Summary" + + update_data_json = { + "id": single_investigation_test_data[0]["id"], + "doi": expected_doi, + "summary": expected_summary, + } + single_investigation_test_data[0]["doi"] = expected_doi + single_investigation_test_data[0]["summary"] = expected_summary + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_json, + ) + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == single_investigation_test_data + + def test_invalid_missing_update_data( + self, flask_test_app, valid_credentials_header, single_investigation_test_data, + ): + """There should be an ID in the request body to know which entity to update""" + + update_data_json = { + "doi": "Test Data Identifier", + "summary": "Test Summary", + } + + test_response = flask_test_app.patch( + "/investigations", headers=valid_credentials_header, json=update_data_json, + ) + + assert test_response.status_code == 400 + + @pytest.mark.parametrize( + "update_key, update_value", + [ + pytest.param("invalidAttr", "Some Value", id="invalid attribute"), + pytest.param("modId", "simple/root", id="meta attribute update"), + ], + ) + def test_invalid_attribute_update( + self, + flask_test_app, + valid_credentials_header, + single_investigation_test_data, + update_key, + update_value, + ): + invalid_update_data_json = { + "id": single_investigation_test_data[0]["id"], + update_key: update_value, + } + + test_response = flask_test_app.patch( + "/investigations", + headers=valid_credentials_header, + json=invalid_update_data_json, + ) + + assert test_response.status_code == 400 diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py index ccb8f274..54a51ab8 100644 --- a/test/icat/test_standard_endpoints.py +++ b/test/icat/test_standard_endpoints.py @@ -1,9 +1,6 @@ -import pytest - from datagateway_api.src.main import api from datagateway_api.src.main import app from datagateway_api.src.resources.entities.entity_map import endpoints -from test.icat.test_query import prepare_icat_data_for_assertion from test.test_base import FlaskAppTest @@ -31,444 +28,3 @@ def test_all_endpoints_exist(self): assert findone_endpoint_exist """ pass - - def test_valid_get_with_filters( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - test_response = flask_test_app.get( - '/investigations?where={"title": {"eq": "Test data for the Python ICAT' - ' Backend on DataGateway API"}}', - headers=valid_credentials_header, - ) - response_json = prepare_icat_data_for_assertion(test_response.json) - - assert response_json == single_investigation_test_data - - def test_valid_no_results_get_with_filters( - self, flask_test_app, valid_credentials_header, - ): - test_response = flask_test_app.get( - '/investigations?where={"title": {"eq": "This filter should cause a 404 for' - 'testing purposes..."}}', - headers=valid_credentials_header, - ) - - assert test_response.status_code == 404 - - @pytest.mark.usefixtures("multiple_investigation_test_data") - def test_valid_get_with_filters_distinct( - self, flask_test_app, valid_credentials_header, - ): - test_response = flask_test_app.get( - '/investigations?where={"title": {"like": "Test data for the Python ICAT' - ' Backend on DataGateway API"}}&distinct="title"', - headers=valid_credentials_header, - ) - - expected = [ - { - "title": f"Test data for the Python ICAT Backend on DataGateway API {i}" - for i in range(5) - }, - ] - - for title in expected: - assert title in test_response.json - - def test_limit_skip_merge_get_with_filters( - self, - flask_test_app, - valid_credentials_header, - multiple_investigation_test_data, - ): - skip_value = 1 - limit_value = 2 - - test_response = flask_test_app.get( - '/investigations?where={"title": {"like": "Test data for the Python ICAT' - ' Backend on DataGateway API"}}' - f'&skip={skip_value}&limit={limit_value}&order="id ASC"', - headers=valid_credentials_header, - ) - response_json = prepare_icat_data_for_assertion(test_response.json) - - filtered_investigation_data = [] - filter_count = 0 - while filter_count < limit_value: - filtered_investigation_data.append( - multiple_investigation_test_data.pop(skip_value), - ) - filter_count += 1 - - assert response_json == filtered_investigation_data - - def test_valid_create_data(self, flask_test_app, valid_credentials_header): - create_investigations_json = [ - { - "name": "Test Data for API Testing, Data Creation 1", - "title": "Test data for the Python ICAT Backend on DataGateway API", - "summary": "Test data for DataGateway API testing", - "releaseDate": "2020-03-03 08:00:08", - "startDate": "2020-02-02 09:00:09", - "endDate": "2020-02-03 10:00:10", - "visitId": "Data Creation Visit", - "doi": "DataGateway API Test DOI", - "facility": 1, - "type": 1, - }, - { - "name": "Test Data for API Testing, Data Creation 2", - "title": "Test data for the Python ICAT Backend on DataGateway API", - "summary": "Test data for DataGateway API testing", - "releaseDate": "2020-03-03 08:00:08", - "startDate": "2020-02-02 09:00:09", - "endDate": "2020-02-03 10:00:10", - "visitId": "Data Creation Visit", - "doi": "DataGateway API Test DOI", - "facility": 1, - "type": 1, - }, - ] - - test_response = flask_test_app.post( - "/investigations", - headers=valid_credentials_header, - json=create_investigations_json, - ) - - test_data_ids = [] - for investigation_request, investigation_response in zip( - create_investigations_json, test_response.json, - ): - investigation_request.pop("facility") - investigation_request.pop("type") - test_data_ids.append(investigation_response["id"]) - - response_json = prepare_icat_data_for_assertion( - test_response.json, remove_id=True, - ) - - assert create_investigations_json == response_json - - # Delete the entities created by this test - for investigation_id in test_data_ids: - flask_test_app.delete( - f"/investigations/{investigation_id}", headers=valid_credentials_header, - ) - - def test_valid_boundary_create_data(self, flask_test_app, valid_credentials_header): - """Create a single investigation, as opposed to multiple""" - - create_investigation_json = { - "name": "Test Data for API Testing, Data Creation 0", - "title": "Test data for the Python ICAT Backend on the API", - "summary": "Test data for DataGateway API testing", - "releaseDate": "2020-03-03 08:00:08", - "startDate": "2020-02-02 09:00:09", - "endDate": "2020-02-03 10:00:10", - "visitId": "Data Creation Visit", - "doi": "DataGateway API Test DOI", - "facility": 1, - "type": 1, - } - - test_response = flask_test_app.post( - "/investigations", - headers=valid_credentials_header, - json=create_investigation_json, - ) - - create_investigation_json.pop("facility") - create_investigation_json.pop("type") - created_test_data_id = test_response.json[0]["id"] - - response_json = prepare_icat_data_for_assertion( - test_response.json, remove_id=True, - ) - - assert [create_investigation_json] == response_json - - flask_test_app.delete( - f"/investigations/{created_test_data_id}", headers=valid_credentials_header, - ) - - def test_invalid_create_data(self, flask_test_app, valid_credentials_header): - """An investigation requires a minimum of: name, visitId, facility, type""" - - invalid_request_body = { - "title": "Test Title for DataGateway API Backend testing", - } - - test_response = flask_test_app.post( - "/investigations", - headers=valid_credentials_header, - json=invalid_request_body, - ) - - assert test_response.status_code == 400 - - def test_invalid_existing_data_create( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - """This test targets raising ICATObjectExistsError, causing a 400""" - - # entity.as_dict() removes details about facility and type, hence they're - # hardcoded here instead of using sinle_investigation_test_data - existing_object_json = { - "name": single_investigation_test_data[0]["name"], - "title": single_investigation_test_data[0]["title"], - "visitId": single_investigation_test_data[0]["visitId"], - "facility": 1, - "type": 1, - } - - test_response = flask_test_app.post( - "/investigations", - headers=valid_credentials_header, - json=existing_object_json, - ) - - assert test_response.status_code == 400 - - def test_valid_multiple_update_data( - self, - flask_test_app, - valid_credentials_header, - multiple_investigation_test_data, - ): - expected_doi = "Test Data Identifier" - expected_summary = "Test Summary" - - update_data_list = [] - - for investigation in multiple_investigation_test_data: - investigation["doi"] = expected_doi - investigation["summary"] = expected_summary - - update_entity = { - "id": investigation["id"], - "doi": expected_doi, - "summary": expected_summary, - } - update_data_list.append(update_entity) - - test_response = flask_test_app.patch( - "/investigations", headers=valid_credentials_header, json=update_data_list, - ) - response_json = prepare_icat_data_for_assertion(test_response.json) - - assert response_json == multiple_investigation_test_data - - def test_valid_boundary_update_data( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - """ Request body is a dictionary, not a list of dictionaries""" - - expected_doi = "Test Data Identifier" - expected_summary = "Test Summary" - - update_data_json = { - "id": single_investigation_test_data[0]["id"], - "doi": expected_doi, - "summary": expected_summary, - } - single_investigation_test_data[0]["doi"] = expected_doi - single_investigation_test_data[0]["summary"] = expected_summary - - test_response = flask_test_app.patch( - "/investigations", headers=valid_credentials_header, json=update_data_json, - ) - response_json = prepare_icat_data_for_assertion(test_response.json) - - assert response_json == single_investigation_test_data - - def test_invalid_missing_update_data( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - """There should be an ID in the request body to know which entity to update""" - - update_data_json = { - "doi": "Test Data Identifier", - "summary": "Test Summary", - } - - test_response = flask_test_app.patch( - "/investigations", headers=valid_credentials_header, json=update_data_json, - ) - - assert test_response.status_code == 400 - - @pytest.mark.parametrize( - "update_key, update_value", - [ - pytest.param("invalidAttr", "Some Value", id="invalid attribute"), - pytest.param("modId", "simple/root", id="meta attribute update"), - ], - ) - def test_invalid_attribute_update( - self, - flask_test_app, - valid_credentials_header, - single_investigation_test_data, - update_key, - update_value, - ): - invalid_update_data_json = { - "id": single_investigation_test_data[0]["id"], - update_key: update_value, - } - - test_response = flask_test_app.patch( - "/investigations", - headers=valid_credentials_header, - json=invalid_update_data_json, - ) - - assert test_response.status_code == 400 - - def test_valid_findone_with_filters( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - test_response = flask_test_app.get( - '/investigations/findone?where={"title": {"eq": "Test data for the Python' - ' ICAT Backend on DataGateway API"}}', - headers=valid_credentials_header, - ) - response_json = prepare_icat_data_for_assertion([test_response.json]) - - assert response_json == single_investigation_test_data - - def test_valid_no_results_findone_with_filters( - self, flask_test_app, valid_credentials_header, - ): - test_response = flask_test_app.get( - '/investigations/findone?where={"title": {"eq": "This filter should cause a' - '404 for testing purposes..."}}', - headers=valid_credentials_header, - ) - - assert test_response.status_code == 404 - - @pytest.mark.usefixtures("single_investigation_test_data") - def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header): - test_response = flask_test_app.get( - '/investigations/count?where={"title": {"eq": "Test data for the Python' - ' ICAT Backend on DataGateway API"}}', - headers=valid_credentials_header, - ) - - assert test_response.json == 1 - - def test_valid_no_results_count_with_filters( - self, flask_test_app, valid_credentials_header, - ): - test_response = flask_test_app.get( - '/investigations/count?where={"title": {"eq": "This filter should cause a' - '404 for testing purposes..."}}', - headers=valid_credentials_header, - ) - - assert test_response.json == 0 - - def test_valid_get_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - # Need to identify the ID given to the test data - investigation_data = flask_test_app.get( - '/investigations?where={"title": {"eq": "Test data for the Python ICAT' - ' Backend on DataGateway API"}}', - headers=valid_credentials_header, - ) - test_data_id = investigation_data.json[0]["id"] - - test_response = flask_test_app.get( - f"/investigations/{test_data_id}", headers=valid_credentials_header, - ) - # Get with ID gives a dictionary response (only ever one result from that kind - # of request), so list around json is required for the call - response_json = prepare_icat_data_for_assertion([test_response.json]) - - assert response_json == single_investigation_test_data - - def test_invalid_get_with_id(self, flask_test_app, valid_credentials_header): - """Request with a non-existent ID""" - - final_investigation_result = flask_test_app.get( - '/investigations/findone?order="id DESC"', headers=valid_credentials_header, - ) - test_data_id = final_investigation_result.json["id"] - - # Adding 100 onto the ID to the most recent result should ensure a 404 - test_response = flask_test_app.get( - f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, - ) - - assert test_response.status_code == 404 - - def test_valid_delete_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - test_response = flask_test_app.delete( - f'/investigations/{single_investigation_test_data[0]["id"]}', - headers=valid_credentials_header, - ) - - assert test_response.status_code == 204 - - def test_invalid_delete_with_id(self, flask_test_app, valid_credentials_header): - """Request with a non-existent ID""" - - final_investigation_result = flask_test_app.get( - '/investigations/findone?order="id DESC"', headers=valid_credentials_header, - ) - test_data_id = final_investigation_result.json["id"] - - # Adding 100 onto the ID to the most recent result should ensure a 404 - test_response = flask_test_app.delete( - f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, - ) - - assert test_response.status_code == 404 - - def test_valid_update_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - expected_doi = "Test Data Identifier" - expected_summary = "Test Summary" - - update_data_json = { - "doi": expected_doi, - "summary": expected_summary, - } - single_investigation_test_data[0]["doi"] = expected_doi - single_investigation_test_data[0]["summary"] = expected_summary - - test_response = flask_test_app.patch( - f"/investigations/{single_investigation_test_data[0]['id']}", - headers=valid_credentials_header, - json=update_data_json, - ) - response_json = prepare_icat_data_for_assertion([test_response.json]) - - assert response_json == single_investigation_test_data - - def test_invalid_update_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, - ): - """This test will attempt to put `icatdb` into an invalid state""" - - # DOI cannot be over 255 characters, which this string is - invalid_update_json = { - "doi": "__________________________________________________________________" - "_________________________________________________________________________" - "_________________________________________________________________________" - "_________________________________________________________________________", - } - - test_response = flask_test_app.patch( - f"/investigations/{single_investigation_test_data[0]['id']}", - headers=valid_credentials_header, - json=invalid_update_json, - ) - - assert test_response.status_code == 400 From 79bdbbe4f8e4c768127a3e05468fc1a112032af1 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 26 Nov 2020 20:29:43 +0000 Subject: [PATCH 050/109] #150: Add tests to check all endpoints exist and have correct HTTP methods on them --- test/icat/endpoints/test_endpoint_rules.py | 74 ++++++++++++++++++++++ test/icat/test_standard_endpoints.py | 30 --------- 2 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 test/icat/endpoints/test_endpoint_rules.py delete mode 100644 test/icat/test_standard_endpoints.py diff --git a/test/icat/endpoints/test_endpoint_rules.py b/test/icat/endpoints/test_endpoint_rules.py new file mode 100644 index 00000000..b92d8ff8 --- /dev/null +++ b/test/icat/endpoints/test_endpoint_rules.py @@ -0,0 +1,74 @@ +import pytest + +from datagateway_api.src.main import api +from datagateway_api.src.resources.entities.entity_map import endpoints + + +class TestEndpointRules: + """ + Test class to ensure all endpoints on the API exist & have the correct HTTP methods + """ + + @pytest.mark.parametrize( + "endpoint_ending, expected_methods", + [ + pytest.param("/findone", ["GET"], id="findone"), + pytest.param("/count", ["GET"], id="count"), + pytest.param("/", ["DELETE", "GET", "PATCH"], id="id"), + pytest.param("", ["GET", "PATCH", "POST"], id="typical endpoints"), + ], + ) + def test_entity_endpoints(self, endpoint_ending, expected_methods): + for endpoint_entity in endpoints.keys(): + endpoint_found = False + + for rule in api.app.url_map.iter_rules(): + if f"/{endpoint_entity.lower()}{endpoint_ending}" == rule.rule: + endpoint_found = True + + for method_name in expected_methods: + # Can't do a simple equality check as .methods contains other + # methods not added by the API which aren't utilised + assert method_name in rule.methods + + assert endpoint_found + + @pytest.mark.parametrize( + "endpoint_name, expected_methods", + [ + pytest.param("/sessions", ["DELETE", "GET", "POST", "PUT"], id="sessions"), + pytest.param( + "/instruments//facilitycycles", + ["GET"], + id="ISIS instrument's facility cycles", + ), + pytest.param( + "/instruments//facilitycycles/count", + ["GET"], + id="count ISIS instrument's facility cycles", + ), + pytest.param( + "/instruments//facilitycycles/" + "/investigations", + ["GET"], + id="ISIS investigations", + ), + pytest.param( + "/instruments//facilitycycles/" + "/investigations/count", + ["GET"], + id="count ISIS investigations", + ), + ], + ) + def test_non_entity_endpoints(self, endpoint_name, expected_methods): + endpoint_found = False + + for rule in api.app.url_map.iter_rules(): + if endpoint_name == rule.rule: + endpoint_found = True + + for method_name in expected_methods: + assert method_name in rule.methods + + assert endpoint_found diff --git a/test/icat/test_standard_endpoints.py b/test/icat/test_standard_endpoints.py deleted file mode 100644 index 54a51ab8..00000000 --- a/test/icat/test_standard_endpoints.py +++ /dev/null @@ -1,30 +0,0 @@ -from datagateway_api.src.main import api -from datagateway_api.src.main import app -from datagateway_api.src.resources.entities.entity_map import endpoints -from test.test_base import FlaskAppTest - - -class TestStandardEndpoints: - def test_all_endpoints_exist(self): - """ - session_endpoint_exist = api.owns_endpoint("sessions") - assert session_endpoint_exist - - for endpoint_entity in endpoints.keys(): - get_endpoint_exist = api.owns_endpoint(endpoint_entity.lower()) - assert get_endpoint_exist - - id_endpoint_exist = api.owns_endpoint( - f"{endpoint_entity.lower()}/", - ) - assert id_endpoint_exist - - count_endpoint_exist = api.owns_endpoint(f"{endpoint_entity.lower()}/count") - assert count_endpoint_exist - - findone_endpoint_exist = api.owns_endpoint( - f"{endpoint_entity.lower()}/findone", - ) - assert findone_endpoint_exist - """ - pass From 861b4f8ac3f57e71d50346a964dd1ce7c8a5965a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 27 Nov 2020 12:35:10 +0000 Subject: [PATCH 051/109] #150: Add session handling tests for ICAT backend --- test/conftest.py | 5 ++ test/icat/test_session_handling.py | 117 +++++++++++++++++++++++++---- 2 files changed, 108 insertions(+), 14 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9b89af78..f8147235 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -22,6 +22,11 @@ def valid_credentials_header(icat_client): return {"Authorization": f"Bearer {icat_client.sessionId}"} +@pytest.fixture() +def invalid_credentials_header(): + return {"Authorization": "Bearer Invalid"} + + @pytest.fixture() def icat_query(icat_client): query = Query(icat_client, "Investigation") diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 53ae6d83..6adf6aea 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -1,24 +1,113 @@ +from datetime import datetime + +import pytest + +from datagateway_api.common.config import config +from datagateway_api.common.icat.filters import PythonICATWhereFilter + + class TestSessionHandling: def test_session_id_decorator(self): pass - def test_get_valid_session_details(self): - pass + def test_get_valid_session_details(self, flask_test_app, valid_credentials_header): + session_details = flask_test_app.get( + "/sessions", headers=valid_credentials_header, + ) - def test_get_invalid_session_details(self): - pass + print(f"JSON: {session_details.json}, Code: {session_details.status_code}") - def test_refresh_session(self): - pass + # Check username is correct + assert ( + session_details.json["USERNAME"] == f"{config.get_test_mechanism()}/" + f"{config.get_test_user_credentials()['username']}" + ) + # Check session ID matches the header from the request + assert ( + session_details.json["ID"] + == valid_credentials_header["Authorization"].split()[1] + ) - def test_valid_login(self): - pass + session_expiry_datetime = datetime.strptime( + session_details.json["EXPIREDATETIME"], "%Y-%m-%d %H:%M:%S.%f", + ) + current_datetime = datetime.now() + time_diff = abs(session_expiry_datetime - current_datetime) + time_diff_minutes = time_diff.seconds / 60 - def test_invalid_login(self): - pass + # Allows a bit of leeway for slow test execution + assert time_diff_minutes < 120 and time_diff_minutes >= 118 - def test_valid_logout(self): - pass + def test_get_invalid_session_details( + self, invalid_credentials_header, flask_test_app, + ): + session_details = flask_test_app.get( + "/sessions", headers=invalid_credentials_header, + ) - def test_invalid_logout(self): - pass + assert session_details.status_code == 403 + + def test_refresh_session(self, valid_credentials_header, flask_test_app): + pre_refresh_session_details = flask_test_app.get( + "/sessions", headers=valid_credentials_header, + ) + + refresh_session = flask_test_app.put( + "/sessions", headers=valid_credentials_header, + ) + assert refresh_session.status_code == 200 + + post_refresh_session_details = flask_test_app.get( + "/sessions", headers=valid_credentials_header, + ) + + assert ( + pre_refresh_session_details.json["EXPIREDATETIME"] + != post_refresh_session_details.json["EXPIREDATETIME"] + ) + + @pytest.mark.usefixtures("single_investigation_test_data") + def test_valid_login(self, flask_test_app, icat_client, icat_query): + user_credentials = config.get_test_user_credentials() + + login_json = { + "username": user_credentials["username"], + "password": user_credentials["password"], + "mechanism": config.get_test_mechanism(), + } + login_response = flask_test_app.post("/sessions", json=login_json) + + icat_client.sessionId = login_response.json["sessionID"] + icat_query.setAggregate("COUNT") + title_filter = PythonICATWhereFilter( + "title", "Test data for the Python ICAT Backend on DataGateway API", "eq", + ) + title_filter.apply_filter(icat_query) + + test_query = icat_client.search(icat_query) + + assert test_query == [1] and login_response.status_code == 201 + + def test_invalid_login(self, flask_test_app): + login_json = { + "username": "Invalid Username", + "password": "InvalidPassword", + "mechanism": config.get_test_mechanism(), + } + login_response = flask_test_app.post("/sessions", json=login_json) + + assert login_response.status_code == 403 + + def test_valid_logout(self, flask_test_app, valid_credentials_header): + logout_response = flask_test_app.delete( + "/sessions", headers=valid_credentials_header, + ) + + assert logout_response.status_code == 200 + + def test_invalid_logout(self, invalid_credentials_header, flask_test_app): + logout_response = flask_test_app.delete( + "/sessions", headers=invalid_credentials_header, + ) + + assert logout_response.status_code == 403 From 1470b2218650070cd7c7682eacee1d4a867d171d Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 27 Nov 2020 16:35:16 +0000 Subject: [PATCH 052/109] #150: Add invalid tests for ISIS specific endpoints --- test/icat/endpoints/test_table_endpoints.py | 87 +++++++++++++++++++++ test/icat/test_isis_endpoints.py | 12 --- 2 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 test/icat/endpoints/test_table_endpoints.py delete mode 100644 test/icat/test_isis_endpoints.py diff --git a/test/icat/endpoints/test_table_endpoints.py b/test/icat/endpoints/test_table_endpoints.py new file mode 100644 index 00000000..6c784244 --- /dev/null +++ b/test/icat/endpoints/test_table_endpoints.py @@ -0,0 +1,87 @@ +class TestTableEndpoints: + """ + This class tests the endpoints defined in table_endpoints.py, commonly referred to + as the ISIS specific endpoints + """ + + def test_valid_get_facility_cycles_with_filters(self): + pass + + def test_invalid_get_facility_cycles_with_filters( + self, flask_test_app, valid_credentials_header, + ): + final_instrument_result = flask_test_app.get( + '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + ) + final_instrument_id = final_instrument_result.json["id"] + + test_response = flask_test_app.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles", + headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 + + def test_valid_get_facility_cycles_count_with_filters(self): + pass + + def test_invalid_get_facility_cycles_count_with_filters( + self, flask_test_app, valid_credentials_header, + ): + final_instrument_result = flask_test_app.get( + '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + ) + final_instrument_id = final_instrument_result.json["id"] + + test_response = flask_test_app.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles/count", + headers=valid_credentials_header, + ) + + assert test_response.json == 0 + + def test_valid_get_investigations_with_filters(self): + pass + + def test_invalid_get_investigations_with_filters( + self, flask_test_app, valid_credentials_header, + ): + final_instrument_result = flask_test_app.get( + '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + ) + final_instrument_id = final_instrument_result.json["id"] + final_facilitycycle_result = flask_test_app.get( + '/facilitycycles/findone?order="id DESC"', headers=valid_credentials_header, + ) + final_facilitycycle_id = final_facilitycycle_result.json["id"] + + test_response = flask_test_app.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles/" + f"{final_facilitycycle_id + 100}/investigations", + headers=valid_credentials_header, + ) + + assert test_response.status_code == 404 + + def test_valid_get_investigations_count_with_filters(self): + pass + + def test_invalid_get_investigations_count_with_filters( + self, flask_test_app, valid_credentials_header, + ): + final_instrument_result = flask_test_app.get( + '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + ) + final_instrument_id = final_instrument_result.json["id"] + final_facilitycycle_result = flask_test_app.get( + '/facilitycycles/findone?order="id DESC"', headers=valid_credentials_header, + ) + final_facilitycycle_id = final_facilitycycle_result.json["id"] + + test_response = flask_test_app.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles/" + f"{final_facilitycycle_id + 100}/investigations/count", + headers=valid_credentials_header, + ) + + assert test_response.json == 0 diff --git a/test/icat/test_isis_endpoints.py b/test/icat/test_isis_endpoints.py deleted file mode 100644 index db39e2f1..00000000 --- a/test/icat/test_isis_endpoints.py +++ /dev/null @@ -1,12 +0,0 @@ -class TestISISEndpoints: - def test_valid_get_facility_cycles_with_filters(self): - pass - - def test_valid_get_facility_cycles_count_with_filters(self): - pass - - def test_valid_get_investigations_with_filters(self): - pass - - def test_valid_get_investigations_count_with_filters(self): - pass From 25f22825c875fba0506c9d3d67e6ee423ebabf80 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 12:12:31 +0000 Subject: [PATCH 053/109] #150: Simplify icat_query fixture --- test/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index f8147235..97bd6e06 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -29,9 +29,7 @@ def invalid_credentials_header(): @pytest.fixture() def icat_query(icat_client): - query = Query(icat_client, "Investigation") - - return query + return Query(icat_client, "Investigation") @pytest.fixture() From 275cb43d45d0c23a6d46367230ec44d0359c36c0 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 13:33:55 +0000 Subject: [PATCH 054/109] #150: Use one-off session ID for logout test - Once this test had completed, any tests afterwards using the icat_client fixture (or valid_credentials_header) would fail because those credentials had been deleted from icatdb --- test/icat/test_session_handling.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 6adf6aea..0a4842cd 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -1,5 +1,6 @@ from datetime import datetime +from icat.client import Client import pytest from datagateway_api.common.config import config @@ -98,10 +99,12 @@ def test_invalid_login(self, flask_test_app): assert login_response.status_code == 403 - def test_valid_logout(self, flask_test_app, valid_credentials_header): - logout_response = flask_test_app.delete( - "/sessions", headers=valid_credentials_header, - ) + def test_valid_logout(self, flask_test_app): + client = Client(config.get_icat_url(), checkCert=config.get_icat_check_cert()) + client.login(config.get_test_mechanism(), config.get_test_user_credentials()) + creds_header = {"Authorization": f"Bearer {client.sessionId}"} + + logout_response = flask_test_app.delete("/sessions", headers=creds_header) assert logout_response.status_code == 200 From 7aae8c8bf9a9a1dd5922c9c13ad4908d96f3eafa Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 13:46:49 +0000 Subject: [PATCH 055/109] #150: Refactor creation of investigation test data --- test/conftest.py | 78 ++++++++++++++++++++--------------------- test/icat/test_query.py | 9 ++++- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 97bd6e06..2977e6b0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,13 +1,14 @@ +from datetime import datetime import uuid from icat.client import Client -from icat.entity import Entity from icat.exception import ICATNoObjectError from icat.query import Query import pytest from datagateway_api.common.config import config from datagateway_api.src.main import app +from test.icat.test_query import prepare_icat_data_for_assertion @pytest.fixture(scope="package") @@ -32,24 +33,41 @@ def icat_query(icat_client): return Query(icat_client, "Investigation") +def create_investigation_test_data(client, num_entities=1): + test_data = [] + + for i in range(num_entities): + investigation = client.new("investigation") + investigation.name = f"Test Data for DataGateway API Testing {i}" + investigation.title = ( + f"Test data for the Python ICAT Backend on DataGateway API {i}" + ) + investigation.startDate = datetime( + year=2020, month=1, day=4, hour=1, minute=1, second=1, + ) + investigation.endDate = datetime( + year=2020, month=1, day=8, hour=1, minute=1, second=1, + ) + # UUID visit ID means uniquesness constraint should always be met + investigation.visitId = str(uuid.uuid1()) + investigation.facility = client.get("Facility", 1) + investigation.type = client.get("InvestigationType", 1) + investigation.create() + + test_data.append(investigation) + + if len(test_data) == 1: + return test_data[0] + else: + return test_data + + @pytest.fixture() def single_investigation_test_data(icat_client): - # Inject data - investigation = icat_client.new("investigation") - investigation.name = "Test Data for DataGateway API Testing" - investigation.title = "Test data for the Python ICAT Backend on DataGateway API" - # UUID visit ID means uniquesness constraint should always be met - investigation.visitId = str(uuid.uuid1()) - investigation.facility = icat_client.get("Facility", 1) - investigation.type = icat_client.get("InvestigationType", 1) - investigation.create() - investigation_dict = investigation.as_dict() - - meta_attributes = Entity.MetaAttr - for attr in meta_attributes: - investigation_dict.pop(attr) - - yield [investigation_dict] + investigation = create_investigation_test_data(icat_client) + investigation_dict = prepare_icat_data_for_assertion([investigation]) + + yield investigation_dict # Remove data from ICAT try: @@ -61,32 +79,14 @@ def single_investigation_test_data(icat_client): @pytest.fixture() def multiple_investigation_test_data(icat_client): - investigation_test_data = [] investigation_dicts = [] - meta_attributes = Entity.MetaAttr - - for i in range(5): - investigation = icat_client.new("investigation") - investigation.name = f"Test Data for DataGateway API Testing {i}" - investigation.title = ( - f"Test data for the Python ICAT Backend on DataGateway API {i}" - ) - investigation.visitId = str(uuid.uuid1()) - investigation.facility = icat_client.get("Facility", 1) - investigation.type = icat_client.get("InvestigationType", 1) - investigation.create() - investigation_test_data.append(investigation) - investigation_dict = investigation.as_dict() - - for attr in meta_attributes: - investigation_dict.pop(attr) - - investigation_dicts.append(investigation_dict) + investigations = create_investigation_test_data(icat_client, num_entities=5) + investigation_dicts = prepare_icat_data_for_assertion(investigations) yield investigation_dicts - for entity in investigation_test_data: - icat_client.delete(entity) + for investigation in investigations: + icat_client.delete(investigation) @pytest.fixture() diff --git a/test/icat/test_query.py b/test/icat/test_query.py index 8e0717c0..27095919 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -1,6 +1,9 @@ +from datetime import datetime + from icat.entity import Entity import pytest +from datagateway_api.common.date_handler import DateHandler from datagateway_api.common.exceptions import PythonICATError from datagateway_api.common.icat.filters import ( PythonICATSkipFilter, @@ -15,7 +18,7 @@ def prepare_icat_data_for_assertion(data, remove_id=False): creation/modification, and should be removed to ensure correct assertion values :param data: ICAT data containing meta attributes such as modTime - :type data: :class:`dict` or an inherited version of :class:`icat.entity.Entity` + :type data: :class:`list` or :class:`icat.entity.EntityList` """ assertable_data = [] meta_attributes = Entity.MetaAttr @@ -28,6 +31,10 @@ def prepare_icat_data_for_assertion(data, remove_id=False): for attr in meta_attributes: entity.pop(attr) + for attr in entity.keys(): + if isinstance(entity[attr], datetime): + entity[attr] = DateHandler.datetime_object_to_str(entity[attr]) + # meta_attributes is immutable if remove_id: entity.pop("id") From 9ed26940fdeb668ebb1e573dd7d6d4e7eab162c5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 13:47:42 +0000 Subject: [PATCH 056/109] #150: Fix test failures due to refactor --- test/icat/endpoints/test_count_with_filters.py | 4 ++-- test/icat/endpoints/test_findone.py | 2 +- test/icat/endpoints/test_get_by_id.py | 2 +- test/icat/endpoints/test_get_with_filters.py | 2 +- test/icat/test_query.py | 4 ++-- test/icat/test_session_handling.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters.py index 3c634af3..d2961c79 100644 --- a/test/icat/endpoints/test_count_with_filters.py +++ b/test/icat/endpoints/test_count_with_filters.py @@ -5,7 +5,7 @@ class TestCountWithFilters: @pytest.mark.usefixtures("single_investigation_test_data") def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header): test_response = flask_test_app.get( - '/investigations/count?where={"title": {"eq": "Test data for the Python' + '/investigations/count?where={"title": {"like": "Test data for the Python' ' ICAT Backend on DataGateway API"}}', headers=valid_credentials_header, ) @@ -16,7 +16,7 @@ def test_valid_no_results_count_with_filters( self, flask_test_app, valid_credentials_header, ): test_response = flask_test_app.get( - '/investigations/count?where={"title": {"eq": "This filter should cause a' + '/investigations/count?where={"title": {"like": "This filter should cause a' '404 for testing purposes..."}}', headers=valid_credentials_header, ) diff --git a/test/icat/endpoints/test_findone.py b/test/icat/endpoints/test_findone.py index 84d3f830..a0051e32 100644 --- a/test/icat/endpoints/test_findone.py +++ b/test/icat/endpoints/test_findone.py @@ -6,7 +6,7 @@ def test_valid_findone_with_filters( self, flask_test_app, valid_credentials_header, single_investigation_test_data, ): test_response = flask_test_app.get( - '/investigations/findone?where={"title": {"eq": "Test data for the Python' + '/investigations/findone?where={"title": {"like": "Test data for the Python' ' ICAT Backend on DataGateway API"}}', headers=valid_credentials_header, ) diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id.py index 24b962c1..dfcbaae8 100644 --- a/test/icat/endpoints/test_get_by_id.py +++ b/test/icat/endpoints/test_get_by_id.py @@ -7,7 +7,7 @@ def test_valid_get_with_id( ): # Need to identify the ID given to the test data investigation_data = flask_test_app.get( - '/investigations?where={"title": {"eq": "Test data for the Python ICAT' + '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}', headers=valid_credentials_header, ) diff --git a/test/icat/endpoints/test_get_with_filters.py b/test/icat/endpoints/test_get_with_filters.py index 46fccf4c..f72ab3c9 100644 --- a/test/icat/endpoints/test_get_with_filters.py +++ b/test/icat/endpoints/test_get_with_filters.py @@ -8,7 +8,7 @@ def test_valid_get_with_filters( self, flask_test_app, valid_credentials_header, single_investigation_test_data, ): test_response = flask_test_app.get( - '/investigations?where={"title": {"eq": "Test data for the Python ICAT' + '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}', headers=valid_credentials_header, ) diff --git a/test/icat/test_query.py b/test/icat/test_query.py index 27095919..8e11722d 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -60,7 +60,7 @@ def test_valid_query_exeuction( ): test_query = ICATQuery(icat_client, "Investigation") test_data_filter = PythonICATWhereFilter( - "title", "Test data for the Python ICAT Backend on DataGateway API", "eq", + "title", "Test data for the Python ICAT Backend on DataGateway API", "like", ) test_data_filter.apply_filter(test_query.query) query_data = test_query.execute_query(icat_client) @@ -91,7 +91,7 @@ def test_json_format_execution_output( ): test_query = ICATQuery(icat_client, "Investigation") test_data_filter = PythonICATWhereFilter( - "title", "Test data for the Python ICAT Backend on DataGateway API", "eq", + "title", "Test data for the Python ICAT Backend on DataGateway API", "like", ) test_data_filter.apply_filter(test_query.query) query_data = test_query.execute_query(icat_client, True) diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 0a4842cd..3aa47d6f 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -81,7 +81,7 @@ def test_valid_login(self, flask_test_app, icat_client, icat_query): icat_client.sessionId = login_response.json["sessionID"] icat_query.setAggregate("COUNT") title_filter = PythonICATWhereFilter( - "title", "Test data for the Python ICAT Backend on DataGateway API", "eq", + "title", "Test data for the Python ICAT Backend on DataGateway API", "like", ) title_filter.apply_filter(icat_query) From 5910404c9bd08ccd113b3bd0aff274c4a7b62efc Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 14:23:21 +0000 Subject: [PATCH 057/109] #150: Add valid tests for table endpoints --- test/conftest.py | 39 +++++++++++++++ test/icat/endpoints/test_table_endpoints.py | 53 +++++++++++++++++---- 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 2977e6b0..92f4f80a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -93,3 +93,42 @@ def multiple_investigation_test_data(icat_client): def flask_test_app(): app.config["TESTING"] = True return app.test_client() + + +@pytest.fixture() +def isis_specific_endpoint_data(icat_client): + facility_cycle = icat_client.new("facilityCycle") + facility_cycle.name = "Test cycle for DataGateway API testing" + facility_cycle.startDate = datetime( + year=2020, month=1, day=1, hour=1, minute=1, second=1, + ) + facility_cycle.endDate = datetime( + year=2020, month=2, day=1, hour=1, minute=1, second=1, + ) + facility_cycle.facility = icat_client.get("Facility", 1) + facility_cycle.create() + + investigation = create_investigation_test_data(icat_client) + investigation_dict = prepare_icat_data_for_assertion([investigation]) + + instrument = icat_client.new("instrument") + instrument.name = "Test Instrument for DataGateway API Endpoint Testing" + instrument.facility = icat_client.get("Facility", 1) + instrument.create() + + investigation_instrument = icat_client.new("investigationInstrument") + investigation_instrument.investigation = investigation + investigation_instrument.instrument = instrument + investigation_instrument.create() + + facility_cycle_dict = prepare_icat_data_for_assertion([facility_cycle]) + + yield (instrument.id, facility_cycle_dict, facility_cycle.id, investigation_dict) + + try: + # investigation_instrument removed when deleting the objects its related objects + icat_client.delete(facility_cycle) + icat_client.delete(investigation) + icat_client.delete(instrument) + except ICATNoObjectError as e: + print(e) diff --git a/test/icat/endpoints/test_table_endpoints.py b/test/icat/endpoints/test_table_endpoints.py index 6c784244..6415d279 100644 --- a/test/icat/endpoints/test_table_endpoints.py +++ b/test/icat/endpoints/test_table_endpoints.py @@ -1,11 +1,23 @@ +from test.icat.test_query import prepare_icat_data_for_assertion + + class TestTableEndpoints: """ This class tests the endpoints defined in table_endpoints.py, commonly referred to as the ISIS specific endpoints """ - def test_valid_get_facility_cycles_with_filters(self): - pass + def test_valid_get_facility_cycles_with_filters( + self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + ): + test_response = flask_test_app.get( + f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles", + headers=valid_credentials_header, + ) + + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == isis_specific_endpoint_data[1] def test_invalid_get_facility_cycles_with_filters( self, flask_test_app, valid_credentials_header, @@ -22,8 +34,15 @@ def test_invalid_get_facility_cycles_with_filters( assert test_response.status_code == 404 - def test_valid_get_facility_cycles_count_with_filters(self): - pass + def test_valid_get_facility_cycles_count_with_filters( + self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + ): + test_response = flask_test_app.get( + f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/count", + headers=valid_credentials_header, + ) + + assert test_response.json == 1 def test_invalid_get_facility_cycles_count_with_filters( self, flask_test_app, valid_credentials_header, @@ -40,8 +59,18 @@ def test_invalid_get_facility_cycles_count_with_filters( assert test_response.json == 0 - def test_valid_get_investigations_with_filters(self): - pass + def test_valid_get_investigations_with_filters( + self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + ): + test_response = flask_test_app.get( + f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/" + f"{isis_specific_endpoint_data[2]}/investigations", + headers=valid_credentials_header, + ) + + response_json = prepare_icat_data_for_assertion(test_response.json) + + assert response_json == isis_specific_endpoint_data[3] def test_invalid_get_investigations_with_filters( self, flask_test_app, valid_credentials_header, @@ -63,8 +92,16 @@ def test_invalid_get_investigations_with_filters( assert test_response.status_code == 404 - def test_valid_get_investigations_count_with_filters(self): - pass + def test_valid_get_investigations_count_with_filters( + self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + ): + test_response = flask_test_app.get( + f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/" + f"{isis_specific_endpoint_data[2]}/investigations/count", + headers=valid_credentials_header, + ) + + assert test_response.json == 1 def test_invalid_get_investigations_count_with_filters( self, flask_test_app, valid_credentials_header, From 49ef7f807129a069ff7beeaa2dbb66663d969be5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 15:35:34 +0000 Subject: [PATCH 058/109] #150: Increase coverage of PythonICATIncludeFilter --- test/icat/filters/test_include_filter.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/icat/filters/test_include_filter.py b/test/icat/filters/test_include_filter.py index 505c61dc..48264897 100644 --- a/test/icat/filters/test_include_filter.py +++ b/test/icat/filters/test_include_filter.py @@ -1,3 +1,4 @@ +from datagateway_api.common.exceptions import FilterError import pytest from datagateway_api.common.icat.filters import PythonICATIncludeFilter @@ -45,3 +46,29 @@ def test_valid_input(self, icat_query, filter_input, expected_output): test_filter.apply_filter(icat_query) assert icat_query.includes == expected_output + + def test_invalid_type(self, icat_query): + with pytest.raises(FilterError): + PythonICATIncludeFilter({"datasets", "facility"}) + + def test_invalid_field(self, icat_query): + test_filter = PythonICATIncludeFilter("invalidField") + with pytest.raises(FilterError): + test_filter.apply_filter(icat_query) + + @pytest.mark.parametrize( + "filter_input", + [ + pytest.param({2: "datasets"}, id="invalid dictionary key"), + pytest.param( + {"datasets": {2: "datafiles"}}, id="invalid inner dictionary key" + ), + pytest.param( + {"datasets": {"datafiles", "sample"}}, + id="invalid inner dictionary value", + ), + ], + ) + def test_invalid_extract_field(self, filter_input): + with pytest.raises(FilterError): + PythonICATIncludeFilter(filter_input) From 90f14a12e55c3e7457d188a2bc8e406889e0e457 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 15:36:02 +0000 Subject: [PATCH 059/109] #150: Fix order filter test failures - test_direction_is_uppercase() wasn't being destroyed correctly --- test/icat/filters/test_order_filter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/icat/filters/test_order_filter.py b/test/icat/filters/test_order_filter.py index 6f2ba8c8..75c7c311 100644 --- a/test/icat/filters/test_order_filter.py +++ b/test/icat/filters/test_order_filter.py @@ -11,8 +11,13 @@ def test_direction_is_uppercase(self, icat_query): """Direction must be uppercase for Python ICAT to see the input as valid""" test_filter = PythonICATOrderFilter("id", "asc") + filter_handler = FilterOrderHandler() + filter_handler.add_filter(test_filter) + assert test_filter.direction == "ASC" + filter_handler.clear_python_icat_order_filters() + def test_result_order_appended(self, icat_query): id_filter = PythonICATOrderFilter("id", "ASC") title_filter = PythonICATOrderFilter("title", "DESC") From e85f3785d3e4028da7701bc30ffb788bafb4860d Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 15:41:35 +0000 Subject: [PATCH 060/109] #150: Solve linting issues --- test/icat/filters/test_include_filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/icat/filters/test_include_filter.py b/test/icat/filters/test_include_filter.py index 48264897..05e7500e 100644 --- a/test/icat/filters/test_include_filter.py +++ b/test/icat/filters/test_include_filter.py @@ -1,6 +1,6 @@ -from datagateway_api.common.exceptions import FilterError import pytest +from datagateway_api.common.exceptions import FilterError from datagateway_api.common.icat.filters import PythonICATIncludeFilter @@ -61,7 +61,7 @@ def test_invalid_field(self, icat_query): [ pytest.param({2: "datasets"}, id="invalid dictionary key"), pytest.param( - {"datasets": {2: "datafiles"}}, id="invalid inner dictionary key" + {"datasets": {2: "datafiles"}}, id="invalid inner dictionary key", ), pytest.param( {"datasets": {"datafiles", "sample"}}, From 2c37baa0687cba125a03940b757327af1a0da173 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 15:46:10 +0000 Subject: [PATCH 061/109] #150: Remove planning comments made at start of branch --- test/test_base.py | 2 -- test/test_database_helpers.py | 2 -- test/test_entityHelper.py | 1 - test/test_helpers.py | 5 ----- 4 files changed, 10 deletions(-) diff --git a/test/test_base.py b/test/test_base.py index 37fbdea9..31677e1d 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -3,8 +3,6 @@ from datagateway_api.src.main import app -# Move this into the test defintions and let it be inherited in test classes that need -# it class FlaskAppTest(TestCase): """ The FlaskAppTest Base class sets up a test client to be used to mock requests diff --git a/test/test_database_helpers.py b/test/test_database_helpers.py index 6ed57755..c97404a5 100644 --- a/test/test_database_helpers.py +++ b/test/test_database_helpers.py @@ -31,8 +31,6 @@ ) -# Common across both, needs parameterized tests for different cased names, need multiple -# tests for each filter class TestQueryFilterFactory(TestCase): def test_order_filter(self): self.assertIs( diff --git a/test/test_entityHelper.py b/test/test_entityHelper.py index f1241473..9fac7c99 100644 --- a/test/test_entityHelper.py +++ b/test/test_entityHelper.py @@ -9,7 +9,6 @@ ) -# DB only tests class TestEntityHelper(TestCase): def setUp(self): self.dataset = DATASET() diff --git a/test/test_helpers.py b/test/test_helpers.py index 059cea1a..ca39f2de 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -30,7 +30,6 @@ from test.test_base import FlaskAppTest -# Put this in test definitions, but do tests there, don't abstract out class TestIsValidJSON(TestCase): def test_array(self): self.assertTrue(is_valid_json("[]")) @@ -57,7 +56,6 @@ def test_list(self): self.assertFalse(is_valid_json([])) -# Common for both backends, setup and teardown will be different class TestRequiresSessionID(FlaskAppTest): def setUp(self): super().setUp() @@ -100,7 +98,6 @@ def test_good_credentials(self): ) -# Common across both, no need to abstract out class TestQueriesRecords(TestCase): def test_missing_record_error(self): @queries_records @@ -168,7 +165,6 @@ def raise_bad_request_error(): self.assertEqual(400, ctx.exception.status_code) -# Common across both, no need to abstract out class TestGetSessionIDFromAuthHeader(FlaskAppTest): def test_no_session_in_header(self): with self.app: @@ -186,7 +182,6 @@ def test_with_good_header(self): self.assertEqual("test", get_session_id_from_auth_header()) -# Common across both, needs abstracting out, class per filter, multiple tests per filter class TestGetFiltersFromQueryString(FlaskAppTest): def test_no_filters(self): with self.app: From 64f9981d339cb175f19cc6dcb95fbf62b03e823a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 30 Nov 2020 19:20:49 +0000 Subject: [PATCH 062/109] #150: Move Flask/API setup into functions - This will help to specify whether a unit test should use the db or python_icat backend, to be fully implemented in future commits --- datagateway_api/src/main.py | 159 ++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 71 deletions(-) diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index da8aba1d..e43090c6 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -8,6 +8,7 @@ from flask_restful import Api from flask_swagger_ui import get_swaggerui_blueprint +from datagateway_api.common.backends import create_backend from datagateway_api.common.config import config from datagateway_api.common.exceptions import ApiError from datagateway_api.common.logger_setup import setup_logger @@ -28,95 +29,108 @@ from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin from datagateway_api.src.swagger.initialise_spec import initialise_spec - -spec = APISpec( - title="DataGateway API", - version="1.0", - openapi_version="3.0.3", - plugins=[RestfulPlugin()], - security=[{"session_id": []}], -) +setup_logger() +log = logging.getLogger() +log.info("Logging now setup") app = Flask(__name__) -cors = CORS(app) -app.url_map.strict_slashes = False -api = Api(app) - - -def handle_error(e): - return str(e), e.status_code - - -app.register_error_handler(ApiError, handle_error) - swaggerui_blueprint = get_swaggerui_blueprint( "", "/openapi.json", config={"app_name": "DataGateway API OpenAPI Spec"}, ) - app.register_blueprint(swaggerui_blueprint, url_prefix="/") -setup_logger() -log = logging.getLogger() -log.info("Logging now setup") -initialise_spec(spec) +def create_app_infrastructure(app): + spec = APISpec( + title="DataGateway API", + version="1.0", + openapi_version="3.0.3", + plugins=[RestfulPlugin()], + security=[{"session_id": []}], + ) -for entity_name in endpoints: - get_endpoint_resource = get_endpoint(entity_name, endpoints[entity_name]) - api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") - spec.path(resource=get_endpoint_resource, api=api) + cors = CORS(app) + app.url_map.strict_slashes = False + api = Api(app) - get_id_endpoint_resource = get_id_endpoint(entity_name, endpoints[entity_name]) - api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") - spec.path(resource=get_id_endpoint_resource, api=api) + app.register_error_handler(ApiError, handle_error) - get_count_endpoint_resource = get_count_endpoint( - entity_name, endpoints[entity_name], - ) - api.add_resource(get_count_endpoint_resource, f"/{entity_name.lower()}/count") - spec.path(resource=get_count_endpoint_resource, api=api) + initialise_spec(spec) - get_find_one_endpoint_resource = get_find_one_endpoint( - entity_name, endpoints[entity_name], - ) - api.add_resource(get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone") - spec.path(resource=get_find_one_endpoint_resource, api=api) + return (api, spec) -# Session endpoint -api.add_resource(Sessions, "/sessions") -spec.path(resource=Sessions, api=api) +def handle_error(e): + return str(e), e.status_code + + +def create_api_endpoints(app, api, spec): + try: + backend_type = app.config["TEST_BACKEND"] + except KeyError: + backend_type = config.get_backend_type() + + backend = create_backend(backend_type) + + for entity_name in endpoints: + get_endpoint_resource = get_endpoint(entity_name, endpoints[entity_name]) + api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") + spec.path(resource=get_endpoint_resource, api=api) + + get_id_endpoint_resource = get_id_endpoint(entity_name, endpoints[entity_name]) + api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") + spec.path(resource=get_id_endpoint_resource, api=api) + + get_count_endpoint_resource = get_count_endpoint( + entity_name, endpoints[entity_name], + ) + api.add_resource(get_count_endpoint_resource, f"/{entity_name.lower()}/count") + spec.path(resource=get_count_endpoint_resource, api=api) + + get_find_one_endpoint_resource = get_find_one_endpoint( + entity_name, endpoints[entity_name], + ) + api.add_resource( + get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone" + ) + spec.path(resource=get_find_one_endpoint_resource, api=api) + + # Session endpoint + api.add_resource(Sessions, "/sessions") + spec.path(resource=Sessions, api=api) + + # Table specific endpoints + api.add_resource(InstrumentsFacilityCycles, "/instruments//facilitycycles") + spec.path(resource=InstrumentsFacilityCycles, api=api) + api.add_resource( + InstrumentsFacilityCyclesCount, "/instruments//facilitycycles/count", + ) + spec.path(resource=InstrumentsFacilityCyclesCount, api=api) + api.add_resource( + InstrumentsFacilityCyclesInvestigations, + "/instruments//facilitycycles//investigations", + ) + spec.path(resource=InstrumentsFacilityCyclesInvestigations, api=api) + api.add_resource( + InstrumentsFacilityCyclesInvestigationsCount, + "/instruments//facilitycycles//investigations" + "/count", + ) + spec.path(resource=InstrumentsFacilityCyclesInvestigationsCount, api=api) -# Table specific endpoints -api.add_resource(InstrumentsFacilityCycles, "/instruments//facilitycycles") -spec.path(resource=InstrumentsFacilityCycles, api=api) -api.add_resource( - InstrumentsFacilityCyclesCount, "/instruments//facilitycycles/count", -) -spec.path(resource=InstrumentsFacilityCyclesCount, api=api) -api.add_resource( - InstrumentsFacilityCyclesInvestigations, - "/instruments//facilitycycles//investigations", -) -spec.path(resource=InstrumentsFacilityCyclesInvestigations, api=api) -api.add_resource( - InstrumentsFacilityCyclesInvestigationsCount, - "/instruments//facilitycycles//investigations" - "/count", -) -spec.path(resource=InstrumentsFacilityCyclesInvestigationsCount, api=api) -# Reorder paths (e.g. get, patch, post) so openapi.yaml only changes when there's a -# change to the Swagger docs, rather than changing on each startup -log.debug("Reordering OpenAPI docs to alphabetical order") -for entity_data in spec._paths.values(): - for endpoint_name in sorted(entity_data.keys()): - entity_data.move_to_end(endpoint_name) +def openapi_config(spec): + # Reorder paths (e.g. get, patch, post) so openapi.yaml only changes when there's a + # change to the Swagger docs, rather than changing on each startup + log.debug("Reordering OpenAPI docs to alphabetical order") + for entity_data in spec._paths.values(): + for endpoint_name in sorted(entity_data.keys()): + entity_data.move_to_end(endpoint_name) -openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml" -with open(openapi_spec_path, "w") as f: - f.write(spec.to_yaml()) + openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml" + with open(openapi_spec_path, "w") as f: + f.write(spec.to_yaml()) @app.route("/openapi.json") @@ -127,6 +141,9 @@ def specs(): if __name__ == "__main__": + api, spec = create_app_infrastructure() + create_api_endpoints(app, api, spec) + openapi_config(spec) app.run( host=config.get_host(), port=config.get_port(), debug=config.is_debug_mode(), ) From 9532d910ffd4491506f221a33b2cd0a22c70763e Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 2 Dec 2020 16:41:43 +0000 Subject: [PATCH 063/109] #150: Checkpoint in making backend configurable for testing --- datagateway_api/src/main.py | 67 ++- .../src/resources/entities/entity_endpoint.py | 10 +- .../non_entities/sessions_endpoints.py | 277 +++++------ .../table_endpoints/table_endpoints.py | 443 ++++++++++-------- test/icat/endpoints/test_endpoint_rules.py | 6 +- 5 files changed, 437 insertions(+), 366 deletions(-) diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index e43090c6..3224ac09 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -19,12 +19,14 @@ get_id_endpoint, ) from datagateway_api.src.resources.entities.entity_map import endpoints -from datagateway_api.src.resources.non_entities.sessions_endpoints import Sessions +from datagateway_api.src.resources.non_entities.sessions_endpoints import ( + session_endpoints, +) from datagateway_api.src.resources.table_endpoints.table_endpoints import ( - InstrumentsFacilityCycles, - InstrumentsFacilityCyclesCount, - InstrumentsFacilityCyclesInvestigations, - InstrumentsFacilityCyclesInvestigationsCount, + count_instrument_facility_cycles_endpoint, + instrument_facility_cycles_endpoint, + instrument_investigation_endpoint, + count_instrument_investigation_endpoint, ) from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin from datagateway_api.src.swagger.initialise_spec import initialise_spec @@ -68,56 +70,79 @@ def handle_error(e): def create_api_endpoints(app, api, spec): try: backend_type = app.config["TEST_BACKEND"] + print(f"test backend: {backend_type}") except KeyError: backend_type = config.get_backend_type() + print(f"config backend: {backend_type}") + # TODO - Add :param backend: to the endpoint functions backend = create_backend(backend_type) + print(f"Backend: {backend}, Type: {type(backend)}") for entity_name in endpoints: - get_endpoint_resource = get_endpoint(entity_name, endpoints[entity_name]) + get_endpoint_resource = get_endpoint( + entity_name, endpoints[entity_name], backend + ) api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") spec.path(resource=get_endpoint_resource, api=api) - get_id_endpoint_resource = get_id_endpoint(entity_name, endpoints[entity_name]) + get_id_endpoint_resource = get_id_endpoint( + entity_name, endpoints[entity_name], backend + ) api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") spec.path(resource=get_id_endpoint_resource, api=api) get_count_endpoint_resource = get_count_endpoint( - entity_name, endpoints[entity_name], + entity_name, endpoints[entity_name], backend, ) api.add_resource(get_count_endpoint_resource, f"/{entity_name.lower()}/count") spec.path(resource=get_count_endpoint_resource, api=api) get_find_one_endpoint_resource = get_find_one_endpoint( - entity_name, endpoints[entity_name], + entity_name, endpoints[entity_name], backend, ) api.add_resource( - get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone" + get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone", ) spec.path(resource=get_find_one_endpoint_resource, api=api) # Session endpoint - api.add_resource(Sessions, "/sessions") - spec.path(resource=Sessions, api=api) + session_endpoint_resource = session_endpoints(backend) + api.add_resource(session_endpoint_resource, "/sessions") + # spec.path(resource=session_endpoint_resource, api=api) # Table specific endpoints - api.add_resource(InstrumentsFacilityCycles, "/instruments//facilitycycles") - spec.path(resource=InstrumentsFacilityCycles, api=api) + instrument_facility_cycle_resource = instrument_facility_cycles_endpoint(backend) + api.add_resource( + instrument_facility_cycle_resource, "/instruments//facilitycycles" + ) + # spec.path(resource=instrument_facility_cycle_resource, api=api) + + count_instrument_facility_cycle_resource = count_instrument_facility_cycles_endpoint( + backend + ) api.add_resource( - InstrumentsFacilityCyclesCount, "/instruments//facilitycycles/count", + count_instrument_facility_cycle_resource, + "/instruments//facilitycycles/count", ) - spec.path(resource=InstrumentsFacilityCyclesCount, api=api) + # spec.path(resource=count_instrument_facility_cycle_resource, api=api) + + instrument_investigation_resource = instrument_investigation_endpoint(backend) api.add_resource( - InstrumentsFacilityCyclesInvestigations, + instrument_investigation_resource, "/instruments//facilitycycles//investigations", ) - spec.path(resource=InstrumentsFacilityCyclesInvestigations, api=api) + # spec.path(resource=instrument_investigation_resource, api=api) + + count_instrument_investigation_resource = count_instrument_investigation_endpoint( + backend + ) api.add_resource( - InstrumentsFacilityCyclesInvestigationsCount, + count_instrument_investigation_resource, "/instruments//facilitycycles//investigations" "/count", ) - spec.path(resource=InstrumentsFacilityCyclesInvestigationsCount, api=api) + # spec.path(resource=count_instrument_investigation_resource, api=api) def openapi_config(spec): @@ -141,7 +166,7 @@ def specs(): if __name__ == "__main__": - api, spec = create_app_infrastructure() + api, spec = create_app_infrastructure(app) create_api_endpoints(app, api, spec) openapi_config(spec) app.run( diff --git a/datagateway_api/src/resources/entities/entity_endpoint.py b/datagateway_api/src/resources/entities/entity_endpoint.py index 568508b6..3446913d 100644 --- a/datagateway_api/src/resources/entities/entity_endpoint.py +++ b/datagateway_api/src/resources/entities/entity_endpoint.py @@ -8,10 +8,10 @@ get_session_id_from_auth_header, ) -backend = create_backend(config.get_backend_type()) +# backend = create_backend(config.get_backend_type()) -def get_endpoint(name, entity_type): +def get_endpoint(name, entity_type, backend): """ Given an entity name generate a flask_restful Resource class. In main.py these generated classes are registered with the api e.g @@ -159,7 +159,7 @@ def patch(self): return Endpoint -def get_id_endpoint(name, entity_type): +def get_id_endpoint(name, entity_type, backend): """ Given an entity name generate a flask_restful Resource class. In main.py these generated classes are registered with the api e.g @@ -289,7 +289,7 @@ def patch(self, id_): return EndpointWithID -def get_count_endpoint(name, entity_type): +def get_count_endpoint(name, entity_type, backend): """ Given an entity name generate a flask_restful Resource class. In main.py these generated classes are registered with the api e.g @@ -342,7 +342,7 @@ def get(self): return CountEndpoint -def get_find_one_endpoint(name, entity_type): +def get_find_one_endpoint(name, entity_type, backend): """ Given an entity name generate a flask_restful Resource class. In main.py these generated classes are registered with the api e.g diff --git a/datagateway_api/src/resources/non_entities/sessions_endpoints.py b/datagateway_api/src/resources/non_entities/sessions_endpoints.py index bfb253d7..36a33922 100644 --- a/datagateway_api/src/resources/non_entities/sessions_endpoints.py +++ b/datagateway_api/src/resources/non_entities/sessions_endpoints.py @@ -11,145 +11,154 @@ log = logging.getLogger() -backend = create_backend(config.get_backend_type()) +# backend = create_backend(config.get_backend_type()) -class Sessions(Resource): - def post(self): - """ - Generates a sessionID if the user has correct credentials - :return: String - SessionID +def session_endpoints(backend): + """ + TODO - Add docstring + """ + log.info("test") - --- - summary: Login - description: Generates a sessionID if the user has correct credentials - tags: - - Sessions - security: [] - requestBody: - description: User credentials to login with - required: true - content: - application/json: - schema: - type: object - properties: - username: - type: string - password: - type: string - mechanism: - type: string - responses: - 201: - description: Success - returns a session ID + class Sessions(Resource): + def post(self): + """ + Generates a sessionID if the user has correct credentials + :return: String - SessionID + --- + summary: Login + description: Generates a sessionID if the user has correct credentials + tags: + - Sessions + security: [] + requestBody: + description: User credentials to login with + required: true content: - application/json: + application/json: schema: - type: object - properties: - sessionID: - type: string - description: Session ID - example: xxxxxx-yyyyyyy-zzzzzz - 400: - description: Bad request. User credentials not provided in request body - 403: - description: Forbidden. User credentials were invalid - """ - if not ( - request.data and "username" in request.json and "password" in request.json - ): - return "Bad request", 400 - # If no mechanism is present in request body, default to simple - if not ("mechanism" in request.json): - request.json["mechanism"] = "simple" - try: - return {"sessionID": backend.login(request.json)}, 201 - except AuthenticationError: - return "Forbidden", 403 + type: object + properties: + username: + type: string + password: + type: string + mechanism: + type: string + responses: + 201: + description: Success - returns a session ID + content: + application/json: + schema: + type: object + properties: + sessionID: + type: string + description: Session ID + example: xxxxxx-yyyyyyy-zzzzzz + 400: + description: Bad request. User credentials not provided in request body + 403: + description: Forbidden. User credentials were invalid + """ + if not ( + request.data + and "username" in request.json + and "password" in request.json + ): + return "Bad request", 400 + # If no mechanism is present in request body, default to simple + if not ("mechanism" in request.json): + request.json["mechanism"] = "simple" + try: + return ({"sessionID": backend.login(request.json)}, 201) + except AuthenticationError: + return ("Forbidden", 403) - def delete(self): - """ - Deletes a users sessionID when they logout - :return: Blank response, 200 - --- - summary: Delete session - description: Deletes a users sessionID when they logout - tags: - - Sessions - responses: - 200: - description: Success - User's session was successfully deleted - 400: - description: Bad request - something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: Not Found - Unable to find session ID - """ - backend.logout(get_session_id_from_auth_header()) - return "", 200 + def delete(self): + """ + Deletes a users sessionID when they logout + :return: Blank response, 200 + --- + summary: Delete session + description: Deletes a users sessionID when they logout + tags: + - Sessions + responses: + 200: + description: Success - User's session was successfully deleted + 400: + description: Bad request - something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: Not Found - Unable to find session ID + """ + backend.logout(get_session_id_from_auth_header()) + return ("", 200) - def get(self): - """ - Gives details of a users session - :return: String: Details of the session, 200 - --- - summary: Get session details - description: Gives details of a user's session - tags: - - Sessions - responses: - 200: - description: Success - a user's session details - content: - application/json: - schema: - type: object - properties: - ID: - type: string - description: The session ID - example: xxxxxx-yyyyyyy-zzzzzz - EXPIREDATETIME: - type: string - format: datetime - description: When this session expires - example: "2017-07-21T17:32:28Z" - USERNAME: - type: string - description: Username associated with this session - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - """ - return backend.get_session_details(get_session_id_from_auth_header()), 200 + def get(self): + """ + Gives details of a users session + :return: String: Details of the session, 200 + --- + summary: Get session details + description: Gives details of a user's session + tags: + - Sessions + responses: + 200: + description: Success - a user's session details + content: + application/json: + schema: + type: object + properties: + ID: + type: string + description: The session ID + example: xxxxxx-yyyyyyy-zzzzzz + EXPIREDATETIME: + type: string + format: datetime + description: When this session expires + example: "2017-07-21T17:32:28Z" + USERNAME: + type: string + description: Username associated with this session + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + """ + return (backend.get_session_details(get_session_id_from_auth_header()), 200) - def put(self): - """ - Refreshes a users session - :return: String: The session ID that has been refreshed, 200 - --- - summary: Refresh session - description: Refreshes a users session - tags: - - Sessions - responses: - 200: - description: Success - the user's session ID that has been refreshed - content: - application/json: - schema: - type: string - description: Session ID - example: xxxxxx-yyyyyyy-zzzzzz - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - """ - return backend.refresh(get_session_id_from_auth_header()), 200 + def put(self): + """ + Refreshes a users session + :return: String: The session ID that has been refreshed, 200 + --- + summary: Refresh session + description: Refreshes a users session + tags: + - Sessions + responses: + 200: + description: Success - the user's session ID that has been refreshed + content: + application/json: + schema: + type: string + description: Session ID + example: xxxxxx-yyyyyyy-zzzzzz + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + """ + return (backend.refresh(get_session_id_from_auth_header()), 200) + + return Sessions diff --git a/datagateway_api/src/resources/table_endpoints/table_endpoints.py b/datagateway_api/src/resources/table_endpoints/table_endpoints.py index ee539462..507d3295 100644 --- a/datagateway_api/src/resources/table_endpoints/table_endpoints.py +++ b/datagateway_api/src/resources/table_endpoints/table_endpoints.py @@ -10,206 +10,243 @@ backend = create_backend(config.get_backend_type()) -class InstrumentsFacilityCycles(Resource): - def get(self, id_): - """ - --- - summary: Get an Instrument's FacilityCycles - description: Given an Instrument id get facility cycles where the instrument has - investigations that occur within that cycle, subject to the given filters - tags: - - FacilityCycles - parameters: - - in: path - required: true - name: id - description: The id of the instrument to retrieve the facility cycles of - schema: - type: integer - - WHERE_FILTER - - ORDER_FILTER - - LIMIT_FILTER - - SKIP_FILTER - - DISTINCT_FILTER - - INCLUDE_FILTER - responses: - 200: - description: Success - returns a list of the instrument's facility - cycles that satisfy the filters - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/FACILITYCYCLE' - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_facility_cycles_for_instrument_with_filters( - get_session_id_from_auth_header(), id_, get_filters_from_query_string(), - ), - 200, - ) - - -class InstrumentsFacilityCyclesCount(Resource): - def get(self, id_): - """ - --- - summary: Count an Instrument's FacilityCycles - description: Return the count of the Facility Cycles that have investigations - that occur within that cycle on the specified instrument that would be - retrieved given the filters provided - tags: - - FacilityCycles - parameters: - - in: path - required: true - name: id - description: The id of the instrument to count the facility cycles of - schema: - type: integer - - WHERE_FILTER - - DISTINCT_FILTER - responses: - 200: - description: Success - The count of the instrument's facility cycles - that satisfy the filters - content: - application/json: - schema: - type: integer - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_facility_cycles_for_instrument_count_with_filters( - get_session_id_from_auth_header(), id_, get_filters_from_query_string(), - ), - 200, - ) - - -class InstrumentsFacilityCyclesInvestigations(Resource): - def get(self, instrument_id, cycle_id): - """ - --- - summary: Get the investigations for a given Facility Cycle & Instrument - description: Given an Instrument id and Facility Cycle id, get the - investigations that occur within that cycle on that instrument, subject to - the given filters - tags: - - Investigations - parameters: - - in: path - required: true - name: instrument_id - description: The id of the instrument to retrieve the investigations of - schema: - type: integer - - in: path - required: true - name: cycle_id - description: The id of the facility cycle to retrieve the investigations - schema: - type: integer - - WHERE_FILTER - - ORDER_FILTER - - LIMIT_FILTER - - SKIP_FILTER - - DISTINCT_FILTER - - INCLUDE_FILTER - responses: - 200: - description: Success - returns a list of the investigations for the - given instrument and facility cycle that satisfy the filters - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/INVESTIGATION' - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_investigations_for_instrument_in_facility_cycle_with_filters( - get_session_id_from_auth_header(), - instrument_id, - cycle_id, - get_filters_from_query_string(), - ), - 200, - ) - - -class InstrumentsFacilityCyclesInvestigationsCount(Resource): - def get(self, instrument_id, cycle_id): - """ - --- - summary: Count investigations for a given Facility Cycle & Instrument - description: Given an Instrument id and Facility Cycle id, get the number of - investigations that occur within that cycle on that instrument, subject to - the given filters - tags: - - Investigations - parameters: - - in: path - required: true - name: instrument_id - description: The id of the instrument to retrieve the investigations of - schema: - type: integer - - in: path - required: true - name: cycle_id - description: The id of the facility cycle to retrieve the investigations - schema: - type: integer - - WHERE_FILTER - - DISTINCT_FILTER - responses: - 200: - description: Success - The count of the investigations for the given - instrument and facility cycle that satisfy the filters - content: - application/json: - schema: - type: integer - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_investigation_count_for_instrument_facility_cycle_with_filters( - get_session_id_from_auth_header(), - instrument_id, - cycle_id, - get_filters_from_query_string(), - ), - 200, - ) +def instrument_facility_cycles_endpoint(backend): + """ + TODO - Add docstring + """ + pass + + class InstrumentsFacilityCycles(Resource): + def get(self, id_): + """ + --- + summary: Get an Instrument's FacilityCycles + description: Given an Instrument id get facility cycles where the instrument + has investigations that occur within that cycle, subject to the given + filters + tags: + - FacilityCycles + parameters: + - in: path + required: true + name: id + description: The id of the instrument to retrieve the facility cycles of + schema: + type: integer + - WHERE_FILTER + - ORDER_FILTER + - LIMIT_FILTER + - SKIP_FILTER + - DISTINCT_FILTER + - INCLUDE_FILTER + responses: + 200: + description: Success - returns a list of the instrument's facility + cycles that satisfy the filters + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/FACILITYCYCLE' + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_facility_cycles_for_instrument_with_filters( + get_session_id_from_auth_header(), + id_, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCycles + + +def count_instrument_facility_cycles_endpoint(backend): + """ + TODO - Add docstring + """ + pass + + class InstrumentsFacilityCyclesCount(Resource): + def get(self, id_): + """ + --- + summary: Count an Instrument's FacilityCycles + description: Return the count of the Facility Cycles that have + investigations that occur within that cycle on the specified instrument + that would be retrieved given the filters provided + tags: + - FacilityCycles + parameters: + - in: path + required: true + name: id + description: The id of the instrument to count the facility cycles of + schema: + type: integer + - WHERE_FILTER + - DISTINCT_FILTER + responses: + 200: + description: Success - The count of the instrument's facility cycles + that satisfy the filters + content: + application/json: + schema: + type: integer + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_facility_cycles_for_instrument_count_with_filters( + get_session_id_from_auth_header(), + id_, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCyclesCount + + +def instrument_investigation_endpoint(backend): + """ + TODO - Add docstring + """ + pass + + class InstrumentsFacilityCyclesInvestigations(Resource): + def get(self, instrument_id, cycle_id): + """ + --- + summary: Get the investigations for a given Facility Cycle & Instrument + description: Given an Instrument id and Facility Cycle id, get the + investigations that occur within that cycle on that instrument, subject + to the given filters + tags: + - Investigations + parameters: + - in: path + required: true + name: instrument_id + description: The id of the instrument to retrieve the investigations of + schema: + type: integer + - in: path + required: true + name: cycle_id + description: The id of the facility cycle to retrieve the investigations + schema: + type: integer + - WHERE_FILTER + - ORDER_FILTER + - LIMIT_FILTER + - SKIP_FILTER + - DISTINCT_FILTER + - INCLUDE_FILTER + responses: + 200: + description: Success - returns a list of the investigations for the + given instrument and facility cycle that satisfy the filters + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/INVESTIGATION' + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_investigations_for_instrument_in_facility_cycle_with_filters( + get_session_id_from_auth_header(), + instrument_id, + cycle_id, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCyclesInvestigations + + +def count_instrument_investigation_endpoint(backend): + """ + TODO - Add docstring + """ + pass + + class InstrumentsFacilityCyclesInvestigationsCount(Resource): + def get(self, instrument_id, cycle_id): + """ + --- + summary: Count investigations for a given Facility Cycle & Instrument + description: Given an Instrument id and Facility Cycle id, get the number of + investigations that occur within that cycle on that instrument, subject + to the given filters + tags: + - Investigations + parameters: + - in: path + required: true + name: instrument_id + description: The id of the instrument to retrieve the investigations of + schema: + type: integer + - in: path + required: true + name: cycle_id + description: The id of the facility cycle to retrieve the investigations + schema: + type: integer + - WHERE_FILTER + - DISTINCT_FILTER + responses: + 200: + description: Success - The count of the investigations for the given + instrument and facility cycle that satisfy the filters + content: + application/json: + schema: + type: integer + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_investigation_count_for_instrument_facility_cycle_with_filters( + get_session_id_from_auth_header(), + instrument_id, + cycle_id, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCyclesInvestigationsCount diff --git a/test/icat/endpoints/test_endpoint_rules.py b/test/icat/endpoints/test_endpoint_rules.py index b92d8ff8..5259fec7 100644 --- a/test/icat/endpoints/test_endpoint_rules.py +++ b/test/icat/endpoints/test_endpoint_rules.py @@ -1,6 +1,6 @@ import pytest -from datagateway_api.src.main import api +from datagateway_api.src.main import app from datagateway_api.src.resources.entities.entity_map import endpoints @@ -22,7 +22,7 @@ def test_entity_endpoints(self, endpoint_ending, expected_methods): for endpoint_entity in endpoints.keys(): endpoint_found = False - for rule in api.app.url_map.iter_rules(): + for rule in app.url_map.iter_rules(): if f"/{endpoint_entity.lower()}{endpoint_ending}" == rule.rule: endpoint_found = True @@ -64,7 +64,7 @@ def test_entity_endpoints(self, endpoint_ending, expected_methods): def test_non_entity_endpoints(self, endpoint_name, expected_methods): endpoint_found = False - for rule in api.app.url_map.iter_rules(): + for rule in app.url_map.iter_rules(): if endpoint_name == rule.rule: endpoint_found = True From 69029214a0d0a2c12351632e32f92e608ad37453 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 2 Dec 2020 16:44:04 +0000 Subject: [PATCH 064/109] #150: Expand flask test app so the backend can be configured - This doesn't fully work yet because filters are still created based on contents of config.json --- test/conftest.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 92f4f80a..af5cb304 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,13 +1,18 @@ from datetime import datetime import uuid +from flask import Flask from icat.client import Client from icat.exception import ICATNoObjectError from icat.query import Query import pytest from datagateway_api.common.config import config -from datagateway_api.src.main import app +from datagateway_api.src.main import ( + app, + create_api_endpoints, + create_app_infrastructure, +) from test.icat.test_query import prepare_icat_data_for_assertion @@ -91,8 +96,16 @@ def multiple_investigation_test_data(icat_client): @pytest.fixture() def flask_test_app(): - app.config["TESTING"] = True - return app.test_client() + my_app = Flask(__name__) + my_app.config["TESTING"] = True + my_app.config["TEST_BACKEND"] = "python_icat" + + api, spec = create_app_infrastructure(my_app) + create_api_endpoints(my_app, api, spec) + + yield my_app.test_client() + + # app.url_map._rules.clear() @pytest.fixture() From 7a6a8ef2e0ee46f8de60c7823091818dc9d51619 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 10:10:34 +0000 Subject: [PATCH 065/109] #150: Move QueryFilterFactory to its own file - It used to be in the database helpers file, but this class is used across both backends - The imports have been moved inside the staticmethod because the Flask TEST_BACKEND config option won't have been set by the time QueryFilterFactory has been imported, so the imports which decide which set of filters to import must be put in some code. --- datagateway_api/common/database/helpers.py | 36 ----------- datagateway_api/common/helpers.py | 2 +- datagateway_api/common/query_filter.py | 72 ++++++++++++++++++++++ 3 files changed, 73 insertions(+), 37 deletions(-) create mode 100644 datagateway_api/common/query_filter.py diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index cf682cd8..68df4267 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -202,42 +202,6 @@ def execute_query(self): self.commit_changes() -class QueryFilterFactory(object): - @staticmethod - def get_query_filter(request_filter): - """ - Given a filter return a matching Query filter object - - This factory is not in common.filters so the created filter can be for the - correct backend. Moving the factory into that file would mean the filters would - be based off the abstract classes (because they're in the same file) which won't - enable filters to be unique to the backend - - :param request_filter: dict - The filter to create the QueryFilter for - :return: The QueryFilter object created - """ - filter_name = list(request_filter)[0].lower() - if filter_name == "where": - field = list(request_filter[filter_name].keys())[0] - operation = list(request_filter[filter_name][field].keys())[0] - value = request_filter[filter_name][field][operation] - return WhereFilter(field, value, operation) - elif filter_name == "order": - field = request_filter["order"].split(" ")[0] - direction = request_filter["order"].split(" ")[1] - return OrderFilter(field, direction) - elif filter_name == "skip": - return SkipFilter(request_filter["skip"]) - elif filter_name == "limit": - return LimitFilter(request_filter["limit"]) - elif filter_name == "include": - return IncludeFilter(request_filter["include"]) - elif filter_name == "distinct": - return DistinctFieldFilter(request_filter["distinct"]) - else: - raise FilterError(f" Bad filter: {request_filter}") - - def insert_row_into_table(table, row): """ Insert the given row into its table diff --git a/datagateway_api/common/helpers.py b/datagateway_api/common/helpers.py index 9115b09f..55dec407 100644 --- a/datagateway_api/common/helpers.py +++ b/datagateway_api/common/helpers.py @@ -6,7 +6,7 @@ from flask_restful import reqparse from sqlalchemy.exc import IntegrityError -from datagateway_api.common.database.helpers import QueryFilterFactory +from datagateway_api.common.query_filter import QueryFilterFactory from datagateway_api.common.exceptions import ( ApiError, AuthenticationError, diff --git a/datagateway_api/common/query_filter.py b/datagateway_api/common/query_filter.py new file mode 100644 index 00000000..f81067b3 --- /dev/null +++ b/datagateway_api/common/query_filter.py @@ -0,0 +1,72 @@ +import logging + +from datagateway_api.common.config import config +from datagateway_api.common.exceptions import ( + ApiError, + FilterError, +) + +log = logging.getLogger() + + +class QueryFilterFactory(object): + @staticmethod + def get_query_filter(request_filter): + """ + Given a filter return a matching Query filter object + + This factory is not in common.filters so the created filter can be for the + correct backend. Moving the factory into that file would mean the filters would + be based off the abstract classes (because they're in the same file) which won't + enable filters to be unique to the backend + + :param request_filter: dict - The filter to create the QueryFilter for + :return: The QueryFilter object created + """ + + backend_type = config.get_backend_type() + print(f"Backend inside staticmethod: {backend_type}") + if backend_type == "db": + from datagateway_api.common.database.filters import ( + DatabaseDistinctFieldFilter as DistinctFieldFilter, + DatabaseIncludeFilter as IncludeFilter, + DatabaseLimitFilter as LimitFilter, + DatabaseOrderFilter as OrderFilter, + DatabaseSkipFilter as SkipFilter, + DatabaseWhereFilter as WhereFilter, + ) + elif backend_type == "python_icat": + from datagateway_api.common.icat.filters import ( + PythonICATDistinctFieldFilter as DistinctFieldFilter, + PythonICATIncludeFilter as IncludeFilter, + PythonICATLimitFilter as LimitFilter, + PythonICATOrderFilter as OrderFilter, + PythonICATSkipFilter as SkipFilter, + PythonICATWhereFilter as WhereFilter, + ) + else: + raise ApiError( + "Cannot select which implementation of filters to import, check the" + " config file has a valid backend type", + ) + + filter_name = list(request_filter)[0].lower() + if filter_name == "where": + field = list(request_filter[filter_name].keys())[0] + operation = list(request_filter[filter_name][field].keys())[0] + value = request_filter[filter_name][field][operation] + return WhereFilter(field, value, operation) + elif filter_name == "order": + field = request_filter["order"].split(" ")[0] + direction = request_filter["order"].split(" ")[1] + return OrderFilter(field, direction) + elif filter_name == "skip": + return SkipFilter(request_filter["skip"]) + elif filter_name == "limit": + return LimitFilter(request_filter["limit"]) + elif filter_name == "include": + return IncludeFilter(request_filter["include"]) + elif filter_name == "distinct": + return DistinctFieldFilter(request_filter["distinct"]) + else: + raise FilterError(f" Bad filter: {request_filter}") From 70dc3937243eee3825beef48b291fea93f2a2771 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 10:26:56 +0000 Subject: [PATCH 066/109] #150: Allow backend type to be set when a flask test fixture runs - This will allow tests to run on a backend set by the fixture instead of relying on the contents of config.json. This means both backends can be tested in one test session and the contents of the config file doesn't matter --- datagateway_api/common/config.py | 7 +++++++ datagateway_api/src/main.py | 1 + test/conftest.py | 17 +++++++++-------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index cdc26b97..6f69c16e 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -5,6 +5,7 @@ import requests +# from datagateway_api.src.main import app log = logging.getLogger() @@ -22,6 +23,12 @@ def get_backend_type(self): except KeyError: sys.exit("Missing config value, backend") + def set_backend_type(self, backend_type): + """ + TODO - Explain the reason behind the setter + """ + self.config["backend"] = backend_type + def get_db_url(self): try: return self.config["DB_URL"] diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 3224ac09..6e680b03 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -70,6 +70,7 @@ def handle_error(e): def create_api_endpoints(app, api, spec): try: backend_type = app.config["TEST_BACKEND"] + config.set_backend_type(backend_type) print(f"test backend: {backend_type}") except KeyError: backend_type = config.get_backend_type() diff --git a/test/conftest.py b/test/conftest.py index af5cb304..dc80ab28 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -94,18 +94,19 @@ def multiple_investigation_test_data(icat_client): icat_client.delete(investigation) -@pytest.fixture() +@pytest.fixture(scope="package") def flask_test_app(): - my_app = Flask(__name__) - my_app.config["TESTING"] = True - my_app.config["TEST_BACKEND"] = "python_icat" + # my_app = Flask(__name__) + app.config["TESTING"] = True + app.config["TEST_BACKEND"] = "python_icat" - api, spec = create_app_infrastructure(my_app) - create_api_endpoints(my_app, api, spec) + api, spec = create_app_infrastructure(app) + create_api_endpoints(app, api, spec) - yield my_app.test_client() + yield app.test_client() - # app.url_map._rules.clear() + # app.url_map._rules.clear() + # app.url_map._rules_by_endpoint.clear() @pytest.fixture() From 3890c5522cce5914f7064e1fc8764d4fa8f1298a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 10:54:37 +0000 Subject: [PATCH 067/109] #150: Add separate flask app fixtures for each backend - The DB app will be used when I rewrite the old tests into pytest, tests which will remain for testing the DB backend --- datagateway_api/src/main.py | 23 ++++---- test/conftest.py | 27 ++++++--- .../icat/endpoints/test_count_with_filters.py | 10 ++-- test/icat/endpoints/test_create.py | 25 +++++---- test/icat/endpoints/test_delete_by_id.py | 15 +++-- test/icat/endpoints/test_findone.py | 11 ++-- test/icat/endpoints/test_get_by_id.py | 15 +++-- test/icat/endpoints/test_get_with_filters.py | 19 ++++--- test/icat/endpoints/test_table_endpoints.py | 56 +++++++++++-------- test/icat/endpoints/test_update_by_id.py | 14 +++-- test/icat/endpoints/test_update_multiple.py | 22 +++++--- test/icat/test_session_handling.py | 34 +++++------ 12 files changed, 163 insertions(+), 108 deletions(-) diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 6e680b03..8162e344 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -37,13 +37,12 @@ app = Flask(__name__) -swaggerui_blueprint = get_swaggerui_blueprint( - "", "/openapi.json", config={"app_name": "DataGateway API OpenAPI Spec"}, -) -app.register_blueprint(swaggerui_blueprint, url_prefix="/") - -def create_app_infrastructure(app): +def create_app_infrastructure(flask_app): + swaggerui_blueprint = get_swaggerui_blueprint( + "", "/openapi.json", config={"app_name": "DataGateway API OpenAPI Spec"}, + ) + flask_app.register_blueprint(swaggerui_blueprint, url_prefix="/") spec = APISpec( title="DataGateway API", version="1.0", @@ -52,11 +51,11 @@ def create_app_infrastructure(app): security=[{"session_id": []}], ) - cors = CORS(app) - app.url_map.strict_slashes = False - api = Api(app) + cors = CORS(flask_app) + flask_app.url_map.strict_slashes = False + api = Api(flask_app) - app.register_error_handler(ApiError, handle_error) + flask_app.register_error_handler(ApiError, handle_error) initialise_spec(spec) @@ -67,9 +66,9 @@ def handle_error(e): return str(e), e.status_code -def create_api_endpoints(app, api, spec): +def create_api_endpoints(flask_app, api, spec): try: - backend_type = app.config["TEST_BACKEND"] + backend_type = flask_app.config["TEST_BACKEND"] config.set_backend_type(backend_type) print(f"test backend: {backend_type}") except KeyError: diff --git a/test/conftest.py b/test/conftest.py index dc80ab28..10d822ab 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -95,18 +95,27 @@ def multiple_investigation_test_data(icat_client): @pytest.fixture(scope="package") -def flask_test_app(): - # my_app = Flask(__name__) - app.config["TESTING"] = True - app.config["TEST_BACKEND"] = "python_icat" +def flask_test_app_icat(): + icat_app = Flask(__name__) + icat_app.config["TESTING"] = True + icat_app.config["TEST_BACKEND"] = "python_icat" - api, spec = create_app_infrastructure(app) - create_api_endpoints(app, api, spec) + api, spec = create_app_infrastructure(icat_app) + create_api_endpoints(icat_app, api, spec) - yield app.test_client() + yield icat_app.test_client() - # app.url_map._rules.clear() - # app.url_map._rules_by_endpoint.clear() + +@pytest.fixture(scope="package") +def flask_test_app_db(): + db_app = Flask(__name__) + db_app.config["TESTING"] = True + db_app.config["TEST_BACKEND"] = "db" + + api, spec = create_app_infrastructure(db_app) + create_api_endpoints(db_app, api, spec) + + yield db_app.test_client() @pytest.fixture() diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters.py index d2961c79..112af2f7 100644 --- a/test/icat/endpoints/test_count_with_filters.py +++ b/test/icat/endpoints/test_count_with_filters.py @@ -3,8 +3,10 @@ class TestCountWithFilters: @pytest.mark.usefixtures("single_investigation_test_data") - def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header): - test_response = flask_test_app.get( + def test_valid_count_with_filters( + self, flask_test_app_icat, valid_credentials_header + ): + test_response = flask_test_app_icat.get( '/investigations/count?where={"title": {"like": "Test data for the Python' ' ICAT Backend on DataGateway API"}}', headers=valid_credentials_header, @@ -13,9 +15,9 @@ def test_valid_count_with_filters(self, flask_test_app, valid_credentials_header assert test_response.json == 1 def test_valid_no_results_count_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations/count?where={"title": {"like": "This filter should cause a' '404 for testing purposes..."}}', headers=valid_credentials_header, diff --git a/test/icat/endpoints/test_create.py b/test/icat/endpoints/test_create.py index 04f8c39e..ed253eea 100644 --- a/test/icat/endpoints/test_create.py +++ b/test/icat/endpoints/test_create.py @@ -2,7 +2,7 @@ class TestCreateData: - def test_valid_create_data(self, flask_test_app, valid_credentials_header): + def test_valid_create_data(self, flask_test_app_icat, valid_credentials_header): create_investigations_json = [ { "name": "Test Data for API Testing, Data Creation 1", @@ -30,7 +30,7 @@ def test_valid_create_data(self, flask_test_app, valid_credentials_header): }, ] - test_response = flask_test_app.post( + test_response = flask_test_app_icat.post( "/investigations", headers=valid_credentials_header, json=create_investigations_json, @@ -52,11 +52,13 @@ def test_valid_create_data(self, flask_test_app, valid_credentials_header): # Delete the entities created by this test for investigation_id in test_data_ids: - flask_test_app.delete( + flask_test_app_icat.delete( f"/investigations/{investigation_id}", headers=valid_credentials_header, ) - def test_valid_boundary_create_data(self, flask_test_app, valid_credentials_header): + def test_valid_boundary_create_data( + self, flask_test_app_icat, valid_credentials_header + ): """Create a single investigation, as opposed to multiple""" create_investigation_json = { @@ -72,7 +74,7 @@ def test_valid_boundary_create_data(self, flask_test_app, valid_credentials_head "type": 1, } - test_response = flask_test_app.post( + test_response = flask_test_app_icat.post( "/investigations", headers=valid_credentials_header, json=create_investigation_json, @@ -88,18 +90,18 @@ def test_valid_boundary_create_data(self, flask_test_app, valid_credentials_head assert [create_investigation_json] == response_json - flask_test_app.delete( + flask_test_app_icat.delete( f"/investigations/{created_test_data_id}", headers=valid_credentials_header, ) - def test_invalid_create_data(self, flask_test_app, valid_credentials_header): + def test_invalid_create_data(self, flask_test_app_icat, valid_credentials_header): """An investigation requires a minimum of: name, visitId, facility, type""" invalid_request_body = { "title": "Test Title for DataGateway API Backend testing", } - test_response = flask_test_app.post( + test_response = flask_test_app_icat.post( "/investigations", headers=valid_credentials_header, json=invalid_request_body, @@ -108,7 +110,10 @@ def test_invalid_create_data(self, flask_test_app, valid_credentials_header): assert test_response.status_code == 400 def test_invalid_existing_data_create( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): """This test targets raising ICATObjectExistsError, causing a 400""" @@ -122,7 +127,7 @@ def test_invalid_existing_data_create( "type": 1, } - test_response = flask_test_app.post( + test_response = flask_test_app_icat.post( "/investigations", headers=valid_credentials_header, json=existing_object_json, diff --git a/test/icat/endpoints/test_delete_by_id.py b/test/icat/endpoints/test_delete_by_id.py index 07d8ff0d..e0124001 100644 --- a/test/icat/endpoints/test_delete_by_id.py +++ b/test/icat/endpoints/test_delete_by_id.py @@ -1,24 +1,29 @@ class TestDeleteByID: def test_valid_delete_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): - test_response = flask_test_app.delete( + test_response = flask_test_app_icat.delete( f'/investigations/{single_investigation_test_data[0]["id"]}', headers=valid_credentials_header, ) assert test_response.status_code == 204 - def test_invalid_delete_with_id(self, flask_test_app, valid_credentials_header): + def test_invalid_delete_with_id( + self, flask_test_app_icat, valid_credentials_header + ): """Request with a non-existent ID""" - final_investigation_result = flask_test_app.get( + final_investigation_result = flask_test_app_icat.get( '/investigations/findone?order="id DESC"', headers=valid_credentials_header, ) test_data_id = final_investigation_result.json["id"] # Adding 100 onto the ID to the most recent result should ensure a 404 - test_response = flask_test_app.delete( + test_response = flask_test_app_icat.delete( f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, ) diff --git a/test/icat/endpoints/test_findone.py b/test/icat/endpoints/test_findone.py index a0051e32..96ee5374 100644 --- a/test/icat/endpoints/test_findone.py +++ b/test/icat/endpoints/test_findone.py @@ -3,9 +3,12 @@ class TestFindone: def test_valid_findone_with_filters( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations/findone?where={"title": {"like": "Test data for the Python' ' ICAT Backend on DataGateway API"}}', headers=valid_credentials_header, @@ -15,9 +18,9 @@ def test_valid_findone_with_filters( assert response_json == single_investigation_test_data def test_valid_no_results_findone_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations/findone?where={"title": {"eq": "This filter should cause a' '404 for testing purposes..."}}', headers=valid_credentials_header, diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id.py index dfcbaae8..1271c9b5 100644 --- a/test/icat/endpoints/test_get_by_id.py +++ b/test/icat/endpoints/test_get_by_id.py @@ -3,17 +3,20 @@ class TestGetByID: def test_valid_get_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): # Need to identify the ID given to the test data - investigation_data = flask_test_app.get( + investigation_data = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}', headers=valid_credentials_header, ) test_data_id = investigation_data.json[0]["id"] - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/investigations/{test_data_id}", headers=valid_credentials_header, ) # Get with ID gives a dictionary response (only ever one result from that kind @@ -22,16 +25,16 @@ def test_valid_get_with_id( assert response_json == single_investigation_test_data - def test_invalid_get_with_id(self, flask_test_app, valid_credentials_header): + def test_invalid_get_with_id(self, flask_test_app_icat, valid_credentials_header): """Request with a non-existent ID""" - final_investigation_result = flask_test_app.get( + final_investigation_result = flask_test_app_icat.get( '/investigations/findone?order="id DESC"', headers=valid_credentials_header, ) test_data_id = final_investigation_result.json["id"] # Adding 100 onto the ID to the most recent result should ensure a 404 - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, ) diff --git a/test/icat/endpoints/test_get_with_filters.py b/test/icat/endpoints/test_get_with_filters.py index f72ab3c9..59ebeab8 100644 --- a/test/icat/endpoints/test_get_with_filters.py +++ b/test/icat/endpoints/test_get_with_filters.py @@ -5,9 +5,12 @@ class TestGetWithFilters: def test_valid_get_with_filters( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}', headers=valid_credentials_header, @@ -17,9 +20,9 @@ def test_valid_get_with_filters( assert response_json == single_investigation_test_data def test_valid_no_results_get_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations?where={"title": {"eq": "This filter should cause a 404 for' 'testing purposes..."}}', headers=valid_credentials_header, @@ -29,9 +32,9 @@ def test_valid_no_results_get_with_filters( @pytest.mark.usefixtures("multiple_investigation_test_data") def test_valid_get_with_filters_distinct( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}&distinct="title"', headers=valid_credentials_header, @@ -49,14 +52,14 @@ def test_valid_get_with_filters_distinct( def test_limit_skip_merge_get_with_filters( self, - flask_test_app, + flask_test_app_icat, valid_credentials_header, multiple_investigation_test_data, ): skip_value = 1 limit_value = 2 - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}' f'&skip={skip_value}&limit={limit_value}&order="id ASC"', diff --git a/test/icat/endpoints/test_table_endpoints.py b/test/icat/endpoints/test_table_endpoints.py index 6415d279..2d07b20e 100644 --- a/test/icat/endpoints/test_table_endpoints.py +++ b/test/icat/endpoints/test_table_endpoints.py @@ -8,9 +8,12 @@ class TestTableEndpoints: """ def test_valid_get_facility_cycles_with_filters( - self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + self, + flask_test_app_icat, + valid_credentials_header, + isis_specific_endpoint_data, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles", headers=valid_credentials_header, ) @@ -20,14 +23,14 @@ def test_valid_get_facility_cycles_with_filters( assert response_json == isis_specific_endpoint_data[1] def test_invalid_get_facility_cycles_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - final_instrument_result = flask_test_app.get( + final_instrument_result = flask_test_app_icat.get( '/instruments/findone?order="id DESC"', headers=valid_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles", headers=valid_credentials_header, ) @@ -35,9 +38,12 @@ def test_invalid_get_facility_cycles_with_filters( assert test_response.status_code == 404 def test_valid_get_facility_cycles_count_with_filters( - self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + self, + flask_test_app_icat, + valid_credentials_header, + isis_specific_endpoint_data, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/count", headers=valid_credentials_header, ) @@ -45,14 +51,14 @@ def test_valid_get_facility_cycles_count_with_filters( assert test_response.json == 1 def test_invalid_get_facility_cycles_count_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - final_instrument_result = flask_test_app.get( + final_instrument_result = flask_test_app_icat.get( '/instruments/findone?order="id DESC"', headers=valid_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/count", headers=valid_credentials_header, ) @@ -60,9 +66,12 @@ def test_invalid_get_facility_cycles_count_with_filters( assert test_response.json == 0 def test_valid_get_investigations_with_filters( - self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + self, + flask_test_app_icat, + valid_credentials_header, + isis_specific_endpoint_data, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/" f"{isis_specific_endpoint_data[2]}/investigations", headers=valid_credentials_header, @@ -73,18 +82,18 @@ def test_valid_get_investigations_with_filters( assert response_json == isis_specific_endpoint_data[3] def test_invalid_get_investigations_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - final_instrument_result = flask_test_app.get( + final_instrument_result = flask_test_app_icat.get( '/instruments/findone?order="id DESC"', headers=valid_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] - final_facilitycycle_result = flask_test_app.get( + final_facilitycycle_result = flask_test_app_icat.get( '/facilitycycles/findone?order="id DESC"', headers=valid_credentials_header, ) final_facilitycycle_id = final_facilitycycle_result.json["id"] - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations", headers=valid_credentials_header, @@ -93,9 +102,12 @@ def test_invalid_get_investigations_with_filters( assert test_response.status_code == 404 def test_valid_get_investigations_count_with_filters( - self, flask_test_app, valid_credentials_header, isis_specific_endpoint_data, + self, + flask_test_app_icat, + valid_credentials_header, + isis_specific_endpoint_data, ): - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/" f"{isis_specific_endpoint_data[2]}/investigations/count", headers=valid_credentials_header, @@ -104,18 +116,18 @@ def test_valid_get_investigations_count_with_filters( assert test_response.json == 1 def test_invalid_get_investigations_count_with_filters( - self, flask_test_app, valid_credentials_header, + self, flask_test_app_icat, valid_credentials_header, ): - final_instrument_result = flask_test_app.get( + final_instrument_result = flask_test_app_icat.get( '/instruments/findone?order="id DESC"', headers=valid_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] - final_facilitycycle_result = flask_test_app.get( + final_facilitycycle_result = flask_test_app_icat.get( '/facilitycycles/findone?order="id DESC"', headers=valid_credentials_header, ) final_facilitycycle_id = final_facilitycycle_result.json["id"] - test_response = flask_test_app.get( + test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations/count", headers=valid_credentials_header, diff --git a/test/icat/endpoints/test_update_by_id.py b/test/icat/endpoints/test_update_by_id.py index aad64e37..127ccdd3 100644 --- a/test/icat/endpoints/test_update_by_id.py +++ b/test/icat/endpoints/test_update_by_id.py @@ -3,7 +3,10 @@ class TestUpdateByID: def test_valid_update_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): expected_doi = "Test Data Identifier" expected_summary = "Test Summary" @@ -15,7 +18,7 @@ def test_valid_update_with_id( single_investigation_test_data[0]["doi"] = expected_doi single_investigation_test_data[0]["summary"] = expected_summary - test_response = flask_test_app.patch( + test_response = flask_test_app_icat.patch( f"/investigations/{single_investigation_test_data[0]['id']}", headers=valid_credentials_header, json=update_data_json, @@ -25,7 +28,10 @@ def test_valid_update_with_id( assert response_json == single_investigation_test_data def test_invalid_update_with_id( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): """This test will attempt to put `icatdb` into an invalid state""" @@ -37,7 +43,7 @@ def test_invalid_update_with_id( "_________________________________________________________________________", } - test_response = flask_test_app.patch( + test_response = flask_test_app_icat.patch( f"/investigations/{single_investigation_test_data[0]['id']}", headers=valid_credentials_header, json=invalid_update_json, diff --git a/test/icat/endpoints/test_update_multiple.py b/test/icat/endpoints/test_update_multiple.py index 3b25c79b..b4fadc26 100644 --- a/test/icat/endpoints/test_update_multiple.py +++ b/test/icat/endpoints/test_update_multiple.py @@ -6,7 +6,7 @@ class TestUpdateMultipleEntities: def test_valid_multiple_update_data( self, - flask_test_app, + flask_test_app_icat, valid_credentials_header, multiple_investigation_test_data, ): @@ -26,7 +26,7 @@ def test_valid_multiple_update_data( } update_data_list.append(update_entity) - test_response = flask_test_app.patch( + test_response = flask_test_app_icat.patch( "/investigations", headers=valid_credentials_header, json=update_data_list, ) response_json = prepare_icat_data_for_assertion(test_response.json) @@ -34,7 +34,10 @@ def test_valid_multiple_update_data( assert response_json == multiple_investigation_test_data def test_valid_boundary_update_data( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): """ Request body is a dictionary, not a list of dictionaries""" @@ -49,7 +52,7 @@ def test_valid_boundary_update_data( single_investigation_test_data[0]["doi"] = expected_doi single_investigation_test_data[0]["summary"] = expected_summary - test_response = flask_test_app.patch( + test_response = flask_test_app_icat.patch( "/investigations", headers=valid_credentials_header, json=update_data_json, ) response_json = prepare_icat_data_for_assertion(test_response.json) @@ -57,7 +60,10 @@ def test_valid_boundary_update_data( assert response_json == single_investigation_test_data def test_invalid_missing_update_data( - self, flask_test_app, valid_credentials_header, single_investigation_test_data, + self, + flask_test_app_icat, + valid_credentials_header, + single_investigation_test_data, ): """There should be an ID in the request body to know which entity to update""" @@ -66,7 +72,7 @@ def test_invalid_missing_update_data( "summary": "Test Summary", } - test_response = flask_test_app.patch( + test_response = flask_test_app_icat.patch( "/investigations", headers=valid_credentials_header, json=update_data_json, ) @@ -81,7 +87,7 @@ def test_invalid_missing_update_data( ) def test_invalid_attribute_update( self, - flask_test_app, + flask_test_app_icat, valid_credentials_header, single_investigation_test_data, update_key, @@ -92,7 +98,7 @@ def test_invalid_attribute_update( update_key: update_value, } - test_response = flask_test_app.patch( + test_response = flask_test_app_icat.patch( "/investigations", headers=valid_credentials_header, json=invalid_update_data_json, diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 3aa47d6f..6e329321 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -11,8 +11,10 @@ class TestSessionHandling: def test_session_id_decorator(self): pass - def test_get_valid_session_details(self, flask_test_app, valid_credentials_header): - session_details = flask_test_app.get( + def test_get_valid_session_details( + self, flask_test_app_icat, valid_credentials_header + ): + session_details = flask_test_app_icat.get( "/sessions", headers=valid_credentials_header, ) @@ -40,25 +42,25 @@ def test_get_valid_session_details(self, flask_test_app, valid_credentials_heade assert time_diff_minutes < 120 and time_diff_minutes >= 118 def test_get_invalid_session_details( - self, invalid_credentials_header, flask_test_app, + self, invalid_credentials_header, flask_test_app_icat, ): - session_details = flask_test_app.get( + session_details = flask_test_app_icat.get( "/sessions", headers=invalid_credentials_header, ) assert session_details.status_code == 403 - def test_refresh_session(self, valid_credentials_header, flask_test_app): - pre_refresh_session_details = flask_test_app.get( + def test_refresh_session(self, valid_credentials_header, flask_test_app_icat): + pre_refresh_session_details = flask_test_app_icat.get( "/sessions", headers=valid_credentials_header, ) - refresh_session = flask_test_app.put( + refresh_session = flask_test_app_icat.put( "/sessions", headers=valid_credentials_header, ) assert refresh_session.status_code == 200 - post_refresh_session_details = flask_test_app.get( + post_refresh_session_details = flask_test_app_icat.get( "/sessions", headers=valid_credentials_header, ) @@ -68,7 +70,7 @@ def test_refresh_session(self, valid_credentials_header, flask_test_app): ) @pytest.mark.usefixtures("single_investigation_test_data") - def test_valid_login(self, flask_test_app, icat_client, icat_query): + def test_valid_login(self, flask_test_app_icat, icat_client, icat_query): user_credentials = config.get_test_user_credentials() login_json = { @@ -76,7 +78,7 @@ def test_valid_login(self, flask_test_app, icat_client, icat_query): "password": user_credentials["password"], "mechanism": config.get_test_mechanism(), } - login_response = flask_test_app.post("/sessions", json=login_json) + login_response = flask_test_app_icat.post("/sessions", json=login_json) icat_client.sessionId = login_response.json["sessionID"] icat_query.setAggregate("COUNT") @@ -89,27 +91,27 @@ def test_valid_login(self, flask_test_app, icat_client, icat_query): assert test_query == [1] and login_response.status_code == 201 - def test_invalid_login(self, flask_test_app): + def test_invalid_login(self, flask_test_app_icat): login_json = { "username": "Invalid Username", "password": "InvalidPassword", "mechanism": config.get_test_mechanism(), } - login_response = flask_test_app.post("/sessions", json=login_json) + login_response = flask_test_app_icat.post("/sessions", json=login_json) assert login_response.status_code == 403 - def test_valid_logout(self, flask_test_app): + def test_valid_logout(self, flask_test_app_icat): client = Client(config.get_icat_url(), checkCert=config.get_icat_check_cert()) client.login(config.get_test_mechanism(), config.get_test_user_credentials()) creds_header = {"Authorization": f"Bearer {client.sessionId}"} - logout_response = flask_test_app.delete("/sessions", headers=creds_header) + logout_response = flask_test_app_icat.delete("/sessions", headers=creds_header) assert logout_response.status_code == 200 - def test_invalid_logout(self, invalid_credentials_header, flask_test_app): - logout_response = flask_test_app.delete( + def test_invalid_logout(self, invalid_credentials_header, flask_test_app_icat): + logout_response = flask_test_app_icat.delete( "/sessions", headers=invalid_credentials_header, ) From db97d8a2934de943457b373cb56fb6e97f1bbb81 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 11:06:28 +0000 Subject: [PATCH 068/109] #150: Fix failing endpoint rule tests - The tests were looking at the flask app from src.main which is only setup if running the file as per README.md. So a generic flask app fixture has been created and that's used to check endpoint rules --- test/conftest.py | 16 +++++++++++++++- test/icat/endpoints/test_endpoint_rules.py | 11 ++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 10d822ab..191770b7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -95,7 +95,20 @@ def multiple_investigation_test_data(icat_client): @pytest.fixture(scope="package") -def flask_test_app_icat(): +def flask_test_app(): + """ + TODO - Explain why a generic test app is needed that doesn't rely on any backend + """ + test_app = Flask(__name__) + api, spec = create_app_infrastructure(test_app) + create_api_endpoints(test_app, api, spec) + + yield test_app + + +@pytest.fixture(scope="package") +def flask_test_app_icat(flask_test_app): + """TODO - Explain ICAT test client""" icat_app = Flask(__name__) icat_app.config["TESTING"] = True icat_app.config["TEST_BACKEND"] = "python_icat" @@ -108,6 +121,7 @@ def flask_test_app_icat(): @pytest.fixture(scope="package") def flask_test_app_db(): + """TODO - Add DB test client doc""" db_app = Flask(__name__) db_app.config["TESTING"] = True db_app.config["TEST_BACKEND"] = "db" diff --git a/test/icat/endpoints/test_endpoint_rules.py b/test/icat/endpoints/test_endpoint_rules.py index 5259fec7..f6e3c716 100644 --- a/test/icat/endpoints/test_endpoint_rules.py +++ b/test/icat/endpoints/test_endpoint_rules.py @@ -1,6 +1,5 @@ import pytest -from datagateway_api.src.main import app from datagateway_api.src.resources.entities.entity_map import endpoints @@ -18,11 +17,11 @@ class TestEndpointRules: pytest.param("", ["GET", "PATCH", "POST"], id="typical endpoints"), ], ) - def test_entity_endpoints(self, endpoint_ending, expected_methods): + def test_entity_endpoints(self, flask_test_app, endpoint_ending, expected_methods): for endpoint_entity in endpoints.keys(): endpoint_found = False - for rule in app.url_map.iter_rules(): + for rule in flask_test_app.url_map.iter_rules(): if f"/{endpoint_entity.lower()}{endpoint_ending}" == rule.rule: endpoint_found = True @@ -61,10 +60,12 @@ def test_entity_endpoints(self, endpoint_ending, expected_methods): ), ], ) - def test_non_entity_endpoints(self, endpoint_name, expected_methods): + def test_non_entity_endpoints( + self, flask_test_app, endpoint_name, expected_methods + ): endpoint_found = False - for rule in app.url_map.iter_rules(): + for rule in flask_test_app.url_map.iter_rules(): if endpoint_name == rule.rule: endpoint_found = True From adc48d181371de4a1ac84302296a9e86112b44b4 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 11:33:37 +0000 Subject: [PATCH 069/109] #150: Fix imports on database helpers --- datagateway_api/common/database/helpers.py | 32 +++------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index 68df4267..0a8df338 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -5,7 +5,10 @@ from sqlalchemy.orm import aliased -from datagateway_api.common.config import config +from datagateway_api.common.database.filters import ( + DatabaseIncludeFilter as IncludeFilter, + DatabaseWhereFilter as WhereFilter, +) from datagateway_api.common.database.models import ( FACILITY, FACILITYCYCLE, @@ -16,40 +19,13 @@ ) from datagateway_api.common.database.session_manager import session_manager from datagateway_api.common.exceptions import ( - ApiError, AuthenticationError, BadRequestError, - FilterError, MissingRecordError, ) from datagateway_api.common.filter_order_handler import FilterOrderHandler -backend_type = config.get_backend_type() -if backend_type == "db": - from datagateway_api.common.database.filters import ( - DatabaseDistinctFieldFilter as DistinctFieldFilter, - DatabaseIncludeFilter as IncludeFilter, - DatabaseLimitFilter as LimitFilter, - DatabaseOrderFilter as OrderFilter, - DatabaseSkipFilter as SkipFilter, - DatabaseWhereFilter as WhereFilter, - ) -elif backend_type == "python_icat": - from datagateway_api.common.icat.filters import ( - PythonICATDistinctFieldFilter as DistinctFieldFilter, - PythonICATIncludeFilter as IncludeFilter, - PythonICATLimitFilter as LimitFilter, - PythonICATOrderFilter as OrderFilter, - PythonICATSkipFilter as SkipFilter, - PythonICATWhereFilter as WhereFilter, - ) -else: - raise ApiError( - "Cannot select which implementation of filters to import, check the config file" - " has a valid backend type", - ) - log = logging.getLogger() From dd9a92ae3f6e8be1fd1eddaae5e605298aab56f5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 11:34:08 +0000 Subject: [PATCH 070/109] #150: Correct import for QueryFilterFactory --- test/test_database_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_database_helpers.py b/test/test_database_helpers.py index c97404a5..b1ee2ae2 100644 --- a/test/test_database_helpers.py +++ b/test/test_database_helpers.py @@ -1,8 +1,8 @@ from unittest import TestCase from datagateway_api.common.config import config -from datagateway_api.common.database.helpers import QueryFilterFactory from datagateway_api.common.exceptions import ApiError +from datagateway_api.common.query_filter import QueryFilterFactory backend_type = config.get_backend_type() if backend_type == "db": From c676f46c8c9a6df9d03d2892bfaba8ad780524ff Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 11:43:35 +0000 Subject: [PATCH 071/109] #150: Fix imports on test helpers --- test/test_helpers.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/test/test_helpers.py b/test/test_helpers.py index ca39f2de..5501a39f 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -3,15 +3,17 @@ from sqlalchemy.exc import IntegrityError +from datagateway_api.common.database.filters import ( + DatabaseDistinctFieldFilter, + DatabaseIncludeFilter, + DatabaseLimitFilter, + DatabaseOrderFilter, + DatabaseSkipFilter, + DatabaseWhereFilter, +) from datagateway_api.common.database.helpers import ( delete_row_by_id, - DistinctFieldFilter, - IncludeFilter, insert_row_into_table, - LimitFilter, - OrderFilter, - SkipFilter, - WhereFilter, ) from datagateway_api.common.database.models import SESSION from datagateway_api.common.exceptions import ( @@ -200,7 +202,9 @@ def test_limit_filter(self): self.assertEqual( 1, len(filters), msg="Returned incorrect number of filters", ) - self.assertIs(LimitFilter, type(filters[0]), msg="Incorrect type of filter") + self.assertIs( + DatabaseLimitFilter, type(filters[0]), msg="Incorrect type of filter" + ) def test_order_filter(self): with self.app: @@ -210,7 +214,9 @@ def test_order_filter(self): 1, len(filters), msg="Returned incorrect number of filters", ) self.assertIs( - OrderFilter, type(filters[0]), msg="Incorrect type of filter returned", + DatabaseOrderFilter, + type(filters[0]), + msg="Incorrect type of filter returned", ) def test_where_filter(self): @@ -221,7 +227,9 @@ def test_where_filter(self): 1, len(filters), msg="Returned incorrect number of filters", ) self.assertIs( - WhereFilter, type(filters[0]), msg="Incorrect type of filter returned", + DatabaseWhereFilter, + type(filters[0]), + msg="Incorrect type of filter returned", ) def test_skip_filter(self): @@ -232,7 +240,9 @@ def test_skip_filter(self): 1, len(filters), msg="Returned incorrect number of filters", ) self.assertIs( - SkipFilter, type(filters[0]), msg="Incorrect type of filter returned", + DatabaseSkipFilter, + type(filters[0]), + msg="Incorrect type of filter returned", ) def test_include_filter(self): @@ -243,7 +253,7 @@ def test_include_filter(self): 1, len(filters), msg="Incorrect number of filters returned", ) self.assertIs( - IncludeFilter, + DatabaseIncludeFilter, type(filters[0]), msg="Incorrect type of filter returned", ) @@ -256,7 +266,7 @@ def test_distinct_filter(self): 1, len(filters), msg="Incorrect number of filters returned", ) self.assertIs( - DistinctFieldFilter, + DatabaseDistinctFieldFilter, type(filters[0]), msg="Incorrect type of filter returned", ) From c29f68e9452a9f5835c2ebe0ab0c54bdc762e393 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 12:58:55 +0000 Subject: [PATCH 072/109] #150: Fix logging issues when dealing with session IDs which are strings, not numbers - When dealing with session IDs, you cannot assume they are numbers, since they're in the format of UUIDs, so %s is more suitable --- datagateway_api/common/database/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index 0a8df338..8a208717 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -225,7 +225,7 @@ def get_row_by_id(table, id_): :return: the record retrieved """ with ReadQuery(table) as read_query: - log.info(" Querying %s for record with ID: %d", table.__tablename__, id_) + log.info(" Querying %s for record with ID: %s", table.__tablename__, id_) where_filter = WhereFilter("ID", id_, "eq") where_filter.apply_filter(read_query) return read_query.get_single_result() @@ -239,7 +239,7 @@ def delete_row_by_id(table, id_): :param table: the table to be searched :param id_: the id of the record to delete """ - log.info(" Deleting row from %s with ID: %d", table.__tablename__, id_) + log.info(" Deleting row from %s with ID: %s", table.__tablename__, id_) row = get_row_by_id(table, id_) with DeleteQuery(table, row) as delete_query: delete_query.execute_query() From dce47c1d2117196b1183699cb931857ac2ddaf85 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 13:02:48 +0000 Subject: [PATCH 073/109] #150: Remove unneeded import --- test/conftest.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 191770b7..7ffb87a8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,11 +8,7 @@ import pytest from datagateway_api.common.config import config -from datagateway_api.src.main import ( - app, - create_api_endpoints, - create_app_infrastructure, -) +from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure from test.icat.test_query import prepare_icat_data_for_assertion From bcea7ec46e0c1819d8a7ae1d55e2f1d0b78fc5d7 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 13:03:20 +0000 Subject: [PATCH 074/109] #150: Fix FlaskAppTest based on changes to main.py - This is a temporary fix before rewriting the original tests in pytest --- test/test_base.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/test_base.py b/test/test_base.py index 31677e1d..5236c379 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -1,6 +1,8 @@ from unittest import TestCase -from datagateway_api.src.main import app +from flask import Flask + +from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure class FlaskAppTest(TestCase): @@ -9,5 +11,11 @@ class FlaskAppTest(TestCase): """ def setUp(self): + app = Flask(__name__) + app.config["TESTING"] = True app.config["TESTING"] = True + app.config["TEST_BACKEND"] = "db" + + api, spec = create_app_infrastructure(app) + create_api_endpoints(app, api, spec) self.app = app.test_client() From c866be37400bdfe27aab65984431d70258c4f1d8 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 13:32:10 +0000 Subject: [PATCH 075/109] #150: General cleanup of recent work - Sort out linting and remove a few lines of code used for logging etc. --- datagateway_api/common/backend.py | 4 ++-- datagateway_api/common/database/backend.py | 4 ++-- datagateway_api/common/helpers.py | 2 +- datagateway_api/common/icat/backend.py | 4 ++-- datagateway_api/common/query_filter.py | 1 - datagateway_api/src/main.py | 21 ++++++++----------- .../src/resources/entities/entity_endpoint.py | 4 ---- .../non_entities/sessions_endpoints.py | 5 ----- .../table_endpoints/table_endpoints.py | 8 ++----- .../icat/endpoints/test_count_with_filters.py | 2 +- test/icat/endpoints/test_create.py | 2 +- test/icat/endpoints/test_delete_by_id.py | 2 +- test/icat/endpoints/test_endpoint_rules.py | 2 +- test/icat/test_query.py | 1 - test/icat/test_session_handling.py | 4 +--- test/test_helpers.py | 2 +- 16 files changed, 24 insertions(+), 44 deletions(-) diff --git a/datagateway_api/common/backend.py b/datagateway_api/common/backend.py index c09d06a6..a6ca914a 100644 --- a/datagateway_api/common/backend.py +++ b/datagateway_api/common/backend.py @@ -172,7 +172,7 @@ def get_facility_cycles_for_instrument_count_with_filters( pass @abstractmethod - def get_investigations_for_instrument_in_facility_cycle_with_filters( + def get_investigations_for_instrument_facility_cycle_with_filters( self, session_id, instrument_id, facilitycycle_id, filters, ): """ @@ -188,7 +188,7 @@ def get_investigations_for_instrument_in_facility_cycle_with_filters( pass @abstractmethod - def get_investigation_count_for_instrument_facility_cycle_with_filters( + def get_investigation_count_instrument_facility_cycle_with_filters( self, session_id, instrument_id, facilitycycle_id, filters, ): """ diff --git a/datagateway_api/common/database/backend.py b/datagateway_api/common/database/backend.py index 65d7eb6c..8ee01d87 100644 --- a/datagateway_api/common/database/backend.py +++ b/datagateway_api/common/database/backend.py @@ -124,7 +124,7 @@ def get_facility_cycles_for_instrument_count_with_filters( @requires_session_id @queries_records - def get_investigations_for_instrument_in_facility_cycle_with_filters( + def get_investigations_for_instrument_facility_cycle_with_filters( self, session_id, instrument_id, facilitycycle_id, filters, ): return get_investigations_for_instrument_in_facility_cycle( @@ -133,7 +133,7 @@ def get_investigations_for_instrument_in_facility_cycle_with_filters( @requires_session_id @queries_records - def get_investigation_count_for_instrument_facility_cycle_with_filters( + def get_investigation_count_instrument_facility_cycle_with_filters( self, session_id, instrument_id, facilitycycle_id, filters, ): return get_investigations_for_instrument_in_facility_cycle_count( diff --git a/datagateway_api/common/helpers.py b/datagateway_api/common/helpers.py index 55dec407..0a86dd33 100644 --- a/datagateway_api/common/helpers.py +++ b/datagateway_api/common/helpers.py @@ -6,7 +6,6 @@ from flask_restful import reqparse from sqlalchemy.exc import IntegrityError -from datagateway_api.common.query_filter import QueryFilterFactory from datagateway_api.common.exceptions import ( ApiError, AuthenticationError, @@ -14,6 +13,7 @@ FilterError, MissingCredentialsError, ) +from datagateway_api.common.query_filter import QueryFilterFactory log = logging.getLogger() diff --git a/datagateway_api/common/icat/backend.py b/datagateway_api/common/icat/backend.py index d4167546..af206560 100644 --- a/datagateway_api/common/icat/backend.py +++ b/datagateway_api/common/icat/backend.py @@ -136,7 +136,7 @@ def get_facility_cycles_for_instrument_count_with_filters( @requires_session_id @queries_records - def get_investigations_for_instrument_in_facility_cycle_with_filters( + def get_investigations_for_instrument_facility_cycle_with_filters( self, session_id, instrument_id, facilitycycle_id, filters, **kwargs, ): client = kwargs["client"] if kwargs["client"] else create_client() @@ -146,7 +146,7 @@ def get_investigations_for_instrument_in_facility_cycle_with_filters( @requires_session_id @queries_records - def get_investigation_count_for_instrument_facility_cycle_with_filters( + def get_investigation_count_instrument_facility_cycle_with_filters( self, session_id, instrument_id, facilitycycle_id, filters, **kwargs, ): client = kwargs["client"] if kwargs["client"] else create_client() diff --git a/datagateway_api/common/query_filter.py b/datagateway_api/common/query_filter.py index f81067b3..e0928ff2 100644 --- a/datagateway_api/common/query_filter.py +++ b/datagateway_api/common/query_filter.py @@ -25,7 +25,6 @@ def get_query_filter(request_filter): """ backend_type = config.get_backend_type() - print(f"Backend inside staticmethod: {backend_type}") if backend_type == "db": from datagateway_api.common.database.filters import ( DatabaseDistinctFieldFilter as DistinctFieldFilter, diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 8162e344..3b06614b 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -24,9 +24,9 @@ ) from datagateway_api.src.resources.table_endpoints.table_endpoints import ( count_instrument_facility_cycles_endpoint, + count_instrument_investigation_endpoint, instrument_facility_cycles_endpoint, instrument_investigation_endpoint, - count_instrument_investigation_endpoint, ) from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin from datagateway_api.src.swagger.initialise_spec import initialise_spec @@ -51,7 +51,7 @@ def create_app_infrastructure(flask_app): security=[{"session_id": []}], ) - cors = CORS(flask_app) + CORS(flask_app) flask_app.url_map.strict_slashes = False api = Api(flask_app) @@ -70,24 +70,21 @@ def create_api_endpoints(flask_app, api, spec): try: backend_type = flask_app.config["TEST_BACKEND"] config.set_backend_type(backend_type) - print(f"test backend: {backend_type}") except KeyError: backend_type = config.get_backend_type() - print(f"config backend: {backend_type}") # TODO - Add :param backend: to the endpoint functions backend = create_backend(backend_type) - print(f"Backend: {backend}, Type: {type(backend)}") for entity_name in endpoints: get_endpoint_resource = get_endpoint( - entity_name, endpoints[entity_name], backend + entity_name, endpoints[entity_name], backend, ) api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") spec.path(resource=get_endpoint_resource, api=api) get_id_endpoint_resource = get_id_endpoint( - entity_name, endpoints[entity_name], backend + entity_name, endpoints[entity_name], backend, ) api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") spec.path(resource=get_id_endpoint_resource, api=api) @@ -114,15 +111,15 @@ def create_api_endpoints(flask_app, api, spec): # Table specific endpoints instrument_facility_cycle_resource = instrument_facility_cycles_endpoint(backend) api.add_resource( - instrument_facility_cycle_resource, "/instruments//facilitycycles" + instrument_facility_cycle_resource, "/instruments//facilitycycles", ) # spec.path(resource=instrument_facility_cycle_resource, api=api) - count_instrument_facility_cycle_resource = count_instrument_facility_cycles_endpoint( - backend + count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( + backend, ) api.add_resource( - count_instrument_facility_cycle_resource, + count_instrument_facility_cycle_res, "/instruments//facilitycycles/count", ) # spec.path(resource=count_instrument_facility_cycle_resource, api=api) @@ -135,7 +132,7 @@ def create_api_endpoints(flask_app, api, spec): # spec.path(resource=instrument_investigation_resource, api=api) count_instrument_investigation_resource = count_instrument_investigation_endpoint( - backend + backend, ) api.add_resource( count_instrument_investigation_resource, diff --git a/datagateway_api/src/resources/entities/entity_endpoint.py b/datagateway_api/src/resources/entities/entity_endpoint.py index 3446913d..da26dbba 100644 --- a/datagateway_api/src/resources/entities/entity_endpoint.py +++ b/datagateway_api/src/resources/entities/entity_endpoint.py @@ -1,15 +1,11 @@ from flask import request from flask_restful import Resource -from datagateway_api.common.backends import create_backend -from datagateway_api.common.config import config from datagateway_api.common.helpers import ( get_filters_from_query_string, get_session_id_from_auth_header, ) -# backend = create_backend(config.get_backend_type()) - def get_endpoint(name, entity_type, backend): """ diff --git a/datagateway_api/src/resources/non_entities/sessions_endpoints.py b/datagateway_api/src/resources/non_entities/sessions_endpoints.py index 36a33922..febc969d 100644 --- a/datagateway_api/src/resources/non_entities/sessions_endpoints.py +++ b/datagateway_api/src/resources/non_entities/sessions_endpoints.py @@ -3,22 +3,17 @@ from flask import request from flask_restful import Resource -from datagateway_api.common.backends import create_backend -from datagateway_api.common.config import config from datagateway_api.common.exceptions import AuthenticationError from datagateway_api.common.helpers import get_session_id_from_auth_header log = logging.getLogger() -# backend = create_backend(config.get_backend_type()) - def session_endpoints(backend): """ TODO - Add docstring """ - log.info("test") class Sessions(Resource): def post(self): diff --git a/datagateway_api/src/resources/table_endpoints/table_endpoints.py b/datagateway_api/src/resources/table_endpoints/table_endpoints.py index 507d3295..6f2907a7 100644 --- a/datagateway_api/src/resources/table_endpoints/table_endpoints.py +++ b/datagateway_api/src/resources/table_endpoints/table_endpoints.py @@ -1,14 +1,10 @@ from flask_restful import Resource -from datagateway_api.common.backends import create_backend -from datagateway_api.common.config import config from datagateway_api.common.helpers import ( get_filters_from_query_string, get_session_id_from_auth_header, ) -backend = create_backend(config.get_backend_type()) - def instrument_facility_cycles_endpoint(backend): """ @@ -179,7 +175,7 @@ def get(self, instrument_id, cycle_id): description: No such record - Unable to find a record in ICAT """ return ( - backend.get_investigations_for_instrument_in_facility_cycle_with_filters( + backend.get_investigations_for_instrument_facility_cycle_with_filters( get_session_id_from_auth_header(), instrument_id, cycle_id, @@ -240,7 +236,7 @@ def get(self, instrument_id, cycle_id): description: No such record - Unable to find a record in ICAT """ return ( - backend.get_investigation_count_for_instrument_facility_cycle_with_filters( + backend.get_investigation_count_instrument_facility_cycle_with_filters( get_session_id_from_auth_header(), instrument_id, cycle_id, diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters.py index 112af2f7..9fb8fe0e 100644 --- a/test/icat/endpoints/test_count_with_filters.py +++ b/test/icat/endpoints/test_count_with_filters.py @@ -4,7 +4,7 @@ class TestCountWithFilters: @pytest.mark.usefixtures("single_investigation_test_data") def test_valid_count_with_filters( - self, flask_test_app_icat, valid_credentials_header + self, flask_test_app_icat, valid_credentials_header, ): test_response = flask_test_app_icat.get( '/investigations/count?where={"title": {"like": "Test data for the Python' diff --git a/test/icat/endpoints/test_create.py b/test/icat/endpoints/test_create.py index ed253eea..e7f5137a 100644 --- a/test/icat/endpoints/test_create.py +++ b/test/icat/endpoints/test_create.py @@ -57,7 +57,7 @@ def test_valid_create_data(self, flask_test_app_icat, valid_credentials_header): ) def test_valid_boundary_create_data( - self, flask_test_app_icat, valid_credentials_header + self, flask_test_app_icat, valid_credentials_header, ): """Create a single investigation, as opposed to multiple""" diff --git a/test/icat/endpoints/test_delete_by_id.py b/test/icat/endpoints/test_delete_by_id.py index e0124001..4c811919 100644 --- a/test/icat/endpoints/test_delete_by_id.py +++ b/test/icat/endpoints/test_delete_by_id.py @@ -13,7 +13,7 @@ def test_valid_delete_with_id( assert test_response.status_code == 204 def test_invalid_delete_with_id( - self, flask_test_app_icat, valid_credentials_header + self, flask_test_app_icat, valid_credentials_header, ): """Request with a non-existent ID""" diff --git a/test/icat/endpoints/test_endpoint_rules.py b/test/icat/endpoints/test_endpoint_rules.py index f6e3c716..f703e4f6 100644 --- a/test/icat/endpoints/test_endpoint_rules.py +++ b/test/icat/endpoints/test_endpoint_rules.py @@ -61,7 +61,7 @@ def test_entity_endpoints(self, flask_test_app, endpoint_ending, expected_method ], ) def test_non_entity_endpoints( - self, flask_test_app, endpoint_name, expected_methods + self, flask_test_app, endpoint_name, expected_methods, ): endpoint_found = False diff --git a/test/icat/test_query.py b/test/icat/test_query.py index 8e11722d..6ee3e53d 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -122,7 +122,6 @@ def test_include_fields_list_flatten(self, icat_client): test_query = ICATQuery(icat_client, "User") flat_list = test_query.flatten_query_included_fields(included_field_set) - print(flat_list) assert flat_list == [ "instrumentScientists", diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 6e329321..4828e6db 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -12,14 +12,12 @@ def test_session_id_decorator(self): pass def test_get_valid_session_details( - self, flask_test_app_icat, valid_credentials_header + self, flask_test_app_icat, valid_credentials_header, ): session_details = flask_test_app_icat.get( "/sessions", headers=valid_credentials_header, ) - print(f"JSON: {session_details.json}, Code: {session_details.status_code}") - # Check username is correct assert ( session_details.json["USERNAME"] == f"{config.get_test_mechanism()}/" diff --git a/test/test_helpers.py b/test/test_helpers.py index 5501a39f..b3972987 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -203,7 +203,7 @@ def test_limit_filter(self): 1, len(filters), msg="Returned incorrect number of filters", ) self.assertIs( - DatabaseLimitFilter, type(filters[0]), msg="Incorrect type of filter" + DatabaseLimitFilter, type(filters[0]), msg="Incorrect type of filter", ) def test_order_filter(self): From b8324db630ad77a20e094f9b40ef0ff90c6d49b6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 13:37:20 +0000 Subject: [PATCH 076/109] #150: Restructure test files - Before this commit, I kept the pytest tests I've created in this branch separate from the unittest tests that were there beforehand to keep them isolated. This restructure means that non-backend specific tests are together, and backend specific tests are put in the correctly named directories --- test/{ => db}/test_database_helpers.py | 0 test/{ => db}/test_entityHelper.py | 0 test/{ => db}/test_helpers.py | 0 test/{icat => }/test_backends.py | 0 test/{icat => }/test_config.py | 2 +- test/{icat => }/test_date_handler.py | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename test/{ => db}/test_database_helpers.py (100%) rename test/{ => db}/test_entityHelper.py (100%) rename test/{ => db}/test_helpers.py (100%) rename test/{icat => }/test_backends.py (100%) rename test/{icat => }/test_config.py (98%) rename test/{icat => }/test_date_handler.py (100%) diff --git a/test/test_database_helpers.py b/test/db/test_database_helpers.py similarity index 100% rename from test/test_database_helpers.py rename to test/db/test_database_helpers.py diff --git a/test/test_entityHelper.py b/test/db/test_entityHelper.py similarity index 100% rename from test/test_entityHelper.py rename to test/db/test_entityHelper.py diff --git a/test/test_helpers.py b/test/db/test_helpers.py similarity index 100% rename from test/test_helpers.py rename to test/db/test_helpers.py diff --git a/test/icat/test_backends.py b/test/test_backends.py similarity index 100% rename from test/icat/test_backends.py rename to test/test_backends.py diff --git a/test/icat/test_config.py b/test/test_config.py similarity index 98% rename from test/icat/test_config.py rename to test/test_config.py index dee771c4..647ad98f 100644 --- a/test/icat/test_config.py +++ b/test/test_config.py @@ -8,7 +8,7 @@ @pytest.fixture() def valid_config(): - return Config(path=Path(__file__).parent.parent.parent / "config.json.example") + return Config(path=Path(__file__).parent.parent / "config.json.example") @pytest.fixture() diff --git a/test/icat/test_date_handler.py b/test/test_date_handler.py similarity index 100% rename from test/icat/test_date_handler.py rename to test/test_date_handler.py From cff2ce477ec9fcc446ba10ad362966ae45cfbbca Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 14:12:57 +0000 Subject: [PATCH 077/109] #150: Rewrite DB entity helper tests in pytest --- test/db/test_entityHelper.py | 171 ----------------------------- test/db/test_entity_helper.py | 201 ++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 171 deletions(-) delete mode 100644 test/db/test_entityHelper.py create mode 100644 test/db/test_entity_helper.py diff --git a/test/db/test_entityHelper.py b/test/db/test_entityHelper.py deleted file mode 100644 index 9fac7c99..00000000 --- a/test/db/test_entityHelper.py +++ /dev/null @@ -1,171 +0,0 @@ -import datetime -from unittest import TestCase - -from datagateway_api.common.database.models import ( - DATAFILE, - DATAFILEFORMAT, - DATASET, - INVESTIGATION, -) - - -class TestEntityHelper(TestCase): - def setUp(self): - self.dataset = DATASET() - self.investigation = INVESTIGATION() - self.dataset.INVESTIGATION = self.investigation - self.datafileformat = DATAFILEFORMAT() - self.datafile = DATAFILE() - self.datafile.ID = 1 - self.datafile.LOCATION = "test location" - self.datafile.DATASET = self.dataset - self.datafile.DATAFILEFORMAT = self.datafileformat - self.datafile.NAME = "test name" - self.datafile.MOD_TIME = datetime.datetime(2000, 1, 1) - self.datafile.CREATE_TIME = datetime.datetime(2000, 1, 1) - self.datafile.CHECKSUM = "test checksum" - self.datafile.FILESIZE = 64 - self.datafile.DATAFILEMODTIME = datetime.datetime(2000, 1, 1) - self.datafile.DATAFILECREATETIME = datetime.datetime(2000, 1, 1) - self.datafile.DATASET_ID = 1 - self.datafile.DOI = "test doi" - self.datafile.DESCRIPTION = "test description" - self.datafile.CREATE_ID = "test create id" - self.datafile.MOD_ID = "test mod id" - self.datafile.DATAFILEFORMAT_ID = 1 - - def test_to_dict(self): - expected_dict = { - "ID": 1, - "LOCATION": "test location", - "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), - "CHECKSUM": "test checksum", - "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), - "DATASET_ID": 1, - "DOI": "test doi", - "DESCRIPTION": "test description", - "CREATE_ID": "test create id", - "MOD_ID": "test mod id", - "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), - } - self.assertEqual(expected_dict, self.datafile.to_dict()) - - def test_to_nested_dict(self): - expected_dict = { - "ID": 1, - "LOCATION": "test location", - "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), - "CHECKSUM": "test checksum", - "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), - "DATASET_ID": 1, - "DOI": "test doi", - "DESCRIPTION": "test description", - "CREATE_ID": "test create id", - "MOD_ID": "test mod id", - "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), - "DATASET": { - "ID": None, - "CREATE_TIME": None, - "MOD_TIME": None, - "CREATE_ID": None, - "MOD_ID": None, - "INVESTIGATION_ID": None, - "COMPLETE": None, - "DESCRIPTION": None, - "DOI": None, - "END_DATE": None, - "LOCATION": None, - "NAME": None, - "STARTDATE": None, - "SAMPLE_ID": None, - "TYPE_ID": None, - }, - } - self.assertEqual(expected_dict, self.datafile.to_nested_dict("DATASET")) - expected_dict = { - "ID": 1, - "LOCATION": "test location", - "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), - "CHECKSUM": "test checksum", - "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), - "DATASET_ID": 1, - "DOI": "test doi", - "DESCRIPTION": "test description", - "CREATE_ID": "test create id", - "MOD_ID": "test mod id", - "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), - "DATASET": { - "ID": None, - "CREATE_TIME": None, - "MOD_TIME": None, - "CREATE_ID": None, - "MOD_ID": None, - "INVESTIGATION_ID": None, - "COMPLETE": None, - "DESCRIPTION": None, - "DOI": None, - "END_DATE": None, - "LOCATION": None, - "NAME": None, - "STARTDATE": None, - "SAMPLE_ID": None, - "TYPE_ID": None, - "INVESTIGATION": { - "ID": None, - "CREATE_ID": None, - "CREATE_TIME": None, - "DOI": None, - "ENDDATE": None, - "MOD_ID": None, - "MOD_TIME": None, - "NAME": None, - "RELEASEDATE": None, - "STARTDATE": None, - "SUMMARY": None, - "TITLE": None, - "VISIT_ID": None, - "FACILITY_ID": None, - "TYPE_ID": None, - }, - }, - } - self.assertEqual( - expected_dict, self.datafile.to_nested_dict({"DATASET": "INVESTIGATION"}), - ) - - def test_get_related_entity(self): - self.assertEqual(self.dataset, self.datafile.get_related_entity("DATASET")) - - def test_update_from_dict(self): - datafile = DATAFILE() - dictionary = { - "ID": 1, - "LOCATION": "test location", - "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), - "CHECKSUM": "test checksum", - "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), - "DATASET_ID": 1, - "DOI": "test doi", - "DESCRIPTION": "test description", - "CREATE_ID": "test create id", - "MOD_ID": "test mod id", - "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), - } - datafile.update_from_dict(dictionary) - self.assertEqual(dictionary, datafile.to_dict()) diff --git a/test/db/test_entity_helper.py b/test/db/test_entity_helper.py new file mode 100644 index 00000000..5f8e9c0e --- /dev/null +++ b/test/db/test_entity_helper.py @@ -0,0 +1,201 @@ +import datetime + +import pytest + +from datagateway_api.common.database.models import ( + DATAFILE, + DATAFILEFORMAT, + DATASET, + INVESTIGATION, +) + + +@pytest.fixture() +def dataset_entity(): + dataset = DATASET() + investigation = INVESTIGATION() + dataset.INVESTIGATION = investigation + + return dataset + + +@pytest.fixture() +def datafile_entity(dataset_entity): + datafileformat = DATAFILEFORMAT() + datafile = DATAFILE() + datafile.ID = 1 + datafile.LOCATION = "test location" + datafile.DATASET = dataset_entity + datafile.DATAFILEFORMAT = datafileformat + datafile.NAME = "test name" + datafile.MOD_TIME = datetime.datetime(2000, 1, 1) + datafile.CREATE_TIME = datetime.datetime(2000, 1, 1) + datafile.CHECKSUM = "test checksum" + datafile.FILESIZE = 64 + datafile.DATAFILEMODTIME = datetime.datetime(2000, 1, 1) + datafile.DATAFILECREATETIME = datetime.datetime(2000, 1, 1) + datafile.DATASET_ID = 1 + datafile.DOI = "test doi" + datafile.DESCRIPTION = "test description" + datafile.CREATE_ID = "test create id" + datafile.MOD_ID = "test mod id" + datafile.DATAFILEFORMAT_ID = 1 + + return datafile + + +class TestEntityHelper: + def test_valid_to_dict(self, datafile_entity): + expected_dict = { + "ID": 1, + "LOCATION": "test location", + "NAME": "test name", + "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "CHECKSUM": "test checksum", + "FILESIZE": 64, + "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATASET_ID": 1, + "DOI": "test doi", + "DESCRIPTION": "test description", + "CREATE_ID": "test create id", + "MOD_ID": "test mod id", + "DATAFILEFORMAT_ID": 1, + "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + } + + test_data = datafile_entity.to_dict() + + assert expected_dict == test_data + + @pytest.mark.parametrize( + "expected_dict, entity_names", + [ + pytest.param( + { + "ID": 1, + "LOCATION": "test location", + "NAME": "test name", + "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "CHECKSUM": "test checksum", + "FILESIZE": 64, + "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATASET_ID": 1, + "DOI": "test doi", + "DESCRIPTION": "test description", + "CREATE_ID": "test create id", + "MOD_ID": "test mod id", + "DATAFILEFORMAT_ID": 1, + "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + "DATASET": { + "ID": None, + "CREATE_TIME": None, + "MOD_TIME": None, + "CREATE_ID": None, + "MOD_ID": None, + "INVESTIGATION_ID": None, + "COMPLETE": None, + "DESCRIPTION": None, + "DOI": None, + "END_DATE": None, + "LOCATION": None, + "NAME": None, + "STARTDATE": None, + "SAMPLE_ID": None, + "TYPE_ID": None, + }, + }, + "DATASET", + id="Dataset", + ), + pytest.param( + { + "ID": 1, + "LOCATION": "test location", + "NAME": "test name", + "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "CHECKSUM": "test checksum", + "FILESIZE": 64, + "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATASET_ID": 1, + "DOI": "test doi", + "DESCRIPTION": "test description", + "CREATE_ID": "test create id", + "MOD_ID": "test mod id", + "DATAFILEFORMAT_ID": 1, + "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + "DATASET": { + "ID": None, + "CREATE_TIME": None, + "MOD_TIME": None, + "CREATE_ID": None, + "MOD_ID": None, + "INVESTIGATION_ID": None, + "COMPLETE": None, + "DESCRIPTION": None, + "DOI": None, + "END_DATE": None, + "LOCATION": None, + "NAME": None, + "STARTDATE": None, + "SAMPLE_ID": None, + "TYPE_ID": None, + "INVESTIGATION": { + "ID": None, + "CREATE_ID": None, + "CREATE_TIME": None, + "DOI": None, + "ENDDATE": None, + "MOD_ID": None, + "MOD_TIME": None, + "NAME": None, + "RELEASEDATE": None, + "STARTDATE": None, + "SUMMARY": None, + "TITLE": None, + "VISIT_ID": None, + "FACILITY_ID": None, + "TYPE_ID": None, + }, + }, + }, + {"DATASET": "INVESTIGATION"}, + id="Dataset including investigation", + ), + ], + ) + def test_valid_to_nested_dict(self, datafile_entity, expected_dict, entity_names): + test_data = datafile_entity.to_nested_dict(entity_names) + + assert expected_dict == test_data + + def test_valid_get_related_entity(self, dataset_entity, datafile_entity): + assert dataset_entity == datafile_entity.get_related_entity("DATASET") + + def test_valid_update_from_dict(self, datafile_entity): + datafile = DATAFILE() + test_dict_data = { + "ID": 1, + "LOCATION": "test location", + "NAME": "test name", + "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "CHECKSUM": "test checksum", + "FILESIZE": 64, + "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATASET_ID": 1, + "DOI": "test doi", + "DESCRIPTION": "test description", + "CREATE_ID": "test create id", + "MOD_ID": "test mod id", + "DATAFILEFORMAT_ID": 1, + "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + } + + datafile.update_from_dict(test_dict_data) + + expected_datafile_dict = datafile_entity.to_dict() + + assert test_dict_data == expected_datafile_dict From cf89d7e15d4b5c084feefef299058ee73cc6bf0b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 15:36:08 +0000 Subject: [PATCH 078/109] #150: Rewrite database helper tests - The file has been renamed to reflect these tests actually test the QueryFilterFactory --- test/db/test_database_helpers.py | 99 ---------------------------- test/db/test_query_filter_factory.py | 74 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 99 deletions(-) delete mode 100644 test/db/test_database_helpers.py create mode 100644 test/db/test_query_filter_factory.py diff --git a/test/db/test_database_helpers.py b/test/db/test_database_helpers.py deleted file mode 100644 index b1ee2ae2..00000000 --- a/test/db/test_database_helpers.py +++ /dev/null @@ -1,99 +0,0 @@ -from unittest import TestCase - -from datagateway_api.common.config import config -from datagateway_api.common.exceptions import ApiError -from datagateway_api.common.query_filter import QueryFilterFactory - -backend_type = config.get_backend_type() -if backend_type == "db": - from datagateway_api.common.database.filters import ( - DatabaseDistinctFieldFilter as DistinctFieldFilter, - DatabaseIncludeFilter as IncludeFilter, - DatabaseLimitFilter as LimitFilter, - DatabaseOrderFilter as OrderFilter, - DatabaseSkipFilter as SkipFilter, - DatabaseWhereFilter as WhereFilter, - ) -elif backend_type == "python_icat": - # TODO - Adapt these tests for the ICAT implementation of filters - from datagateway_api.common.icat.filters import ( - PythonICATDistinctFieldFilter as DistinctFieldFilter, - PythonICATIncludeFilter as IncludeFilter, - PythonICATLimitFilter as LimitFilter, - PythonICATOrderFilter as OrderFilter, - PythonICATSkipFilter as SkipFilter, - PythonICATWhereFilter as WhereFilter, - ) -else: - raise ApiError( - "Cannot select which implementation of filters to import, check the config file" - " has a valid backend type", - ) - - -class TestQueryFilterFactory(TestCase): - def test_order_filter(self): - self.assertIs( - OrderFilter, - type(QueryFilterFactory.get_query_filter({"order": "ID DESC"})), - ) - - def test_limit_filter(self): - self.assertIs( - LimitFilter, type(QueryFilterFactory.get_query_filter({"limit": 10})), - ) - - def test_skip_filter(self): - self.assertIs( - SkipFilter, type(QueryFilterFactory.get_query_filter({"skip": 10})), - ) - - def test_where_filter(self): - self.assertIs( - WhereFilter, - type(QueryFilterFactory.get_query_filter({"where": {"ID": {"eq": "1"}}})), - ) - self.assertIs( - WhereFilter, - type(QueryFilterFactory.get_query_filter({"where": {"ID": {"lte": "1"}}})), - ) - self.assertIs( - WhereFilter, - type(QueryFilterFactory.get_query_filter({"where": {"ID": {"gte": "1"}}})), - ) - self.assertIs( - WhereFilter, - type(QueryFilterFactory.get_query_filter({"where": {"ID": {"like": "3"}}})), - ) - self.assertIs( - WhereFilter, - type( - QueryFilterFactory.get_query_filter( - {"where": {"ID": {"in": ["1", "2", "3"]}}}, - ), - ), - ) - - def test_include_filter(self): - self.assertIs( - IncludeFilter, - type(QueryFilterFactory.get_query_filter({"include": "DATAFILE"})), - ) - self.assertIs( - IncludeFilter, - type(QueryFilterFactory.get_query_filter({"include": ["TEST"]})), - ) - self.assertIs( - IncludeFilter, - type( - QueryFilterFactory.get_query_filter( - {"include": {"Test": ["TEST1", "Test2"]}}, - ), - ), - ) - - def test_distinct_filter(self): - self.assertIs( - DistinctFieldFilter, - type(QueryFilterFactory.get_query_filter({"distinct": "TEST"})), - ) diff --git a/test/db/test_query_filter_factory.py b/test/db/test_query_filter_factory.py new file mode 100644 index 00000000..dfc6f745 --- /dev/null +++ b/test/db/test_query_filter_factory.py @@ -0,0 +1,74 @@ +import pytest + +from datagateway_api.common.database.filters import ( + DatabaseDistinctFieldFilter, + DatabaseIncludeFilter, + DatabaseLimitFilter, + DatabaseOrderFilter, + DatabaseSkipFilter, + DatabaseWhereFilter, +) +from datagateway_api.common.query_filter import QueryFilterFactory + + +class TestQueryFilterFactory: + @pytest.mark.usefixtures("flask_test_app_db") + def test_valid_distinct_filter(self): + assert isinstance( + QueryFilterFactory.get_query_filter({"distinct": "TEST"}), + DatabaseDistinctFieldFilter, + ) + + @pytest.mark.usefixtures("flask_test_app_db") + @pytest.mark.parametrize( + "filter_input", + [ + pytest.param({"include": "DATAFILE"}, id="string"), + pytest.param({"include": ["TEST"]}, id="list of strings inside dictionary"), + pytest.param( + {"include": {"Test": ["TEST1", "Test2"]}}, + id="list of strings inside nested dictionary", + ), + ], + ) + def test_valid_include_filter(self, filter_input): + assert isinstance( + QueryFilterFactory.get_query_filter(filter_input), DatabaseIncludeFilter, + ) + + @pytest.mark.usefixtures("flask_test_app_db") + def test_valid_limit_filter(self): + assert isinstance( + QueryFilterFactory.get_query_filter({"limit": 10}), DatabaseLimitFilter, + ) + + @pytest.mark.usefixtures("flask_test_app_db") + def test_valid_order_filter(self): + assert isinstance( + QueryFilterFactory.get_query_filter({"order": "ID DESC"}), + DatabaseOrderFilter, + ) + + @pytest.mark.usefixtures("flask_test_app_db") + def test_valid_skip_filter(self): + assert isinstance( + QueryFilterFactory.get_query_filter({"skip": 10}), DatabaseSkipFilter, + ) + + @pytest.mark.usefixtures("flask_test_app_db") + @pytest.mark.parametrize( + "filter_input", + [ + pytest.param({"where": {"ID": {"eq": "1"}}}, id="eq operator"), + pytest.param({"where": {"ID": {"gt": "1"}}}, id="gt operator"), + pytest.param({"where": {"ID": {"gte": "1"}}}, id="gte operator"), + pytest.param({"where": {"ID": {"in": ["1", "2", "3"]}}}, id="in operator"), + pytest.param({"where": {"ID": {"like": "3"}}}, id="like operator"), + pytest.param({"where": {"ID": {"lt": "1"}}}, id="lt operator"), + pytest.param({"where": {"ID": {"lte": "1"}}}, id="lte operator"), + ], + ) + def test_valid_where_filter(self, filter_input): + assert isinstance( + QueryFilterFactory.get_query_filter(filter_input), DatabaseWhereFilter, + ) From b97b19bff6b74239f3f6d49f92acb2bdf98ea3e9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 16:02:40 +0000 Subject: [PATCH 079/109] #150: Rewrite JSON validity tests --- test/db/test_helpers.py | 27 --------------------------- test/test_is_valid_json.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 test/test_is_valid_json.py diff --git a/test/db/test_helpers.py b/test/db/test_helpers.py index b3972987..2af98158 100644 --- a/test/db/test_helpers.py +++ b/test/db/test_helpers.py @@ -26,38 +26,11 @@ from datagateway_api.common.helpers import ( get_filters_from_query_string, get_session_id_from_auth_header, - is_valid_json, queries_records, ) from test.test_base import FlaskAppTest -class TestIsValidJSON(TestCase): - def test_array(self): - self.assertTrue(is_valid_json("[]")) - - def test_null(self): - self.assertTrue("null") - - def test_valid_json(self): - self.assertTrue(is_valid_json('{"test":1}')) - - def test_single_quoted_json(self): - self.assertFalse(is_valid_json("{'test':1}")) - - def test_none(self): - self.assertFalse(is_valid_json(None)) - - def test_int(self): - self.assertFalse(is_valid_json(1)) - - def test_dict(self): - self.assertFalse(is_valid_json({"test": 1})) - - def test_list(self): - self.assertFalse(is_valid_json([])) - - class TestRequiresSessionID(FlaskAppTest): def setUp(self): super().setUp() diff --git a/test/test_is_valid_json.py b/test/test_is_valid_json.py new file mode 100644 index 00000000..b83f3eec --- /dev/null +++ b/test/test_is_valid_json.py @@ -0,0 +1,34 @@ +import pytest + +from datagateway_api.common.helpers import is_valid_json + + +class TestIsValidJSON: + @pytest.mark.parametrize( + "input_json", + [ + pytest.param("[]", id="empty array"), + pytest.param("null", id="null"), + pytest.param('{"test":1}', id="key-value pair"), + pytest.param('{"test":{"inner_key":"inner_value"}}', id="nested json"), + ], + ) + def test_valid_json_input(self, input_json): + valid_json = is_valid_json(input_json) + + assert valid_json + + @pytest.mark.parametrize( + "invalid_json", + [ + pytest.param("{'test':1}", id="single quotes"), + pytest.param(None, id="none"), + pytest.param(1, id="integer"), + pytest.param({"test": 1}, id="dictionary"), + pytest.param([], id="empty list"), + ], + ) + def test_invalid_json_input(self, invalid_json): + valid_json = is_valid_json(invalid_json) + + assert not valid_json From a07f0dbd4ed432862458d311cd8689e5488e60c6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 16:46:29 +0000 Subject: [PATCH 080/109] #150: Rewrite DB session decorator tests --- test/conftest.py | 29 +++++++++++++++-- test/db/test_helpers.py | 48 ----------------------------- test/db/test_requires_session_id.py | 31 +++++++++++++++++++ 3 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 test/db/test_requires_session_id.py diff --git a/test/conftest.py b/test/conftest.py index 7ffb87a8..efb55afa 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta import uuid from flask import Flask @@ -8,6 +8,11 @@ import pytest from datagateway_api.common.config import config +from datagateway_api.common.database.helpers import ( + delete_row_by_id, + insert_row_into_table, +) +from datagateway_api.common.database.models import SESSION from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure from test.icat.test_query import prepare_icat_data_for_assertion @@ -25,10 +30,30 @@ def valid_credentials_header(icat_client): @pytest.fixture() -def invalid_credentials_header(): +def valid_db_credentials_header(): + session = SESSION() + session.ID = "Test" + session.EXPIREDATETIME = datetime.now() + timedelta(hours=1) + session.username = "Test User" + + insert_row_into_table(SESSION, session) + + yield {"Authorization": "Bearer Test"} + + delete_row_by_id(SESSION, "Test") + + +@pytest.fixture() +def bad_credentials_header(): return {"Authorization": "Bearer Invalid"} +# TODO - Implement this in test_session_handling.py +@pytest.fixture() +def invalid_credentials_header(): + return {"Authorization": "Test"} + + @pytest.fixture() def icat_query(icat_client): return Query(icat_client, "Investigation") diff --git a/test/db/test_helpers.py b/test/db/test_helpers.py index 2af98158..f69eb71b 100644 --- a/test/db/test_helpers.py +++ b/test/db/test_helpers.py @@ -1,4 +1,3 @@ -from datetime import datetime, timedelta from unittest import TestCase from sqlalchemy.exc import IntegrityError @@ -11,11 +10,6 @@ DatabaseSkipFilter, DatabaseWhereFilter, ) -from datagateway_api.common.database.helpers import ( - delete_row_by_id, - insert_row_into_table, -) -from datagateway_api.common.database.models import SESSION from datagateway_api.common.exceptions import ( AuthenticationError, BadRequestError, @@ -31,48 +25,6 @@ from test.test_base import FlaskAppTest -class TestRequiresSessionID(FlaskAppTest): - def setUp(self): - super().setUp() - self.good_credentials_header = {"Authorization": "Bearer Test"} - self.invalid_credentials_header = {"Authorization": "Test"} - self.bad_credentials_header = {"Authorization": "Bearer BadTest"} - session = SESSION() - session.ID = "Test" - session.EXPIREDATETIME = datetime.now() + timedelta(hours=1) - session.username = "Test User" - - insert_row_into_table(SESSION, session) - - def tearDown(self): - delete_row_by_id(SESSION, "Test") - - def test_missing_credentials(self): - self.assertEqual(401, self.app.get("/datafiles").status_code) - - def test_invalid_credentials(self): - self.assertEqual( - 403, - self.app.get( - "/datafiles", headers=self.invalid_credentials_header, - ).status_code, - ) - - def test_bad_credentials(self): - self.assertEqual( - 403, - self.app.get("/datafiles", headers=self.bad_credentials_header).status_code, - ) - - def test_good_credentials(self): - self.assertEqual( - 200, - self.app.get( - "/datafiles?limit=0", headers=self.good_credentials_header, - ).status_code, - ) - - class TestQueriesRecords(TestCase): def test_missing_record_error(self): @queries_records diff --git a/test/db/test_requires_session_id.py b/test/db/test_requires_session_id.py new file mode 100644 index 00000000..4ebf43a7 --- /dev/null +++ b/test/db/test_requires_session_id.py @@ -0,0 +1,31 @@ +class TestRequiresSessionID: + """ + This class tests the session decorator used for the database backend. The equivalent + decorator for the Python ICAT backend is tested in `test_session_handling.py` + """ + + def test_invalid_missing_credentials(self, flask_test_app_db): + test_response = flask_test_app_db.get("/datafiles") + + assert test_response.status_code == 401 + + def test_invalid_credentials(self, flask_test_app_db, invalid_credentials_header): + test_response = flask_test_app_db.get( + "/datafiles", headers=invalid_credentials_header, + ) + + assert test_response.status_code == 403 + + def test_bad_credentials(self, flask_test_app_db, bad_credentials_header): + test_response = flask_test_app_db.get( + "/datafiles", headers=bad_credentials_header, + ) + + assert test_response.status_code == 403 + + def test_valid_credentials(self, flask_test_app_db, valid_db_credentials_header): + test_response = flask_test_app_db.get( + "/datafiles?limit=0", headers=valid_db_credentials_header, + ) + + assert test_response.status_code == 200 From 89d58d6b7c361404821503df86f5fd9a7dc5386f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 16:47:33 +0000 Subject: [PATCH 081/109] #150: Rename valid ICAT creds header - This has been renamed due to the addition of a valid credentials header for the DB backend --- test/conftest.py | 2 +- .../icat/endpoints/test_count_with_filters.py | 8 +-- test/icat/endpoints/test_create.py | 26 ++++++---- test/icat/endpoints/test_delete_by_id.py | 12 +++-- test/icat/endpoints/test_findone.py | 8 +-- test/icat/endpoints/test_get_by_id.py | 16 +++--- test/icat/endpoints/test_get_with_filters.py | 16 +++--- test/icat/endpoints/test_table_endpoints.py | 50 +++++++++++-------- test/icat/endpoints/test_update_by_id.py | 8 +-- test/icat/endpoints/test_update_multiple.py | 22 +++++--- test/icat/test_session_handling.py | 22 ++++---- 11 files changed, 107 insertions(+), 83 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index efb55afa..b47e8f25 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -25,7 +25,7 @@ def icat_client(): @pytest.fixture() -def valid_credentials_header(icat_client): +def valid_icat_credentials_header(icat_client): return {"Authorization": f"Bearer {icat_client.sessionId}"} diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters.py index 9fb8fe0e..4b667edd 100644 --- a/test/icat/endpoints/test_count_with_filters.py +++ b/test/icat/endpoints/test_count_with_filters.py @@ -4,23 +4,23 @@ class TestCountWithFilters: @pytest.mark.usefixtures("single_investigation_test_data") def test_valid_count_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): test_response = flask_test_app_icat.get( '/investigations/count?where={"title": {"like": "Test data for the Python' ' ICAT Backend on DataGateway API"}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.json == 1 def test_valid_no_results_count_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): test_response = flask_test_app_icat.get( '/investigations/count?where={"title": {"like": "This filter should cause a' '404 for testing purposes..."}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.json == 0 diff --git a/test/icat/endpoints/test_create.py b/test/icat/endpoints/test_create.py index e7f5137a..635e5754 100644 --- a/test/icat/endpoints/test_create.py +++ b/test/icat/endpoints/test_create.py @@ -2,7 +2,9 @@ class TestCreateData: - def test_valid_create_data(self, flask_test_app_icat, valid_credentials_header): + def test_valid_create_data( + self, flask_test_app_icat, valid_icat_credentials_header + ): create_investigations_json = [ { "name": "Test Data for API Testing, Data Creation 1", @@ -32,7 +34,7 @@ def test_valid_create_data(self, flask_test_app_icat, valid_credentials_header): test_response = flask_test_app_icat.post( "/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=create_investigations_json, ) @@ -53,11 +55,12 @@ def test_valid_create_data(self, flask_test_app_icat, valid_credentials_header): # Delete the entities created by this test for investigation_id in test_data_ids: flask_test_app_icat.delete( - f"/investigations/{investigation_id}", headers=valid_credentials_header, + f"/investigations/{investigation_id}", + headers=valid_icat_credentials_header, ) def test_valid_boundary_create_data( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): """Create a single investigation, as opposed to multiple""" @@ -76,7 +79,7 @@ def test_valid_boundary_create_data( test_response = flask_test_app_icat.post( "/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=create_investigation_json, ) @@ -91,10 +94,13 @@ def test_valid_boundary_create_data( assert [create_investigation_json] == response_json flask_test_app_icat.delete( - f"/investigations/{created_test_data_id}", headers=valid_credentials_header, + f"/investigations/{created_test_data_id}", + headers=valid_icat_credentials_header, ) - def test_invalid_create_data(self, flask_test_app_icat, valid_credentials_header): + def test_invalid_create_data( + self, flask_test_app_icat, valid_icat_credentials_header + ): """An investigation requires a minimum of: name, visitId, facility, type""" invalid_request_body = { @@ -103,7 +109,7 @@ def test_invalid_create_data(self, flask_test_app_icat, valid_credentials_header test_response = flask_test_app_icat.post( "/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=invalid_request_body, ) @@ -112,7 +118,7 @@ def test_invalid_create_data(self, flask_test_app_icat, valid_credentials_header def test_invalid_existing_data_create( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): """This test targets raising ICATObjectExistsError, causing a 400""" @@ -129,7 +135,7 @@ def test_invalid_existing_data_create( test_response = flask_test_app_icat.post( "/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=existing_object_json, ) diff --git a/test/icat/endpoints/test_delete_by_id.py b/test/icat/endpoints/test_delete_by_id.py index 4c811919..942fe0bb 100644 --- a/test/icat/endpoints/test_delete_by_id.py +++ b/test/icat/endpoints/test_delete_by_id.py @@ -2,29 +2,31 @@ class TestDeleteByID: def test_valid_delete_with_id( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): test_response = flask_test_app_icat.delete( f'/investigations/{single_investigation_test_data[0]["id"]}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.status_code == 204 def test_invalid_delete_with_id( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): """Request with a non-existent ID""" final_investigation_result = flask_test_app_icat.get( - '/investigations/findone?order="id DESC"', headers=valid_credentials_header, + '/investigations/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) test_data_id = final_investigation_result.json["id"] # Adding 100 onto the ID to the most recent result should ensure a 404 test_response = flask_test_app_icat.delete( - f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, + f"/investigations/{test_data_id + 100}", + headers=valid_icat_credentials_header, ) assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_findone.py b/test/icat/endpoints/test_findone.py index 96ee5374..3934eb22 100644 --- a/test/icat/endpoints/test_findone.py +++ b/test/icat/endpoints/test_findone.py @@ -5,25 +5,25 @@ class TestFindone: def test_valid_findone_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): test_response = flask_test_app_icat.get( '/investigations/findone?where={"title": {"like": "Test data for the Python' ' ICAT Backend on DataGateway API"}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) response_json = prepare_icat_data_for_assertion([test_response.json]) assert response_json == single_investigation_test_data def test_valid_no_results_findone_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): test_response = flask_test_app_icat.get( '/investigations/findone?where={"title": {"eq": "This filter should cause a' '404 for testing purposes..."}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id.py index 1271c9b5..daed4610 100644 --- a/test/icat/endpoints/test_get_by_id.py +++ b/test/icat/endpoints/test_get_by_id.py @@ -5,19 +5,19 @@ class TestGetByID: def test_valid_get_with_id( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): # Need to identify the ID given to the test data investigation_data = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) test_data_id = investigation_data.json[0]["id"] test_response = flask_test_app_icat.get( - f"/investigations/{test_data_id}", headers=valid_credentials_header, + f"/investigations/{test_data_id}", headers=valid_icat_credentials_header, ) # Get with ID gives a dictionary response (only ever one result from that kind # of request), so list around json is required for the call @@ -25,17 +25,21 @@ def test_valid_get_with_id( assert response_json == single_investigation_test_data - def test_invalid_get_with_id(self, flask_test_app_icat, valid_credentials_header): + def test_invalid_get_with_id( + self, flask_test_app_icat, valid_icat_credentials_header + ): """Request with a non-existent ID""" final_investigation_result = flask_test_app_icat.get( - '/investigations/findone?order="id DESC"', headers=valid_credentials_header, + '/investigations/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) test_data_id = final_investigation_result.json["id"] # Adding 100 onto the ID to the most recent result should ensure a 404 test_response = flask_test_app_icat.get( - f"/investigations/{test_data_id + 100}", headers=valid_credentials_header, + f"/investigations/{test_data_id + 100}", + headers=valid_icat_credentials_header, ) assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_get_with_filters.py b/test/icat/endpoints/test_get_with_filters.py index 59ebeab8..d8f67f39 100644 --- a/test/icat/endpoints/test_get_with_filters.py +++ b/test/icat/endpoints/test_get_with_filters.py @@ -7,37 +7,37 @@ class TestGetWithFilters: def test_valid_get_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): test_response = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) response_json = prepare_icat_data_for_assertion(test_response.json) assert response_json == single_investigation_test_data def test_valid_no_results_get_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): test_response = flask_test_app_icat.get( '/investigations?where={"title": {"eq": "This filter should cause a 404 for' 'testing purposes..."}}', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.status_code == 404 @pytest.mark.usefixtures("multiple_investigation_test_data") def test_valid_get_with_filters_distinct( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): test_response = flask_test_app_icat.get( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}&distinct="title"', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) expected = [ @@ -53,7 +53,7 @@ def test_valid_get_with_filters_distinct( def test_limit_skip_merge_get_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, multiple_investigation_test_data, ): skip_value = 1 @@ -63,7 +63,7 @@ def test_limit_skip_merge_get_with_filters( '/investigations?where={"title": {"like": "Test data for the Python ICAT' ' Backend on DataGateway API"}}' f'&skip={skip_value}&limit={limit_value}&order="id ASC"', - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) response_json = prepare_icat_data_for_assertion(test_response.json) diff --git a/test/icat/endpoints/test_table_endpoints.py b/test/icat/endpoints/test_table_endpoints.py index 2d07b20e..6d607537 100644 --- a/test/icat/endpoints/test_table_endpoints.py +++ b/test/icat/endpoints/test_table_endpoints.py @@ -10,12 +10,12 @@ class TestTableEndpoints: def test_valid_get_facility_cycles_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, isis_specific_endpoint_data, ): test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) response_json = prepare_icat_data_for_assertion(test_response.json) @@ -23,16 +23,17 @@ def test_valid_get_facility_cycles_with_filters( assert response_json == isis_specific_endpoint_data[1] def test_invalid_get_facility_cycles_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + '/instruments/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.status_code == 404 @@ -40,27 +41,28 @@ def test_invalid_get_facility_cycles_with_filters( def test_valid_get_facility_cycles_count_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, isis_specific_endpoint_data, ): test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/count", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.json == 1 def test_invalid_get_facility_cycles_count_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + '/instruments/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/count", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.json == 0 @@ -68,13 +70,13 @@ def test_invalid_get_facility_cycles_count_with_filters( def test_valid_get_investigations_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, isis_specific_endpoint_data, ): test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/" f"{isis_specific_endpoint_data[2]}/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) response_json = prepare_icat_data_for_assertion(test_response.json) @@ -82,21 +84,23 @@ def test_valid_get_investigations_with_filters( assert response_json == isis_specific_endpoint_data[3] def test_invalid_get_investigations_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + '/instruments/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] final_facilitycycle_result = flask_test_app_icat.get( - '/facilitycycles/findone?order="id DESC"', headers=valid_credentials_header, + '/facilitycycles/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) final_facilitycycle_id = final_facilitycycle_result.json["id"] test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.status_code == 404 @@ -104,33 +108,35 @@ def test_invalid_get_investigations_with_filters( def test_valid_get_investigations_count_with_filters( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, isis_specific_endpoint_data, ): test_response = flask_test_app_icat.get( f"/instruments/{isis_specific_endpoint_data[0]}/facilitycycles/" f"{isis_specific_endpoint_data[2]}/investigations/count", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.json == 1 def test_invalid_get_investigations_count_with_filters( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', headers=valid_credentials_header, + '/instruments/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) final_instrument_id = final_instrument_result.json["id"] final_facilitycycle_result = flask_test_app_icat.get( - '/facilitycycles/findone?order="id DESC"', headers=valid_credentials_header, + '/facilitycycles/findone?order="id DESC"', + headers=valid_icat_credentials_header, ) final_facilitycycle_id = final_facilitycycle_result.json["id"] test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations/count", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, ) assert test_response.json == 0 diff --git a/test/icat/endpoints/test_update_by_id.py b/test/icat/endpoints/test_update_by_id.py index 127ccdd3..bd6d4727 100644 --- a/test/icat/endpoints/test_update_by_id.py +++ b/test/icat/endpoints/test_update_by_id.py @@ -5,7 +5,7 @@ class TestUpdateByID: def test_valid_update_with_id( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): expected_doi = "Test Data Identifier" @@ -20,7 +20,7 @@ def test_valid_update_with_id( test_response = flask_test_app_icat.patch( f"/investigations/{single_investigation_test_data[0]['id']}", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=update_data_json, ) response_json = prepare_icat_data_for_assertion([test_response.json]) @@ -30,7 +30,7 @@ def test_valid_update_with_id( def test_invalid_update_with_id( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): """This test will attempt to put `icatdb` into an invalid state""" @@ -45,7 +45,7 @@ def test_invalid_update_with_id( test_response = flask_test_app_icat.patch( f"/investigations/{single_investigation_test_data[0]['id']}", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=invalid_update_json, ) diff --git a/test/icat/endpoints/test_update_multiple.py b/test/icat/endpoints/test_update_multiple.py index b4fadc26..0cd02415 100644 --- a/test/icat/endpoints/test_update_multiple.py +++ b/test/icat/endpoints/test_update_multiple.py @@ -7,7 +7,7 @@ class TestUpdateMultipleEntities: def test_valid_multiple_update_data( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, multiple_investigation_test_data, ): expected_doi = "Test Data Identifier" @@ -27,7 +27,9 @@ def test_valid_multiple_update_data( update_data_list.append(update_entity) test_response = flask_test_app_icat.patch( - "/investigations", headers=valid_credentials_header, json=update_data_list, + "/investigations", + headers=valid_icat_credentials_header, + json=update_data_list, ) response_json = prepare_icat_data_for_assertion(test_response.json) @@ -36,7 +38,7 @@ def test_valid_multiple_update_data( def test_valid_boundary_update_data( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): """ Request body is a dictionary, not a list of dictionaries""" @@ -53,7 +55,9 @@ def test_valid_boundary_update_data( single_investigation_test_data[0]["summary"] = expected_summary test_response = flask_test_app_icat.patch( - "/investigations", headers=valid_credentials_header, json=update_data_json, + "/investigations", + headers=valid_icat_credentials_header, + json=update_data_json, ) response_json = prepare_icat_data_for_assertion(test_response.json) @@ -62,7 +66,7 @@ def test_valid_boundary_update_data( def test_invalid_missing_update_data( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, ): """There should be an ID in the request body to know which entity to update""" @@ -73,7 +77,9 @@ def test_invalid_missing_update_data( } test_response = flask_test_app_icat.patch( - "/investigations", headers=valid_credentials_header, json=update_data_json, + "/investigations", + headers=valid_icat_credentials_header, + json=update_data_json, ) assert test_response.status_code == 400 @@ -88,7 +94,7 @@ def test_invalid_missing_update_data( def test_invalid_attribute_update( self, flask_test_app_icat, - valid_credentials_header, + valid_icat_credentials_header, single_investigation_test_data, update_key, update_value, @@ -100,7 +106,7 @@ def test_invalid_attribute_update( test_response = flask_test_app_icat.patch( "/investigations", - headers=valid_credentials_header, + headers=valid_icat_credentials_header, json=invalid_update_data_json, ) diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 4828e6db..c01fd33e 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -12,10 +12,10 @@ def test_session_id_decorator(self): pass def test_get_valid_session_details( - self, flask_test_app_icat, valid_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, ): session_details = flask_test_app_icat.get( - "/sessions", headers=valid_credentials_header, + "/sessions", headers=valid_icat_credentials_header, ) # Check username is correct @@ -26,7 +26,7 @@ def test_get_valid_session_details( # Check session ID matches the header from the request assert ( session_details.json["ID"] - == valid_credentials_header["Authorization"].split()[1] + == valid_icat_credentials_header["Authorization"].split()[1] ) session_expiry_datetime = datetime.strptime( @@ -40,26 +40,26 @@ def test_get_valid_session_details( assert time_diff_minutes < 120 and time_diff_minutes >= 118 def test_get_invalid_session_details( - self, invalid_credentials_header, flask_test_app_icat, + self, bad_credentials_header, flask_test_app_icat, ): session_details = flask_test_app_icat.get( - "/sessions", headers=invalid_credentials_header, + "/sessions", headers=bad_credentials_header, ) assert session_details.status_code == 403 - def test_refresh_session(self, valid_credentials_header, flask_test_app_icat): + def test_refresh_session(self, valid_icat_credentials_header, flask_test_app_icat): pre_refresh_session_details = flask_test_app_icat.get( - "/sessions", headers=valid_credentials_header, + "/sessions", headers=valid_icat_credentials_header, ) refresh_session = flask_test_app_icat.put( - "/sessions", headers=valid_credentials_header, + "/sessions", headers=valid_icat_credentials_header, ) assert refresh_session.status_code == 200 post_refresh_session_details = flask_test_app_icat.get( - "/sessions", headers=valid_credentials_header, + "/sessions", headers=valid_icat_credentials_header, ) assert ( @@ -108,9 +108,9 @@ def test_valid_logout(self, flask_test_app_icat): assert logout_response.status_code == 200 - def test_invalid_logout(self, invalid_credentials_header, flask_test_app_icat): + def test_invalid_logout(self, bad_credentials_header, flask_test_app_icat): logout_response = flask_test_app_icat.delete( - "/sessions", headers=invalid_credentials_header, + "/sessions", headers=bad_credentials_header, ) assert logout_response.status_code == 403 From e8f113975077b463f62c78be71778d20b2c8be3d Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 17:36:11 +0000 Subject: [PATCH 082/109] #150: Rewrite session ID from header tests --- test/conftest.py | 2 +- test/db/test_helpers.py | 20 -------------- test/icat/endpoints/test_create.py | 4 +-- test/icat/endpoints/test_get_by_id.py | 2 +- test/test_get_session_id_from_auth_header.py | 28 ++++++++++++++++++++ 5 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 test/test_get_session_id_from_auth_header.py diff --git a/test/conftest.py b/test/conftest.py index b47e8f25..25d9e4a8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -38,7 +38,7 @@ def valid_db_credentials_header(): insert_row_into_table(SESSION, session) - yield {"Authorization": "Bearer Test"} + yield {"Authorization": f"Bearer {session.ID}"} delete_row_by_id(SESSION, "Test") diff --git a/test/db/test_helpers.py b/test/db/test_helpers.py index f69eb71b..0795db57 100644 --- a/test/db/test_helpers.py +++ b/test/db/test_helpers.py @@ -11,15 +11,12 @@ DatabaseWhereFilter, ) from datagateway_api.common.exceptions import ( - AuthenticationError, BadRequestError, FilterError, - MissingCredentialsError, MissingRecordError, ) from datagateway_api.common.helpers import ( get_filters_from_query_string, - get_session_id_from_auth_header, queries_records, ) from test.test_base import FlaskAppTest @@ -92,23 +89,6 @@ def raise_bad_request_error(): self.assertEqual(400, ctx.exception.status_code) -class TestGetSessionIDFromAuthHeader(FlaskAppTest): - def test_no_session_in_header(self): - with self.app: - self.app.get("/") - self.assertRaises(MissingCredentialsError, get_session_id_from_auth_header) - - def test_with_bad_header(self): - with self.app: - self.app.get("/", headers={"Authorization": "test"}) - self.assertRaises(AuthenticationError, get_session_id_from_auth_header) - - def test_with_good_header(self): - with self.app: - self.app.get("/", headers={"Authorization": "Bearer test"}) - self.assertEqual("test", get_session_id_from_auth_header()) - - class TestGetFiltersFromQueryString(FlaskAppTest): def test_no_filters(self): with self.app: diff --git a/test/icat/endpoints/test_create.py b/test/icat/endpoints/test_create.py index 635e5754..fd6e607e 100644 --- a/test/icat/endpoints/test_create.py +++ b/test/icat/endpoints/test_create.py @@ -3,7 +3,7 @@ class TestCreateData: def test_valid_create_data( - self, flask_test_app_icat, valid_icat_credentials_header + self, flask_test_app_icat, valid_icat_credentials_header, ): create_investigations_json = [ { @@ -99,7 +99,7 @@ def test_valid_boundary_create_data( ) def test_invalid_create_data( - self, flask_test_app_icat, valid_icat_credentials_header + self, flask_test_app_icat, valid_icat_credentials_header, ): """An investigation requires a minimum of: name, visitId, facility, type""" diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id.py index daed4610..5075dee0 100644 --- a/test/icat/endpoints/test_get_by_id.py +++ b/test/icat/endpoints/test_get_by_id.py @@ -26,7 +26,7 @@ def test_valid_get_with_id( assert response_json == single_investigation_test_data def test_invalid_get_with_id( - self, flask_test_app_icat, valid_icat_credentials_header + self, flask_test_app_icat, valid_icat_credentials_header, ): """Request with a non-existent ID""" diff --git a/test/test_get_session_id_from_auth_header.py b/test/test_get_session_id_from_auth_header.py new file mode 100644 index 00000000..f28a5ce7 --- /dev/null +++ b/test/test_get_session_id_from_auth_header.py @@ -0,0 +1,28 @@ +import pytest + +from datagateway_api.common.exceptions import ( + AuthenticationError, + MissingCredentialsError, +) +from datagateway_api.common.helpers import get_session_id_from_auth_header + + +class TestGetSessionIDFromAuthHeader: + def test_invalid_no_credentials_in_header(self, flask_test_app_db): + with flask_test_app_db: + flask_test_app_db.get("/") + with pytest.raises(MissingCredentialsError): + get_session_id_from_auth_header() + + def test_invalid_header(self, flask_test_app_db, invalid_credentials_header): + with flask_test_app_db: + flask_test_app_db.get("/", headers=invalid_credentials_header) + with pytest.raises(AuthenticationError): + get_session_id_from_auth_header() + + def test_valid_header(self, flask_test_app_db, valid_db_credentials_header): + with flask_test_app_db: + flask_test_app_db.get("/", headers=valid_db_credentials_header) + session_id = valid_db_credentials_header["Authorization"].split()[1] + + assert session_id == get_session_id_from_auth_header() From 74215cdbfda4d1b4e4c417be3cd2d568d4143bb4 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Dec 2020 19:11:23 +0000 Subject: [PATCH 083/109] #150: Complete TODOs from 'backend can be set for tests' work --- datagateway_api/common/backends.py | 12 ++++++- datagateway_api/common/config.py | 10 ++++-- datagateway_api/common/query_filter.py | 15 +++++---- datagateway_api/src/main.py | 1 - .../src/resources/entities/entity_endpoint.py | 16 ++++++++++ .../non_entities/sessions_endpoints.py | 8 ++++- .../table_endpoints/table_endpoints.py | 32 ++++++++++++++++--- test/conftest.py | 6 +--- test/test_date_handler.py | 4 --- 9 files changed, 80 insertions(+), 24 deletions(-) diff --git a/datagateway_api/common/backends.py b/datagateway_api/common/backends.py index 552e0834..ed19a93a 100644 --- a/datagateway_api/common/backends.py +++ b/datagateway_api/common/backends.py @@ -7,7 +7,17 @@ def create_backend(backend_type): """ - TODO - Add docstring + Create an instance of a backend dependent on the value parsed into the function. The + value will typically be from the contents of `config.json`, however when creating a + backend during automated tests the value will be from the Flask app's config (which + will be set in the API's config at `common.config` + + The API will exit if a valid value isn't given. + + :param backend_type: The type of backend that should be created and used for the API + :type backend_type: :class:`str` + :return: Either an instance of `common.dataase.backend.DatabaseBackend` or + `common.icat.backend.PythonICATBackend` """ if backend_type == "db": diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index 6f69c16e..e5d55ec4 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -5,7 +5,6 @@ import requests -# from datagateway_api.src.main import app log = logging.getLogger() @@ -25,7 +24,14 @@ def get_backend_type(self): def set_backend_type(self, backend_type): """ - TODO - Explain the reason behind the setter + This setter is used as a way for automated tests to set the backend type. The + API can detect if the Flask app setup is from an automated test by checking the + app's config for a `TEST_BACKEND`. If this value exists (a KeyError will be + raised when the API is run normally, which will then grab the backend type from + `config.json`), it needs to be set using this function. This is required because + creating filters in the `QueryFilterFactory` is backend-specific so the backend + type must be fetched. This must be done using this module (rather than directly + importing and checking the Flask app's config) to avoid circular import issues. """ self.config["backend"] = backend_type diff --git a/datagateway_api/common/query_filter.py b/datagateway_api/common/query_filter.py index e0928ff2..e4e90308 100644 --- a/datagateway_api/common/query_filter.py +++ b/datagateway_api/common/query_filter.py @@ -13,15 +13,18 @@ class QueryFilterFactory(object): @staticmethod def get_query_filter(request_filter): """ - Given a filter return a matching Query filter object + Given a filter, return a matching Query filter object - This factory is not in common.filters so the created filter can be for the - correct backend. Moving the factory into that file would mean the filters would - be based off the abstract classes (because they're in the same file) which won't - enable filters to be unique to the backend + The filters are imported inside this method to enable the unit tests to not rely + on the contents of `config.json`. If they're imported at the top of the file, + the backend type won't have been updated if the Flask app has been created from + an automated test (file imports occur before `create_api_endpoints()` executes). - :param request_filter: dict - The filter to create the QueryFilter for + :param request_filter: The filter to create the QueryFilter for + :type request_filter: :class:`dict` :return: The QueryFilter object created + :raises ApiError: If the backend type contains an invalid value + :raises FilterError: If the filter name is not recognised """ backend_type = config.get_backend_type() diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 3b06614b..47ab71f1 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -73,7 +73,6 @@ def create_api_endpoints(flask_app, api, spec): except KeyError: backend_type = config.get_backend_type() - # TODO - Add :param backend: to the endpoint functions backend = create_backend(backend_type) for entity_name in endpoints: diff --git a/datagateway_api/src/resources/entities/entity_endpoint.py b/datagateway_api/src/resources/entities/entity_endpoint.py index da26dbba..3338f7b4 100644 --- a/datagateway_api/src/resources/entities/entity_endpoint.py +++ b/datagateway_api/src/resources/entities/entity_endpoint.py @@ -14,7 +14,11 @@ def get_endpoint(name, entity_type, backend): api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles") :param name: The name of the entity + :type name: :class:`str` :param entity_type: The entity the endpoint will use in queries + :type entity_type: :class:`str` + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` :return: The generated endpoint class """ @@ -162,7 +166,11 @@ def get_id_endpoint(name, entity_type, backend): api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/") :param name: The name of the entity + :type name: :class:`str` :param entity_type: The entity the endpoint will use in queries + :type entity_type: :class:`str` + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` :return: The generated id endpoint class """ @@ -292,7 +300,11 @@ def get_count_endpoint(name, entity_type, backend): api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/count") :param name: The name of the entity + :type name: :class:`str` :param entity_type: The entity the endpoint will use in queries + :type entity_type: :class:`str` + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` :return: The generated count endpoint class """ @@ -345,7 +357,11 @@ def get_find_one_endpoint(name, entity_type, backend): api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/findone") :param name: The name of the entity + :type name: :class:`str` :param entity_type: The entity the endpoint will use in queries + :type entity_type: :class:`str` + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` :return: The generated findOne endpoint class """ diff --git a/datagateway_api/src/resources/non_entities/sessions_endpoints.py b/datagateway_api/src/resources/non_entities/sessions_endpoints.py index febc969d..28af931a 100644 --- a/datagateway_api/src/resources/non_entities/sessions_endpoints.py +++ b/datagateway_api/src/resources/non_entities/sessions_endpoints.py @@ -12,7 +12,13 @@ def session_endpoints(backend): """ - TODO - Add docstring + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated session endpoint class """ class Sessions(Resource): diff --git a/datagateway_api/src/resources/table_endpoints/table_endpoints.py b/datagateway_api/src/resources/table_endpoints/table_endpoints.py index 6f2907a7..ad923ced 100644 --- a/datagateway_api/src/resources/table_endpoints/table_endpoints.py +++ b/datagateway_api/src/resources/table_endpoints/table_endpoints.py @@ -8,7 +8,13 @@ def instrument_facility_cycles_endpoint(backend): """ - TODO - Add docstring + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class """ pass @@ -68,7 +74,13 @@ def get(self, id_): def count_instrument_facility_cycles_endpoint(backend): """ - TODO - Add docstring + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class """ pass @@ -122,7 +134,13 @@ def get(self, id_): def instrument_investigation_endpoint(backend): """ - TODO - Add docstring + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class """ pass @@ -189,7 +207,13 @@ def get(self, instrument_id, cycle_id): def count_instrument_investigation_endpoint(backend): """ - TODO - Add docstring + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class """ pass diff --git a/test/conftest.py b/test/conftest.py index 25d9e4a8..ad70c0a0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -117,9 +117,7 @@ def multiple_investigation_test_data(icat_client): @pytest.fixture(scope="package") def flask_test_app(): - """ - TODO - Explain why a generic test app is needed that doesn't rely on any backend - """ + """This is used to check the endpoints exist and have the correct HTTP methods""" test_app = Flask(__name__) api, spec = create_app_infrastructure(test_app) create_api_endpoints(test_app, api, spec) @@ -129,7 +127,6 @@ def flask_test_app(): @pytest.fixture(scope="package") def flask_test_app_icat(flask_test_app): - """TODO - Explain ICAT test client""" icat_app = Flask(__name__) icat_app.config["TESTING"] = True icat_app.config["TEST_BACKEND"] = "python_icat" @@ -142,7 +139,6 @@ def flask_test_app_icat(flask_test_app): @pytest.fixture(scope="package") def flask_test_app_db(): - """TODO - Add DB test client doc""" db_app = Flask(__name__) db_app.config["TESTING"] = True db_app.config["TEST_BACKEND"] = "db" diff --git a/test/test_date_handler.py b/test/test_date_handler.py index aff4ba6f..8de17fed 100644 --- a/test/test_date_handler.py +++ b/test/test_date_handler.py @@ -71,7 +71,3 @@ def test_valid_boundary_datetime(self): ) str_date_output = DateHandler.datetime_object_to_str(example_date) assert str_date_output == "2020-02-29 23:59:59" - - def test_invalid_datetime(self): - # TODO - Not sure how to create an invalid datetime object - pass From d6f6ab8c82851c1adcf36b54ab44d21590685b4d Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 10:04:36 +0000 Subject: [PATCH 084/109] #150: Rewrite get_filter_from_query_string tests --- test/db/test_helpers.py | 108 +--------------------------- test/test_get_filters_from_query.py | 54 ++++++++++++++ 2 files changed, 55 insertions(+), 107 deletions(-) create mode 100644 test/test_get_filters_from_query.py diff --git a/test/db/test_helpers.py b/test/db/test_helpers.py index 0795db57..40e735ac 100644 --- a/test/db/test_helpers.py +++ b/test/db/test_helpers.py @@ -2,24 +2,12 @@ from sqlalchemy.exc import IntegrityError -from datagateway_api.common.database.filters import ( - DatabaseDistinctFieldFilter, - DatabaseIncludeFilter, - DatabaseLimitFilter, - DatabaseOrderFilter, - DatabaseSkipFilter, - DatabaseWhereFilter, -) from datagateway_api.common.exceptions import ( BadRequestError, FilterError, MissingRecordError, ) -from datagateway_api.common.helpers import ( - get_filters_from_query_string, - queries_records, -) -from test.test_base import FlaskAppTest +from datagateway_api.common.helpers import queries_records class TestQueriesRecords(TestCase): @@ -87,97 +75,3 @@ def raise_bad_request_error(): self.assertEqual("Bad request", str(ctx.exception)) self.assertEqual(400, ctx.exception.status_code) - - -class TestGetFiltersFromQueryString(FlaskAppTest): - def test_no_filters(self): - with self.app: - self.app.get("/") - self.assertEqual([], get_filters_from_query_string()) - - def test_bad_filter(self): - with self.app: - self.app.get('/?test="test"') - self.assertRaises(FilterError, get_filters_from_query_string) - - def test_limit_filter(self): - with self.app: - self.app.get("/?limit=10") - filters = get_filters_from_query_string() - self.assertEqual( - 1, len(filters), msg="Returned incorrect number of filters", - ) - self.assertIs( - DatabaseLimitFilter, type(filters[0]), msg="Incorrect type of filter", - ) - - def test_order_filter(self): - with self.app: - self.app.get('/?order="ID DESC"') - filters = get_filters_from_query_string() - self.assertEqual( - 1, len(filters), msg="Returned incorrect number of filters", - ) - self.assertIs( - DatabaseOrderFilter, - type(filters[0]), - msg="Incorrect type of filter returned", - ) - - def test_where_filter(self): - with self.app: - self.app.get('/?where={"ID":{"eq":3}}') - filters = get_filters_from_query_string() - self.assertEqual( - 1, len(filters), msg="Returned incorrect number of filters", - ) - self.assertIs( - DatabaseWhereFilter, - type(filters[0]), - msg="Incorrect type of filter returned", - ) - - def test_skip_filter(self): - with self.app: - self.app.get("/?skip=10") - filters = get_filters_from_query_string() - self.assertEqual( - 1, len(filters), msg="Returned incorrect number of filters", - ) - self.assertIs( - DatabaseSkipFilter, - type(filters[0]), - msg="Incorrect type of filter returned", - ) - - def test_include_filter(self): - with self.app: - self.app.get('/?include="TEST"') - filters = get_filters_from_query_string() - self.assertEqual( - 1, len(filters), msg="Incorrect number of filters returned", - ) - self.assertIs( - DatabaseIncludeFilter, - type(filters[0]), - msg="Incorrect type of filter returned", - ) - - def test_distinct_filter(self): - with self.app: - self.app.get('/?distinct="ID"') - filters = get_filters_from_query_string() - self.assertEqual( - 1, len(filters), msg="Incorrect number of filters returned", - ) - self.assertIs( - DatabaseDistinctFieldFilter, - type(filters[0]), - msg="Incorrect type of filter returned", - ) - - def test_multiple_filters(self): - with self.app: - self.app.get("/?limit=10&skip=4") - filters = get_filters_from_query_string() - self.assertEqual(2, len(filters)) diff --git a/test/test_get_filters_from_query.py b/test/test_get_filters_from_query.py new file mode 100644 index 00000000..2a0a3a48 --- /dev/null +++ b/test/test_get_filters_from_query.py @@ -0,0 +1,54 @@ +import pytest + +from datagateway_api.common.database.filters import ( + DatabaseDistinctFieldFilter, + DatabaseIncludeFilter, + DatabaseLimitFilter, + DatabaseOrderFilter, + DatabaseSkipFilter, +) +from datagateway_api.common.exceptions import FilterError +from datagateway_api.common.helpers import get_filters_from_query_string + + +class TestGetFiltersFromQueryString: + def test_valid_no_filters(self, flask_test_app_db): + with flask_test_app_db: + flask_test_app_db.get("/") + + assert [] == get_filters_from_query_string() + + def test_invalid_filter(self, flask_test_app_db): + with flask_test_app_db: + flask_test_app_db.get('/?test="test"') + + with pytest.raises(FilterError): + get_filters_from_query_string() + + @pytest.mark.parametrize( + "filter_input, filter_type", + [ + pytest.param( + 'distinct="ID"', DatabaseDistinctFieldFilter, id="DB distinct filter", + ), + pytest.param( + 'include="TEST"', DatabaseIncludeFilter, id="DB include filter", + ), + pytest.param("limit=10", DatabaseLimitFilter, id="DB limit filter"), + pytest.param('order="ID DESC"', DatabaseOrderFilter, id="DB order filter"), + pytest.param("skip=10", DatabaseSkipFilter, id="DB skip filter"), + ], + ) + def test_valid_filter(self, flask_test_app_db, filter_input, filter_type): + with flask_test_app_db: + flask_test_app_db.get(f"/?{filter_input}") + filters = get_filters_from_query_string() + + assert isinstance(filters[0], filter_type) + + def test_valid_multiple_filters(self, flask_test_app_db): + with flask_test_app_db: + flask_test_app_db.get("/?limit=10&skip=4") + filters = get_filters_from_query_string() + + assert len(filters) == 2 From 97bf1a4f5edab0f2c467b6bb64d6b8c60396a889 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 10:19:28 +0000 Subject: [PATCH 085/109] #150: Rewrite queries_records tests --- test/db/test_helpers.py | 77 ------------------------------------ test/test_queries_records.py | 36 +++++++++++++++++ 2 files changed, 36 insertions(+), 77 deletions(-) delete mode 100644 test/db/test_helpers.py create mode 100644 test/test_queries_records.py diff --git a/test/db/test_helpers.py b/test/db/test_helpers.py deleted file mode 100644 index 40e735ac..00000000 --- a/test/db/test_helpers.py +++ /dev/null @@ -1,77 +0,0 @@ -from unittest import TestCase - -from sqlalchemy.exc import IntegrityError - -from datagateway_api.common.exceptions import ( - BadRequestError, - FilterError, - MissingRecordError, -) -from datagateway_api.common.helpers import queries_records - - -class TestQueriesRecords(TestCase): - def test_missing_record_error(self): - @queries_records - def raise_missing_record(): - raise MissingRecordError() - - with self.assertRaises(MissingRecordError) as ctx: - raise_missing_record() - self.assertEqual("No such record in table", str(ctx.exception)) - self.assertEqual(404, ctx.exception.status_code) - - def test_bad_filter_error(self): - @queries_records - def raise_bad_filter_error(): - raise FilterError() - - with self.assertRaises(FilterError) as ctx: - raise_bad_filter_error() - - self.assertEqual("Invalid filter requested", str(ctx.exception)) - self.assertEqual(400, ctx.exception.status_code) - - def test_value_error(self): - @queries_records - def raise_value_error(): - raise ValueError() - - with self.assertRaises(BadRequestError) as ctx: - raise_value_error() - - self.assertEqual("Bad request", str(ctx.exception)) - self.assertEqual(400, ctx.exception.status_code) - - def test_type_error(self): - @queries_records - def raise_type_error(): - raise TypeError() - - with self.assertRaises(BadRequestError) as ctx: - raise_type_error() - - self.assertEqual("Bad request", str(ctx.exception)) - self.assertEqual(400, ctx.exception.status_code) - - def test_integrity_error(self): - @queries_records - def raise_integrity_error(): - raise IntegrityError() - - with self.assertRaises(BadRequestError) as ctx: - raise_integrity_error() - - self.assertEqual("Bad request", str(ctx.exception)) - self.assertEqual(400, ctx.exception.status_code) - - def test_bad_request_error(self): - @queries_records - def raise_bad_request_error(): - raise BadRequestError() - - with self.assertRaises(BadRequestError) as ctx: - raise_bad_request_error() - - self.assertEqual("Bad request", str(ctx.exception)) - self.assertEqual(400, ctx.exception.status_code) diff --git a/test/test_queries_records.py b/test/test_queries_records.py new file mode 100644 index 00000000..6879df45 --- /dev/null +++ b/test/test_queries_records.py @@ -0,0 +1,36 @@ +import pytest +from sqlalchemy.exc import IntegrityError + +from datagateway_api.common.exceptions import ( + BadRequestError, + FilterError, + MissingRecordError, +) +from datagateway_api.common.helpers import queries_records + + +class TestQueriesRecords: + @pytest.mark.parametrize( + "raised_exception, expected_exception, status_code", + [ + pytest.param(BadRequestError, BadRequestError, 400, id="bad request error"), + pytest.param(IntegrityError, BadRequestError, 400, id="integrity error"), + pytest.param(FilterError, FilterError, 400, id="invalid filter"), + pytest.param( + MissingRecordError, MissingRecordError, 404, id="missing record", + ), + pytest.param(TypeError, BadRequestError, 400, id="type error"), + pytest.param(ValueError, BadRequestError, 400, id="value error"), + ], + ) + def test_valid_error_raised( + self, raised_exception, expected_exception, status_code, + ): + @queries_records + def raise_exception(): + raise raised_exception() + + with pytest.raises(expected_exception) as ctx: + raise_exception() + + assert ctx.exception.status_code == status_code From f5c4558f084ac3f4fe9b02a43b5d885480831ff9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 10:51:19 +0000 Subject: [PATCH 086/109] #150: Add test to ensure all required abstract methods are present for Backend --- test/test_backends.py | 56 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/test_backends.py b/test/test_backends.py index 30b2f48d..dc79e7c9 100644 --- a/test/test_backends.py +++ b/test/test_backends.py @@ -1,5 +1,6 @@ import pytest +from datagateway_api.common.backend import Backend from datagateway_api.common.backends import create_backend from datagateway_api.common.database.backend import DatabaseBackend from datagateway_api.common.icat.backend import PythonICATBackend @@ -17,3 +18,58 @@ def test_backend_creation(self, backend_name, backend_type): test_backend = create_backend(backend_name) assert type(test_backend) == backend_type + + def test_abstract_class(self): + """Test the `Backend` abstract class has all the required classes for the API""" + Backend.__abstractmethods__ = set() + + class DummyBackend(Backend): + pass + + d = DummyBackend() + + credentials = "credentials" + session_id = "session_id" + entity_type = "entity_type" + filters = "filters" + data = "data" + instrument_id = "instrument_id" + facilitycycle_id = "facilitycycle_id" + id_ = "id_" + + assert d.login(credentials) is None + assert d.get_session_details(session_id) is None + assert d.refresh(session_id) is None + assert d.logout(session_id) is None + assert d.get_with_filters(session_id, entity_type, filters) is None + assert d.create(session_id, entity_type, data) is None + assert d.update(session_id, entity_type, data) is None + assert d.get_one_with_filters(session_id, entity_type, filters) is None + assert d.count_with_filters(session_id, entity_type, filters) is None + assert d.get_with_id(session_id, entity_type, id_) is None + assert d.delete_with_id(session_id, entity_type, id_) is None + assert d.update_with_id(session_id, entity_type, id_, data) is None + assert ( + d.get_facility_cycles_for_instrument_with_filters( + session_id, instrument_id, filters, + ) + is None + ) + assert ( + d.get_facility_cycles_for_instrument_count_with_filters( + session_id, instrument_id, filters, + ) + is None + ) + assert ( + d.get_investigations_for_instrument_facility_cycle_with_filters( + session_id, instrument_id, facilitycycle_id, filters, + ) + is None + ) + assert ( + d.get_investigation_count_instrument_facility_cycle_with_filters( + session_id, instrument_id, facilitycycle_id, filters, + ) + is None + ) From b6c7c66cd53cb73ddb220141ace3ee5a27e4d92e Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 14:03:46 +0000 Subject: [PATCH 087/109] #150: Add get by ID tests for DB backend --- test/conftest.py | 29 +++++++++++++++++++- test/db/endpoints/test_get_by_id.py | 38 +++++++++++++++++++++++++++ test/icat/endpoints/test_get_by_id.py | 2 +- 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 test/db/endpoints/test_get_by_id.py diff --git a/test/conftest.py b/test/conftest.py index ad70c0a0..86e995dc 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -12,7 +12,7 @@ delete_row_by_id, insert_row_into_table, ) -from datagateway_api.common.database.models import SESSION +from datagateway_api.common.database.models import INVESTIGATION, SESSION from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure from test.icat.test_query import prepare_icat_data_for_assertion @@ -103,6 +103,33 @@ def single_investigation_test_data(icat_client): print(e) +@pytest.fixture() +def single_investigation_test_data_db(): + investigation = INVESTIGATION() + investigation.NAME = "Test Data for DataGateway API Testing (DB)" + investigation.TITLE = "Title for DataGateway API Testing (DB)" + investigation.STARTDATE = datetime( + year=2020, month=1, day=4, hour=1, minute=1, second=1, + ) + investigation.ENDDATE = datetime( + year=2020, month=1, day=8, hour=1, minute=1, second=1, + ) + investigation.VISIT_ID = str(uuid.uuid1()) + investigation.FACILITY_ID = 1 + investigation.TYPE_ID = 1 + + investigation.CREATE_TIME = datetime(2000, 1, 1) + investigation.MOD_TIME = datetime(2000, 1, 1) + investigation.CREATE_ID = "test create id" + investigation.MOD_ID = "test mod id" + + insert_row_into_table(INVESTIGATION, investigation) + + yield investigation + + delete_row_by_id(INVESTIGATION, investigation.ID) + + @pytest.fixture() def multiple_investigation_test_data(icat_client): investigation_dicts = [] diff --git a/test/db/endpoints/test_get_by_id.py b/test/db/endpoints/test_get_by_id.py new file mode 100644 index 00000000..8bb4ded5 --- /dev/null +++ b/test/db/endpoints/test_get_by_id.py @@ -0,0 +1,38 @@ +class TestDBGetByID: + def test_valid_get_with_id( + self, + flask_test_app_db, + valid_db_credentials_header, + single_investigation_test_data_db, + ): + # Need to identify the ID given to the test data + investigation_data = flask_test_app_db.get( + '/investigations?where={"TITLE": {"like": "Title for DataGateway API' + ' Testing (DB)"}}', + headers=valid_db_credentials_header, + ) + print(investigation_data.json) + test_data_id = investigation_data.json[0]["ID"] + + test_response = flask_test_app_db.get( + f"/investigations/{test_data_id}", headers=valid_db_credentials_header, + ) + + assert test_response.json == single_investigation_test_data_db.to_dict() + + def test_invalid_get_with_id( + self, flask_test_app_db, valid_db_credentials_header, + ): + final_investigation_result = flask_test_app_db.get( + '/investigations/findone?order="ID DESC"', + headers=valid_db_credentials_header, + ) + test_data_id = final_investigation_result.json["ID"] + + # Adding 100 onto the ID to the most recent result should ensure a 404 + test_response = flask_test_app_db.get( + f"/investigations/{test_data_id + 100}", + headers=valid_db_credentials_header, + ) + + assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id.py index 5075dee0..c57ea838 100644 --- a/test/icat/endpoints/test_get_by_id.py +++ b/test/icat/endpoints/test_get_by_id.py @@ -1,7 +1,7 @@ from test.icat.test_query import prepare_icat_data_for_assertion -class TestGetByID: +class TestICATGetByID: def test_valid_get_with_id( self, flask_test_app_icat, From bbca31cd3291f6433baf10c6c5c6372a6204faac Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 14:10:17 +0000 Subject: [PATCH 088/109] #150: Add tests for /count for DB backend --- test/db/endpoints/test_count_with_filters.py | 26 +++++++++++++++++++ .../icat/endpoints/test_count_with_filters.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 test/db/endpoints/test_count_with_filters.py diff --git a/test/db/endpoints/test_count_with_filters.py b/test/db/endpoints/test_count_with_filters.py new file mode 100644 index 00000000..0dd0795d --- /dev/null +++ b/test/db/endpoints/test_count_with_filters.py @@ -0,0 +1,26 @@ +import pytest + + +class TestDBCountWithFilters: + @pytest.mark.usefixtures("single_investigation_test_data_db") + def test_valid_count_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + test_response = flask_test_app_db.get( + '/investigations/count?where={"TITLE": {"like": "Title for DataGateway API' + ' Testing (DB)"}}', + headers=valid_db_credentials_header, + ) + + assert test_response.json == 1 + + def test_valid_no_results_count_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + test_response = flask_test_app_db.get( + '/investigations/count?where={"TITLE": {"like": "This filter should cause a' + '404 for testing purposes..."}}', + headers=valid_db_credentials_header, + ) + + assert test_response.json == 0 diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters.py index 4b667edd..3f4d09c1 100644 --- a/test/icat/endpoints/test_count_with_filters.py +++ b/test/icat/endpoints/test_count_with_filters.py @@ -1,7 +1,7 @@ import pytest -class TestCountWithFilters: +class TestICATCountWithFilters: @pytest.mark.usefixtures("single_investigation_test_data") def test_valid_count_with_filters( self, flask_test_app_icat, valid_icat_credentials_header, From 6baf55dd797f5811bef65dd4300b88296f3cef55 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 14:11:34 +0000 Subject: [PATCH 089/109] #150: Move endpoint rules tests - This aren't backend-specific --- test/{icat/endpoints => }/test_endpoint_rules.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{icat/endpoints => }/test_endpoint_rules.py (100%) diff --git a/test/icat/endpoints/test_endpoint_rules.py b/test/test_endpoint_rules.py similarity index 100% rename from test/icat/endpoints/test_endpoint_rules.py rename to test/test_endpoint_rules.py From 578f897268e02f9ad619df4d3b6c8612300d5653 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 14:26:46 +0000 Subject: [PATCH 090/109] #150: Ensure filenames are unique across test suite - pytest requires that all filenames are unique, even if they're stored in different directories --- .../{test_count_with_filters.py => test_count_with_filters_db.py} | 0 test/db/endpoints/{test_get_by_id.py => test_get_by_id_db.py} | 0 ...test_count_with_filters.py => test_count_with_filters_icat.py} | 0 test/icat/endpoints/{test_create.py => test_create_icat.py} | 0 .../endpoints/{test_delete_by_id.py => test_delete_by_id_icat.py} | 0 test/icat/endpoints/{test_findone.py => test_findone_icat.py} | 0 test/icat/endpoints/{test_get_by_id.py => test_get_by_id_icat.py} | 0 .../{test_get_with_filters.py => test_get_with_filters_icat.py} | 0 .../{test_table_endpoints.py => test_table_endpoints_icat.py} | 0 .../endpoints/{test_update_by_id.py => test_update_by_id_icat.py} | 0 .../{test_update_multiple.py => test_update_multiple_icat.py} | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename test/db/endpoints/{test_count_with_filters.py => test_count_with_filters_db.py} (100%) rename test/db/endpoints/{test_get_by_id.py => test_get_by_id_db.py} (100%) rename test/icat/endpoints/{test_count_with_filters.py => test_count_with_filters_icat.py} (100%) rename test/icat/endpoints/{test_create.py => test_create_icat.py} (100%) rename test/icat/endpoints/{test_delete_by_id.py => test_delete_by_id_icat.py} (100%) rename test/icat/endpoints/{test_findone.py => test_findone_icat.py} (100%) rename test/icat/endpoints/{test_get_by_id.py => test_get_by_id_icat.py} (100%) rename test/icat/endpoints/{test_get_with_filters.py => test_get_with_filters_icat.py} (100%) rename test/icat/endpoints/{test_table_endpoints.py => test_table_endpoints_icat.py} (100%) rename test/icat/endpoints/{test_update_by_id.py => test_update_by_id_icat.py} (100%) rename test/icat/endpoints/{test_update_multiple.py => test_update_multiple_icat.py} (100%) diff --git a/test/db/endpoints/test_count_with_filters.py b/test/db/endpoints/test_count_with_filters_db.py similarity index 100% rename from test/db/endpoints/test_count_with_filters.py rename to test/db/endpoints/test_count_with_filters_db.py diff --git a/test/db/endpoints/test_get_by_id.py b/test/db/endpoints/test_get_by_id_db.py similarity index 100% rename from test/db/endpoints/test_get_by_id.py rename to test/db/endpoints/test_get_by_id_db.py diff --git a/test/icat/endpoints/test_count_with_filters.py b/test/icat/endpoints/test_count_with_filters_icat.py similarity index 100% rename from test/icat/endpoints/test_count_with_filters.py rename to test/icat/endpoints/test_count_with_filters_icat.py diff --git a/test/icat/endpoints/test_create.py b/test/icat/endpoints/test_create_icat.py similarity index 100% rename from test/icat/endpoints/test_create.py rename to test/icat/endpoints/test_create_icat.py diff --git a/test/icat/endpoints/test_delete_by_id.py b/test/icat/endpoints/test_delete_by_id_icat.py similarity index 100% rename from test/icat/endpoints/test_delete_by_id.py rename to test/icat/endpoints/test_delete_by_id_icat.py diff --git a/test/icat/endpoints/test_findone.py b/test/icat/endpoints/test_findone_icat.py similarity index 100% rename from test/icat/endpoints/test_findone.py rename to test/icat/endpoints/test_findone_icat.py diff --git a/test/icat/endpoints/test_get_by_id.py b/test/icat/endpoints/test_get_by_id_icat.py similarity index 100% rename from test/icat/endpoints/test_get_by_id.py rename to test/icat/endpoints/test_get_by_id_icat.py diff --git a/test/icat/endpoints/test_get_with_filters.py b/test/icat/endpoints/test_get_with_filters_icat.py similarity index 100% rename from test/icat/endpoints/test_get_with_filters.py rename to test/icat/endpoints/test_get_with_filters_icat.py diff --git a/test/icat/endpoints/test_table_endpoints.py b/test/icat/endpoints/test_table_endpoints_icat.py similarity index 100% rename from test/icat/endpoints/test_table_endpoints.py rename to test/icat/endpoints/test_table_endpoints_icat.py diff --git a/test/icat/endpoints/test_update_by_id.py b/test/icat/endpoints/test_update_by_id_icat.py similarity index 100% rename from test/icat/endpoints/test_update_by_id.py rename to test/icat/endpoints/test_update_by_id_icat.py diff --git a/test/icat/endpoints/test_update_multiple.py b/test/icat/endpoints/test_update_multiple_icat.py similarity index 100% rename from test/icat/endpoints/test_update_multiple.py rename to test/icat/endpoints/test_update_multiple_icat.py From 42359fc0ca156db134f7ae732f24f1488e94e20d Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 14:39:16 +0000 Subject: [PATCH 091/109] #150: Create localised conftest files - Local to each backend, with a shared file in test/ --- test/conftest.py | 181 +++--------------------------------------- test/db/conftest.py | 39 +++++++++ test/icat/conftest.py | 136 +++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 169 deletions(-) create mode 100644 test/db/conftest.py create mode 100644 test/icat/conftest.py diff --git a/test/conftest.py b/test/conftest.py index 86e995dc..9c83d3dd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,46 +1,14 @@ from datetime import datetime, timedelta -import uuid from flask import Flask -from icat.client import Client -from icat.exception import ICATNoObjectError -from icat.query import Query import pytest -from datagateway_api.common.config import config from datagateway_api.common.database.helpers import ( delete_row_by_id, insert_row_into_table, ) -from datagateway_api.common.database.models import INVESTIGATION, SESSION +from datagateway_api.common.database.models import SESSION from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure -from test.icat.test_query import prepare_icat_data_for_assertion - - -@pytest.fixture(scope="package") -def icat_client(): - client = Client(config.get_icat_url(), checkCert=config.get_icat_check_cert()) - client.login(config.get_test_mechanism(), config.get_test_user_credentials()) - return client - - -@pytest.fixture() -def valid_icat_credentials_header(icat_client): - return {"Authorization": f"Bearer {icat_client.sessionId}"} - - -@pytest.fixture() -def valid_db_credentials_header(): - session = SESSION() - session.ID = "Test" - session.EXPIREDATETIME = datetime.now() + timedelta(hours=1) - session.username = "Test User" - - insert_row_into_table(SESSION, session) - - yield {"Authorization": f"Bearer {session.ID}"} - - delete_row_by_id(SESSION, "Test") @pytest.fixture() @@ -54,94 +22,6 @@ def invalid_credentials_header(): return {"Authorization": "Test"} -@pytest.fixture() -def icat_query(icat_client): - return Query(icat_client, "Investigation") - - -def create_investigation_test_data(client, num_entities=1): - test_data = [] - - for i in range(num_entities): - investigation = client.new("investigation") - investigation.name = f"Test Data for DataGateway API Testing {i}" - investigation.title = ( - f"Test data for the Python ICAT Backend on DataGateway API {i}" - ) - investigation.startDate = datetime( - year=2020, month=1, day=4, hour=1, minute=1, second=1, - ) - investigation.endDate = datetime( - year=2020, month=1, day=8, hour=1, minute=1, second=1, - ) - # UUID visit ID means uniquesness constraint should always be met - investigation.visitId = str(uuid.uuid1()) - investigation.facility = client.get("Facility", 1) - investigation.type = client.get("InvestigationType", 1) - investigation.create() - - test_data.append(investigation) - - if len(test_data) == 1: - return test_data[0] - else: - return test_data - - -@pytest.fixture() -def single_investigation_test_data(icat_client): - investigation = create_investigation_test_data(icat_client) - investigation_dict = prepare_icat_data_for_assertion([investigation]) - - yield investigation_dict - - # Remove data from ICAT - try: - icat_client.delete(investigation) - except ICATNoObjectError as e: - # This should occur on DELETE endpoints, normal behaviour for those tests - print(e) - - -@pytest.fixture() -def single_investigation_test_data_db(): - investigation = INVESTIGATION() - investigation.NAME = "Test Data for DataGateway API Testing (DB)" - investigation.TITLE = "Title for DataGateway API Testing (DB)" - investigation.STARTDATE = datetime( - year=2020, month=1, day=4, hour=1, minute=1, second=1, - ) - investigation.ENDDATE = datetime( - year=2020, month=1, day=8, hour=1, minute=1, second=1, - ) - investigation.VISIT_ID = str(uuid.uuid1()) - investigation.FACILITY_ID = 1 - investigation.TYPE_ID = 1 - - investigation.CREATE_TIME = datetime(2000, 1, 1) - investigation.MOD_TIME = datetime(2000, 1, 1) - investigation.CREATE_ID = "test create id" - investigation.MOD_ID = "test mod id" - - insert_row_into_table(INVESTIGATION, investigation) - - yield investigation - - delete_row_by_id(INVESTIGATION, investigation.ID) - - -@pytest.fixture() -def multiple_investigation_test_data(icat_client): - investigation_dicts = [] - investigations = create_investigation_test_data(icat_client, num_entities=5) - investigation_dicts = prepare_icat_data_for_assertion(investigations) - - yield investigation_dicts - - for investigation in investigations: - icat_client.delete(investigation) - - @pytest.fixture(scope="package") def flask_test_app(): """This is used to check the endpoints exist and have the correct HTTP methods""" @@ -152,18 +32,6 @@ def flask_test_app(): yield test_app -@pytest.fixture(scope="package") -def flask_test_app_icat(flask_test_app): - icat_app = Flask(__name__) - icat_app.config["TESTING"] = True - icat_app.config["TEST_BACKEND"] = "python_icat" - - api, spec = create_app_infrastructure(icat_app) - create_api_endpoints(icat_app, api, spec) - - yield icat_app.test_client() - - @pytest.fixture(scope="package") def flask_test_app_db(): db_app = Flask(__name__) @@ -177,39 +45,14 @@ def flask_test_app_db(): @pytest.fixture() -def isis_specific_endpoint_data(icat_client): - facility_cycle = icat_client.new("facilityCycle") - facility_cycle.name = "Test cycle for DataGateway API testing" - facility_cycle.startDate = datetime( - year=2020, month=1, day=1, hour=1, minute=1, second=1, - ) - facility_cycle.endDate = datetime( - year=2020, month=2, day=1, hour=1, minute=1, second=1, - ) - facility_cycle.facility = icat_client.get("Facility", 1) - facility_cycle.create() - - investigation = create_investigation_test_data(icat_client) - investigation_dict = prepare_icat_data_for_assertion([investigation]) - - instrument = icat_client.new("instrument") - instrument.name = "Test Instrument for DataGateway API Endpoint Testing" - instrument.facility = icat_client.get("Facility", 1) - instrument.create() - - investigation_instrument = icat_client.new("investigationInstrument") - investigation_instrument.investigation = investigation - investigation_instrument.instrument = instrument - investigation_instrument.create() - - facility_cycle_dict = prepare_icat_data_for_assertion([facility_cycle]) - - yield (instrument.id, facility_cycle_dict, facility_cycle.id, investigation_dict) - - try: - # investigation_instrument removed when deleting the objects its related objects - icat_client.delete(facility_cycle) - icat_client.delete(investigation) - icat_client.delete(instrument) - except ICATNoObjectError as e: - print(e) +def valid_db_credentials_header(): + session = SESSION() + session.ID = "Test" + session.EXPIREDATETIME = datetime.now() + timedelta(hours=1) + session.username = "Test User" + + insert_row_into_table(SESSION, session) + + yield {"Authorization": f"Bearer {session.ID}"} + + delete_row_by_id(SESSION, "Test") diff --git a/test/db/conftest.py b/test/db/conftest.py new file mode 100644 index 00000000..8f7cfd12 --- /dev/null +++ b/test/db/conftest.py @@ -0,0 +1,39 @@ +from datetime import datetime, timedelta +import uuid + +from flask import Flask +import pytest + +from datagateway_api.common.database.helpers import ( + delete_row_by_id, + insert_row_into_table, +) +from datagateway_api.common.database.models import INVESTIGATION, SESSION +from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure + + +@pytest.fixture() +def single_investigation_test_data_db(): + investigation = INVESTIGATION() + investigation.NAME = "Test Data for DataGateway API Testing (DB)" + investigation.TITLE = "Title for DataGateway API Testing (DB)" + investigation.STARTDATE = datetime( + year=2020, month=1, day=4, hour=1, minute=1, second=1, + ) + investigation.ENDDATE = datetime( + year=2020, month=1, day=8, hour=1, minute=1, second=1, + ) + investigation.VISIT_ID = str(uuid.uuid1()) + investigation.FACILITY_ID = 1 + investigation.TYPE_ID = 1 + + investigation.CREATE_TIME = datetime(2000, 1, 1) + investigation.MOD_TIME = datetime(2000, 1, 1) + investigation.CREATE_ID = "test create id" + investigation.MOD_ID = "test mod id" + + insert_row_into_table(INVESTIGATION, investigation) + + yield investigation + + delete_row_by_id(INVESTIGATION, investigation.ID) diff --git a/test/icat/conftest.py b/test/icat/conftest.py new file mode 100644 index 00000000..38bf6b01 --- /dev/null +++ b/test/icat/conftest.py @@ -0,0 +1,136 @@ +from datetime import datetime +import uuid + +from flask import Flask +from icat.client import Client +from icat.exception import ICATNoObjectError +from icat.query import Query +import pytest + +from datagateway_api.common.config import config +from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure +from test.icat.test_query import prepare_icat_data_for_assertion + + +@pytest.fixture(scope="package") +def icat_client(): + client = Client(config.get_icat_url(), checkCert=config.get_icat_check_cert()) + client.login(config.get_test_mechanism(), config.get_test_user_credentials()) + return client + + +@pytest.fixture() +def valid_icat_credentials_header(icat_client): + return {"Authorization": f"Bearer {icat_client.sessionId}"} + + +@pytest.fixture() +def icat_query(icat_client): + return Query(icat_client, "Investigation") + + +def create_investigation_test_data(client, num_entities=1): + test_data = [] + + for i in range(num_entities): + investigation = client.new("investigation") + investigation.name = f"Test Data for DataGateway API Testing {i}" + investigation.title = ( + f"Test data for the Python ICAT Backend on DataGateway API {i}" + ) + investigation.startDate = datetime( + year=2020, month=1, day=4, hour=1, minute=1, second=1, + ) + investigation.endDate = datetime( + year=2020, month=1, day=8, hour=1, minute=1, second=1, + ) + # UUID visit ID means uniquesness constraint should always be met + investigation.visitId = str(uuid.uuid1()) + investigation.facility = client.get("Facility", 1) + investigation.type = client.get("InvestigationType", 1) + investigation.create() + + test_data.append(investigation) + + if len(test_data) == 1: + return test_data[0] + else: + return test_data + + +@pytest.fixture() +def single_investigation_test_data(icat_client): + investigation = create_investigation_test_data(icat_client) + investigation_dict = prepare_icat_data_for_assertion([investigation]) + + yield investigation_dict + + # Remove data from ICAT + try: + icat_client.delete(investigation) + except ICATNoObjectError as e: + # This should occur on DELETE endpoints, normal behaviour for those tests + print(e) + + +@pytest.fixture() +def multiple_investigation_test_data(icat_client): + investigation_dicts = [] + investigations = create_investigation_test_data(icat_client, num_entities=5) + investigation_dicts = prepare_icat_data_for_assertion(investigations) + + yield investigation_dicts + + for investigation in investigations: + icat_client.delete(investigation) + + +@pytest.fixture(scope="package") +def flask_test_app_icat(flask_test_app): + icat_app = Flask(__name__) + icat_app.config["TESTING"] = True + icat_app.config["TEST_BACKEND"] = "python_icat" + + api, spec = create_app_infrastructure(icat_app) + create_api_endpoints(icat_app, api, spec) + + yield icat_app.test_client() + + +@pytest.fixture() +def isis_specific_endpoint_data(icat_client): + facility_cycle = icat_client.new("facilityCycle") + facility_cycle.name = "Test cycle for DataGateway API testing" + facility_cycle.startDate = datetime( + year=2020, month=1, day=1, hour=1, minute=1, second=1, + ) + facility_cycle.endDate = datetime( + year=2020, month=2, day=1, hour=1, minute=1, second=1, + ) + facility_cycle.facility = icat_client.get("Facility", 1) + facility_cycle.create() + + investigation = create_investigation_test_data(icat_client) + investigation_dict = prepare_icat_data_for_assertion([investigation]) + + instrument = icat_client.new("instrument") + instrument.name = "Test Instrument for DataGateway API Endpoint Testing" + instrument.facility = icat_client.get("Facility", 1) + instrument.create() + + investigation_instrument = icat_client.new("investigationInstrument") + investigation_instrument.investigation = investigation + investigation_instrument.instrument = instrument + investigation_instrument.create() + + facility_cycle_dict = prepare_icat_data_for_assertion([facility_cycle]) + + yield (instrument.id, facility_cycle_dict, facility_cycle.id, investigation_dict) + + try: + # investigation_instrument removed when deleting the objects its related objects + icat_client.delete(facility_cycle) + icat_client.delete(investigation) + icat_client.delete(instrument) + except ICATNoObjectError as e: + print(e) From b2af27dabe656b392aeb936d7abed7a7f3dfa745 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 14:58:28 +0000 Subject: [PATCH 092/109] #150: Add remaining data creation fixtures for DB backend tests - These fixtures haven't been tested, they will when I write the remaining endpoint tests for the DB backend --- test/db/conftest.py | 112 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 20 deletions(-) diff --git a/test/db/conftest.py b/test/db/conftest.py index 8f7cfd12..6f8fc208 100644 --- a/test/db/conftest.py +++ b/test/db/conftest.py @@ -1,39 +1,111 @@ -from datetime import datetime, timedelta +from datetime import datetime import uuid -from flask import Flask import pytest from datagateway_api.common.database.helpers import ( delete_row_by_id, insert_row_into_table, ) -from datagateway_api.common.database.models import INVESTIGATION, SESSION -from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure +from datagateway_api.common.database.models import ( + FACILITYCYCLE, + INSTRUMENT, + INVESTIGATION, + INVESTIGATIONINSTRUMENT, +) + + +def set_meta_attributes(entity): + db_meta_attributes = { + "CREATE_TIME": datetime(2000, 1, 1), + "MOD_TIME": datetime(2000, 1, 1), + "CREATE_ID": "test create id", + "MOD_ID": "test mod id", + } + + for attr, value in db_meta_attributes.items(): + setattr(entity, attr, value) + + +def create_investigation_db_data(num_entities=1): + test_data = [] + + for i in range(num_entities): + investigation = INVESTIGATION() + investigation.NAME = f"Test Data for DataGateway API Testing (DB) {i}" + investigation.TITLE = f"Title for DataGateway API Testing (DB) {i}" + investigation.STARTDATE = datetime( + year=2020, month=1, day=4, hour=1, minute=1, second=1, + ) + investigation.ENDDATE = datetime( + year=2020, month=1, day=8, hour=1, minute=1, second=1, + ) + investigation.VISIT_ID = str(uuid.uuid1()) + investigation.FACILITY_ID = 1 + investigation.TYPE_ID = 1 + + set_meta_attributes(investigation) + + insert_row_into_table(INVESTIGATION, investigation) + + test_data.append(investigation) + + if len(test_data) == 1: + return test_data[0] + else: + return test_data @pytest.fixture() def single_investigation_test_data_db(): - investigation = INVESTIGATION() - investigation.NAME = "Test Data for DataGateway API Testing (DB)" - investigation.TITLE = "Title for DataGateway API Testing (DB)" - investigation.STARTDATE = datetime( - year=2020, month=1, day=4, hour=1, minute=1, second=1, + investigation = create_investigation_db_data() + + yield investigation + + delete_row_by_id(INVESTIGATION, investigation.ID) + + +@pytest.fixture() +def multiple_investigation_test_data_db(): + investigations = create_investigation_db_data(num_entities=5) + + yield investigations + + for investigation in investigations: + delete_row_by_id(INVESTIGATION, investigation.ID) + + +@pytest.fixture() +def isis_specific_endpoint_data_db(): + facility_cycle = FACILITYCYCLE() + facility_cycle.NAME = "Test cycle for DG API testing (DB)" + facility_cycle.STARTDATE = datetime( + year=2020, month=1, day=1, hour=1, minute=1, second=1, ) - investigation.ENDDATE = datetime( - year=2020, month=1, day=8, hour=1, minute=1, second=1, + facility_cycle.ENDDATE = datetime( + year=2020, month=2, day=1, hour=1, minute=1, second=1, ) - investigation.VISIT_ID = str(uuid.uuid1()) - investigation.FACILITY_ID = 1 - investigation.TYPE_ID = 1 + facility_cycle.FACILITY_ID = 1 + set_meta_attributes(facility_cycle) - investigation.CREATE_TIME = datetime(2000, 1, 1) - investigation.MOD_TIME = datetime(2000, 1, 1) - investigation.CREATE_ID = "test create id" - investigation.MOD_ID = "test mod id" + investigation = create_investigation_db_data() - insert_row_into_table(INVESTIGATION, investigation) + instrument = INSTRUMENT() + instrument.NAME = "Test Instrument for DataGateway API Endpoint Testing (DB)" + instrument.FACILITY_ID = 1 + set_meta_attributes(instrument) - yield investigation + insert_row_into_table(INSTRUMENT, instrument) + + investigation_instrument = INVESTIGATIONINSTRUMENT() + investigation_instrument.INVESTIGATION_ID = investigation.ID + investigation_instrument.INSTRUMENT_ID = instrument.ID + set_meta_attributes(investigation_instrument) + + insert_row_into_table(INVESTIGATIONINSTRUMENT, investigation_instrument) + + yield (instrument.ID, facility_cycle, investigation) + delete_row_by_id(FACILITYCYCLE, facility_cycle.ID) delete_row_by_id(INVESTIGATION, investigation.ID) + delete_row_by_id(INSTRUMENT, instrument.ID) From 1b9121749a3e5cb2d7ee5fd900fbb72bfc6d4a5b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 4 Dec 2020 16:14:24 +0000 Subject: [PATCH 093/109] #150: Add get with filter tests for DB backend --- test/db/endpoints/test_get_with_filters.py | 74 +++++++++++++++++++ test/icat/endpoints/test_create_icat.py | 2 +- .../endpoints/test_get_with_filters_icat.py | 2 +- 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 test/db/endpoints/test_get_with_filters.py diff --git a/test/db/endpoints/test_get_with_filters.py b/test/db/endpoints/test_get_with_filters.py new file mode 100644 index 00000000..1d0d7018 --- /dev/null +++ b/test/db/endpoints/test_get_with_filters.py @@ -0,0 +1,74 @@ +import pytest + + +class TestDBGetWithFilters: + def test_valid_get_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + single_investigation_test_data_db, + ): + test_response = flask_test_app_db.get( + '/investigations?where={"TITLE": {"like": "Title for DataGateway API' + ' Testing (DB)"}}', + headers=valid_db_credentials_header, + ) + + assert test_response.json == [single_investigation_test_data_db.to_dict()] + + def test_valid_no_results_get_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + test_response = flask_test_app_db.get( + '/investigations?where={"TITLE": {"eq": "This filter should cause a 404 for' + 'testing purposes..."}}', + headers=valid_db_credentials_header, + ) + print(test_response.json) + + assert test_response.json == [] + + @pytest.mark.usefixtures("multiple_investigation_test_data_db") + def test_valid_get_with_filters_distinct( + self, flask_test_app_db, valid_db_credentials_header, + ): + test_response = flask_test_app_db.get( + '/investigations?where={"TITLE": {"like": "Title for DataGateway API' + ' Testing (DB)"}}&distinct="TITLE"', + headers=valid_db_credentials_header, + ) + + expected = [ + {"TITLE": f"Title for DataGateway API Testing (DB) {i}" for i in range(5)}, + ] + + for title in expected: + assert title in test_response.json + + def test_limit_skip_merge_get_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + multiple_investigation_test_data_db, + ): + skip_value = 1 + limit_value = 2 + + test_response = flask_test_app_db.get( + '/investigations?where={"TITLE": {"like": "Title for DataGateway API' + ' Testing (DB)"}}' + f'&skip={skip_value}&limit={limit_value}&order="ID ASC"', + headers=valid_db_credentials_header, + ) + + # Copy required to ensure data is deleted at the end of the test + investigation_test_data_copy = multiple_investigation_test_data_db.copy() + filtered_investigation_data = [] + filter_count = 0 + while filter_count < limit_value: + filtered_investigation_data.append( + investigation_test_data_copy.pop(skip_value).to_dict(), + ) + filter_count += 1 + + assert test_response.json == filtered_investigation_data diff --git a/test/icat/endpoints/test_create_icat.py b/test/icat/endpoints/test_create_icat.py index fd6e607e..b00d0bb8 100644 --- a/test/icat/endpoints/test_create_icat.py +++ b/test/icat/endpoints/test_create_icat.py @@ -1,7 +1,7 @@ from test.icat.test_query import prepare_icat_data_for_assertion -class TestCreateData: +class TestICATCreateData: def test_valid_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): diff --git a/test/icat/endpoints/test_get_with_filters_icat.py b/test/icat/endpoints/test_get_with_filters_icat.py index d8f67f39..900fa73d 100644 --- a/test/icat/endpoints/test_get_with_filters_icat.py +++ b/test/icat/endpoints/test_get_with_filters_icat.py @@ -3,7 +3,7 @@ from test.icat.test_query import prepare_icat_data_for_assertion -class TestGetWithFilters: +class TestICATGetWithFilters: def test_valid_get_with_filters( self, flask_test_app_icat, From 8c1941dfbda5d77092a19a61c2e14a2a300f5786 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 7 Dec 2020 11:32:44 +0000 Subject: [PATCH 094/109] #150: Fix openapi YAML creation - I've reverted the setup for the ISIS and session endpoints back to way it was done at the start of this branch, there's no need to parse backend now the test backend is set in the config object, just create it locally on each endpoint file --- datagateway_api/src/main.py | 42 +- .../non_entities/sessions_endpoints.py | 281 +++++------ .../table_endpoints/table_endpoints.py | 472 ++++++++---------- 3 files changed, 357 insertions(+), 438 deletions(-) diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 47ab71f1..7670b22f 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -19,14 +19,12 @@ get_id_endpoint, ) from datagateway_api.src.resources.entities.entity_map import endpoints -from datagateway_api.src.resources.non_entities.sessions_endpoints import ( - session_endpoints, -) +from datagateway_api.src.resources.non_entities.sessions_endpoints import Sessions from datagateway_api.src.resources.table_endpoints.table_endpoints import ( - count_instrument_facility_cycles_endpoint, - count_instrument_investigation_endpoint, - instrument_facility_cycles_endpoint, - instrument_investigation_endpoint, + InstrumentsFacilityCycles, + InstrumentsFacilityCyclesCount, + InstrumentsFacilityCyclesInvestigations, + InstrumentsFacilityCyclesInvestigationsCount, ) from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin from datagateway_api.src.swagger.initialise_spec import initialise_spec @@ -103,42 +101,32 @@ def create_api_endpoints(flask_app, api, spec): spec.path(resource=get_find_one_endpoint_resource, api=api) # Session endpoint - session_endpoint_resource = session_endpoints(backend) - api.add_resource(session_endpoint_resource, "/sessions") - # spec.path(resource=session_endpoint_resource, api=api) + api.add_resource(Sessions, "/sessions") + spec.path(resource=Sessions, api=api) # Table specific endpoints - instrument_facility_cycle_resource = instrument_facility_cycles_endpoint(backend) api.add_resource( - instrument_facility_cycle_resource, "/instruments//facilitycycles", + InstrumentsFacilityCycles, "/instruments//facilitycycles", ) - # spec.path(resource=instrument_facility_cycle_resource, api=api) + spec.path(resource=InstrumentsFacilityCycles, api=api) - count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( - backend, - ) api.add_resource( - count_instrument_facility_cycle_res, - "/instruments//facilitycycles/count", + InstrumentsFacilityCyclesCount, "/instruments//facilitycycles/count", ) - # spec.path(resource=count_instrument_facility_cycle_resource, api=api) + spec.path(resource=InstrumentsFacilityCyclesCount, api=api) - instrument_investigation_resource = instrument_investigation_endpoint(backend) api.add_resource( - instrument_investigation_resource, + InstrumentsFacilityCyclesInvestigations, "/instruments//facilitycycles//investigations", ) - # spec.path(resource=instrument_investigation_resource, api=api) + spec.path(resource=InstrumentsFacilityCyclesInvestigations, api=api) - count_instrument_investigation_resource = count_instrument_investigation_endpoint( - backend, - ) api.add_resource( - count_instrument_investigation_resource, + InstrumentsFacilityCyclesInvestigationsCount, "/instruments//facilitycycles//investigations" "/count", ) - # spec.path(resource=count_instrument_investigation_resource, api=api) + spec.path(resource=InstrumentsFacilityCyclesInvestigationsCount, api=api) def openapi_config(spec): diff --git a/datagateway_api/src/resources/non_entities/sessions_endpoints.py b/datagateway_api/src/resources/non_entities/sessions_endpoints.py index 28af931a..6049afab 100644 --- a/datagateway_api/src/resources/non_entities/sessions_endpoints.py +++ b/datagateway_api/src/resources/non_entities/sessions_endpoints.py @@ -3,163 +3,152 @@ from flask import request from flask_restful import Resource +from datagateway_api.common.config import config from datagateway_api.common.exceptions import AuthenticationError from datagateway_api.common.helpers import get_session_id_from_auth_header log = logging.getLogger() +backend = config.get_backend_type() -def session_endpoints(backend): - """ - Generate a flask_restful Resource class using the configured backend. In main.py - these generated classes are registered with the api e.g. - `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` - :param backend: The backend instance used for processing requests - :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` - :return: The generated session endpoint class - """ +class Sessions(Resource): + def post(self): + """ + Generates a sessionID if the user has correct credentials + :return: String - SessionID - class Sessions(Resource): - def post(self): - """ - Generates a sessionID if the user has correct credentials - :return: String - SessionID - --- - summary: Login - description: Generates a sessionID if the user has correct credentials - tags: - - Sessions - security: [] - requestBody: - description: User credentials to login with - required: true + --- + summary: Login + description: Generates a sessionID if the user has correct credentials + tags: + - Sessions + security: [] + requestBody: + description: User credentials to login with + required: true + content: + application/json: + schema: + type: object + properties: + username: + type: string + password: + type: string + mechanism: + type: string + responses: + 201: + description: Success - returns a session ID content: - application/json: + application/json: schema: - type: object - properties: - username: - type: string - password: - type: string - mechanism: - type: string - responses: - 201: - description: Success - returns a session ID - content: - application/json: - schema: - type: object - properties: - sessionID: - type: string - description: Session ID - example: xxxxxx-yyyyyyy-zzzzzz - 400: - description: Bad request. User credentials not provided in request body - 403: - description: Forbidden. User credentials were invalid - """ - if not ( - request.data - and "username" in request.json - and "password" in request.json - ): - return "Bad request", 400 - # If no mechanism is present in request body, default to simple - if not ("mechanism" in request.json): - request.json["mechanism"] = "simple" - try: - return ({"sessionID": backend.login(request.json)}, 201) - except AuthenticationError: - return ("Forbidden", 403) + type: object + properties: + sessionID: + type: string + description: Session ID + example: xxxxxx-yyyyyyy-zzzzzz + 400: + description: Bad request. User credentials not provided in request body + 403: + description: Forbidden. User credentials were invalid + """ + if not ( + request.data and "username" in request.json and "password" in request.json + ): + return "Bad request", 400 + # If no mechanism is present in request body, default to simple + if not ("mechanism" in request.json): + request.json["mechanism"] = "simple" + try: + return {"sessionID": backend.login(request.json)}, 201 + except AuthenticationError: + return "Forbidden", 403 - def delete(self): - """ - Deletes a users sessionID when they logout - :return: Blank response, 200 - --- - summary: Delete session - description: Deletes a users sessionID when they logout - tags: - - Sessions - responses: - 200: - description: Success - User's session was successfully deleted - 400: - description: Bad request - something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: Not Found - Unable to find session ID - """ - backend.logout(get_session_id_from_auth_header()) - return ("", 200) + def delete(self): + """ + Deletes a users sessionID when they logout + :return: Blank response, 200 + --- + summary: Delete session + description: Deletes a users sessionID when they logout + tags: + - Sessions + responses: + 200: + description: Success - User's session was successfully deleted + 400: + description: Bad request - something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: Not Found - Unable to find session ID + """ + backend.logout(get_session_id_from_auth_header()) + return "", 200 - def get(self): - """ - Gives details of a users session - :return: String: Details of the session, 200 - --- - summary: Get session details - description: Gives details of a user's session - tags: - - Sessions - responses: - 200: - description: Success - a user's session details - content: - application/json: - schema: - type: object - properties: - ID: - type: string - description: The session ID - example: xxxxxx-yyyyyyy-zzzzzz - EXPIREDATETIME: - type: string - format: datetime - description: When this session expires - example: "2017-07-21T17:32:28Z" - USERNAME: - type: string - description: Username associated with this session - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - """ - return (backend.get_session_details(get_session_id_from_auth_header()), 200) - - def put(self): - """ - Refreshes a users session - :return: String: The session ID that has been refreshed, 200 - --- - summary: Refresh session - description: Refreshes a users session - tags: - - Sessions - responses: - 200: - description: Success - the user's session ID that has been refreshed - content: - application/json: - schema: - type: string - description: Session ID - example: xxxxxx-yyyyyyy-zzzzzz - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - """ - return (backend.refresh(get_session_id_from_auth_header()), 200) + def get(self): + """ + Gives details of a users session + :return: String: Details of the session, 200 + --- + summary: Get session details + description: Gives details of a user's session + tags: + - Sessions + responses: + 200: + description: Success - a user's session details + content: + application/json: + schema: + type: object + properties: + ID: + type: string + description: The session ID + example: xxxxxx-yyyyyyy-zzzzzz + EXPIREDATETIME: + type: string + format: datetime + description: When this session expires + example: "2017-07-21T17:32:28Z" + USERNAME: + type: string + description: Username associated with this session + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + """ + return backend.get_session_details(get_session_id_from_auth_header()), 200 - return Sessions + def put(self): + """ + Refreshes a users session + :return: String: The session ID that has been refreshed, 200 + --- + summary: Refresh session + description: Refreshes a users session + tags: + - Sessions + responses: + 200: + description: Success - the user's session ID that has been refreshed + content: + application/json: + schema: + type: string + description: Session ID + example: xxxxxx-yyyyyyy-zzzzzz + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + """ + return backend.refresh(get_session_id_from_auth_header()), 200 diff --git a/datagateway_api/src/resources/table_endpoints/table_endpoints.py b/datagateway_api/src/resources/table_endpoints/table_endpoints.py index ad923ced..88637bcf 100644 --- a/datagateway_api/src/resources/table_endpoints/table_endpoints.py +++ b/datagateway_api/src/resources/table_endpoints/table_endpoints.py @@ -1,272 +1,214 @@ from flask_restful import Resource +from datagateway_api.common.config import config from datagateway_api.common.helpers import ( get_filters_from_query_string, get_session_id_from_auth_header, ) - -def instrument_facility_cycles_endpoint(backend): - """ - Generate a flask_restful Resource class using the configured backend. In main.py - these generated classes are registered with the api e.g. - `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` - - :param backend: The backend instance used for processing requests - :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` - :return: The generated endpoint class - """ - pass - - class InstrumentsFacilityCycles(Resource): - def get(self, id_): - """ - --- - summary: Get an Instrument's FacilityCycles - description: Given an Instrument id get facility cycles where the instrument - has investigations that occur within that cycle, subject to the given - filters - tags: - - FacilityCycles - parameters: - - in: path - required: true - name: id - description: The id of the instrument to retrieve the facility cycles of - schema: - type: integer - - WHERE_FILTER - - ORDER_FILTER - - LIMIT_FILTER - - SKIP_FILTER - - DISTINCT_FILTER - - INCLUDE_FILTER - responses: - 200: - description: Success - returns a list of the instrument's facility - cycles that satisfy the filters - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/FACILITYCYCLE' - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_facility_cycles_for_instrument_with_filters( - get_session_id_from_auth_header(), - id_, - get_filters_from_query_string(), - ), - 200, - ) - - return InstrumentsFacilityCycles - - -def count_instrument_facility_cycles_endpoint(backend): - """ - Generate a flask_restful Resource class using the configured backend. In main.py - these generated classes are registered with the api e.g. - `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` - - :param backend: The backend instance used for processing requests - :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` - :return: The generated endpoint class - """ - pass - - class InstrumentsFacilityCyclesCount(Resource): - def get(self, id_): - """ - --- - summary: Count an Instrument's FacilityCycles - description: Return the count of the Facility Cycles that have - investigations that occur within that cycle on the specified instrument - that would be retrieved given the filters provided - tags: - - FacilityCycles - parameters: - - in: path - required: true - name: id - description: The id of the instrument to count the facility cycles of - schema: - type: integer - - WHERE_FILTER - - DISTINCT_FILTER - responses: - 200: - description: Success - The count of the instrument's facility cycles - that satisfy the filters - content: - application/json: - schema: - type: integer - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_facility_cycles_for_instrument_count_with_filters( - get_session_id_from_auth_header(), - id_, - get_filters_from_query_string(), - ), - 200, - ) - - return InstrumentsFacilityCyclesCount - - -def instrument_investigation_endpoint(backend): - """ - Generate a flask_restful Resource class using the configured backend. In main.py - these generated classes are registered with the api e.g. - `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` - - :param backend: The backend instance used for processing requests - :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` - :return: The generated endpoint class - """ - pass - - class InstrumentsFacilityCyclesInvestigations(Resource): - def get(self, instrument_id, cycle_id): - """ - --- - summary: Get the investigations for a given Facility Cycle & Instrument - description: Given an Instrument id and Facility Cycle id, get the - investigations that occur within that cycle on that instrument, subject - to the given filters - tags: - - Investigations - parameters: - - in: path - required: true - name: instrument_id - description: The id of the instrument to retrieve the investigations of - schema: - type: integer - - in: path - required: true - name: cycle_id - description: The id of the facility cycle to retrieve the investigations - schema: - type: integer - - WHERE_FILTER - - ORDER_FILTER - - LIMIT_FILTER - - SKIP_FILTER - - DISTINCT_FILTER - - INCLUDE_FILTER - responses: - 200: - description: Success - returns a list of the investigations for the - given instrument and facility cycle that satisfy the filters - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/INVESTIGATION' - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_investigations_for_instrument_facility_cycle_with_filters( - get_session_id_from_auth_header(), - instrument_id, - cycle_id, - get_filters_from_query_string(), - ), - 200, - ) - - return InstrumentsFacilityCyclesInvestigations - - -def count_instrument_investigation_endpoint(backend): - """ - Generate a flask_restful Resource class using the configured backend. In main.py - these generated classes are registered with the api e.g. - `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` - - :param backend: The backend instance used for processing requests - :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` - :return: The generated endpoint class - """ - pass - - class InstrumentsFacilityCyclesInvestigationsCount(Resource): - def get(self, instrument_id, cycle_id): - """ - --- - summary: Count investigations for a given Facility Cycle & Instrument - description: Given an Instrument id and Facility Cycle id, get the number of - investigations that occur within that cycle on that instrument, subject - to the given filters - tags: - - Investigations - parameters: - - in: path - required: true - name: instrument_id - description: The id of the instrument to retrieve the investigations of - schema: - type: integer - - in: path - required: true - name: cycle_id - description: The id of the facility cycle to retrieve the investigations - schema: - type: integer - - WHERE_FILTER - - DISTINCT_FILTER - responses: - 200: - description: Success - The count of the investigations for the given - instrument and facility cycle that satisfy the filters - content: - application/json: - schema: - type: integer - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_investigation_count_instrument_facility_cycle_with_filters( - get_session_id_from_auth_header(), - instrument_id, - cycle_id, - get_filters_from_query_string(), - ), - 200, - ) - - return InstrumentsFacilityCyclesInvestigationsCount +backend = config.get_backend_type() + + +class InstrumentsFacilityCycles(Resource): + def get(self, id_): + """ + --- + summary: Get an Instrument's FacilityCycles + description: Given an Instrument id get facility cycles where the instrument has + investigations that occur within that cycle, subject to the given filters + tags: + - FacilityCycles + parameters: + - in: path + required: true + name: id + description: The id of the instrument to retrieve the facility cycles of + schema: + type: integer + - WHERE_FILTER + - ORDER_FILTER + - LIMIT_FILTER + - SKIP_FILTER + - DISTINCT_FILTER + - INCLUDE_FILTER + responses: + 200: + description: Success - returns a list of the instrument's facility + cycles that satisfy the filters + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/FACILITYCYCLE' + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_facility_cycles_for_instrument_with_filters( + get_session_id_from_auth_header(), id_, get_filters_from_query_string(), + ), + 200, + ) + + +class InstrumentsFacilityCyclesCount(Resource): + def get(self, id_): + """ + --- + summary: Count an Instrument's FacilityCycles + description: Return the count of the Facility Cycles that have investigations + that occur within that cycle on the specified instrument that would be + retrieved given the filters provided + tags: + - FacilityCycles + parameters: + - in: path + required: true + name: id + description: The id of the instrument to count the facility cycles of + schema: + type: integer + - WHERE_FILTER + - DISTINCT_FILTER + responses: + 200: + description: Success - The count of the instrument's facility cycles + that satisfy the filters + content: + application/json: + schema: + type: integer + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_facility_cycles_for_instrument_count_with_filters( + get_session_id_from_auth_header(), id_, get_filters_from_query_string(), + ), + 200, + ) + + +class InstrumentsFacilityCyclesInvestigations(Resource): + def get(self, instrument_id, cycle_id): + """ + --- + summary: Get the investigations for a given Facility Cycle & Instrument + description: Given an Instrument id and Facility Cycle id, get the + investigations that occur within that cycle on that instrument, subject to + the given filters + tags: + - Investigations + parameters: + - in: path + required: true + name: instrument_id + description: The id of the instrument to retrieve the investigations of + schema: + type: integer + - in: path + required: true + name: cycle_id + description: The id of the facility cycle to retrieve the investigations + schema: + type: integer + - WHERE_FILTER + - ORDER_FILTER + - LIMIT_FILTER + - SKIP_FILTER + - DISTINCT_FILTER + - INCLUDE_FILTER + responses: + 200: + description: Success - returns a list of the investigations for the + given instrument and facility cycle that satisfy the filters + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/INVESTIGATION' + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_investigations_for_instrument_facility_cycle_with_filters( + get_session_id_from_auth_header(), + instrument_id, + cycle_id, + get_filters_from_query_string(), + ), + 200, + ) + + +class InstrumentsFacilityCyclesInvestigationsCount(Resource): + def get(self, instrument_id, cycle_id): + """ + --- + summary: Count investigations for a given Facility Cycle & Instrument + description: Given an Instrument id and Facility Cycle id, get the number of + investigations that occur within that cycle on that instrument, subject to + the given filters + tags: + - Investigations + parameters: + - in: path + required: true + name: instrument_id + description: The id of the instrument to retrieve the investigations of + schema: + type: integer + - in: path + required: true + name: cycle_id + description: The id of the facility cycle to retrieve the investigations + schema: + type: integer + - WHERE_FILTER + - DISTINCT_FILTER + responses: + 200: + description: Success - The count of the investigations for the given + instrument and facility cycle that satisfy the filters + content: + application/json: + schema: + type: integer + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_investigation_count_instrument_facility_cycle_with_filters( + get_session_id_from_auth_header(), + instrument_id, + cycle_id, + get_filters_from_query_string(), + ), + 200, + ) From 3eec3464e9badaccd69981e5b6dd09c652a520fe Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 7 Dec 2020 12:01:36 +0000 Subject: [PATCH 095/109] #150: Add findone tests for DB backend - This commit also includes a fixes a bug found on the 404 of that test class --- datagateway_api/common/database/helpers.py | 6 +++++- test/db/endpoints/test_findone_db.py | 25 ++++++++++++++++++++++ test/icat/endpoints/test_findone_icat.py | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test/db/endpoints/test_findone_db.py diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index 8a208717..55dbb40f 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -329,7 +329,11 @@ def get_first_filtered_row(table, filters): :return: the first row matching the filter """ log.info(" Getting first filtered row for %s", table.__tablename__) - return get_rows_by_filter(table, filters)[0] + try: + result = get_rows_by_filter(table, filters)[0] + except IndexError: + raise MissingRecordError() + return result def get_filtered_row_count(table, filters): diff --git a/test/db/endpoints/test_findone_db.py b/test/db/endpoints/test_findone_db.py new file mode 100644 index 00000000..3da73274 --- /dev/null +++ b/test/db/endpoints/test_findone_db.py @@ -0,0 +1,25 @@ +class TestDBFindone: + def test_valid_findone_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + single_investigation_test_data_db, + ): + test_response = flask_test_app_db.get( + '/investigations/findone?where={"TITLE": {"like": "Title for DataGateway' + ' API Testing (DB)"}}', + headers=valid_db_credentials_header, + ) + + assert test_response.json == single_investigation_test_data_db.to_dict() + + def test_valid_no_results_findone_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + test_response = flask_test_app_db.get( + '/investigations/findone?where={"TITLE": {"eq": "This filter should cause a' + '404 for testing purposes..."}}', + headers=valid_db_credentials_header, + ) + + assert test_response.status_code == 404 diff --git a/test/icat/endpoints/test_findone_icat.py b/test/icat/endpoints/test_findone_icat.py index 3934eb22..778c7ebb 100644 --- a/test/icat/endpoints/test_findone_icat.py +++ b/test/icat/endpoints/test_findone_icat.py @@ -1,7 +1,7 @@ from test.icat.test_query import prepare_icat_data_for_assertion -class TestFindone: +class TestICATFindone: def test_valid_findone_with_filters( self, flask_test_app_icat, From c5a867c6c8d101fc639ae6a7e138ddb58f60d29a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 7 Dec 2020 13:33:43 +0000 Subject: [PATCH 096/109] #150: Actually fix openapi YAML creation - This is essentially an undo of 8c1941dfbda5d77092a19a61c2e14a2a300f5786 as I've realised the backend must get parsed into the endpoint classes for testing reasons --- datagateway_api/src/main.py | 42 +- .../non_entities/sessions_endpoints.py | 282 ++++++----- .../table_endpoints/table_endpoints.py | 477 ++++++++++-------- 3 files changed, 444 insertions(+), 357 deletions(-) diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 7670b22f..22b21357 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -19,12 +19,14 @@ get_id_endpoint, ) from datagateway_api.src.resources.entities.entity_map import endpoints -from datagateway_api.src.resources.non_entities.sessions_endpoints import Sessions +from datagateway_api.src.resources.non_entities.sessions_endpoints import ( + session_endpoints, +) from datagateway_api.src.resources.table_endpoints.table_endpoints import ( - InstrumentsFacilityCycles, - InstrumentsFacilityCyclesCount, - InstrumentsFacilityCyclesInvestigations, - InstrumentsFacilityCyclesInvestigationsCount, + count_instrument_facility_cycles_endpoint, + count_instrument_investigation_endpoint, + instrument_facility_cycles_endpoint, + instrument_investigation_endpoint, ) from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin from datagateway_api.src.swagger.initialise_spec import initialise_spec @@ -101,32 +103,42 @@ def create_api_endpoints(flask_app, api, spec): spec.path(resource=get_find_one_endpoint_resource, api=api) # Session endpoint - api.add_resource(Sessions, "/sessions") - spec.path(resource=Sessions, api=api) + session_endpoint_resource = session_endpoints(backend) + api.add_resource(session_endpoint_resource, "/sessions") + spec.path(resource=session_endpoint_resource, api=api) # Table specific endpoints + instrument_facility_cycle_resource = instrument_facility_cycles_endpoint(backend) api.add_resource( - InstrumentsFacilityCycles, "/instruments//facilitycycles", + instrument_facility_cycle_resource, "/instruments//facilitycycles", ) - spec.path(resource=InstrumentsFacilityCycles, api=api) + spec.path(resource=instrument_facility_cycle_resource, api=api) + count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( + backend, + ) api.add_resource( - InstrumentsFacilityCyclesCount, "/instruments//facilitycycles/count", + count_instrument_facility_cycle_res, + "/instruments//facilitycycles/count", ) - spec.path(resource=InstrumentsFacilityCyclesCount, api=api) + spec.path(resource=count_instrument_facility_cycle_res, api=api) + instrument_investigation_resource = instrument_investigation_endpoint(backend) api.add_resource( - InstrumentsFacilityCyclesInvestigations, + instrument_investigation_resource, "/instruments//facilitycycles//investigations", ) - spec.path(resource=InstrumentsFacilityCyclesInvestigations, api=api) + spec.path(resource=instrument_investigation_resource, api=api) + count_instrument_investigation_resource = count_instrument_investigation_endpoint( + backend, + ) api.add_resource( - InstrumentsFacilityCyclesInvestigationsCount, + count_instrument_investigation_resource, "/instruments//facilitycycles//investigations" "/count", ) - spec.path(resource=InstrumentsFacilityCyclesInvestigationsCount, api=api) + spec.path(resource=count_instrument_investigation_resource, api=api) def openapi_config(spec): diff --git a/datagateway_api/src/resources/non_entities/sessions_endpoints.py b/datagateway_api/src/resources/non_entities/sessions_endpoints.py index 6049afab..888c20af 100644 --- a/datagateway_api/src/resources/non_entities/sessions_endpoints.py +++ b/datagateway_api/src/resources/non_entities/sessions_endpoints.py @@ -3,152 +3,164 @@ from flask import request from flask_restful import Resource -from datagateway_api.common.config import config from datagateway_api.common.exceptions import AuthenticationError from datagateway_api.common.helpers import get_session_id_from_auth_header log = logging.getLogger() -backend = config.get_backend_type() +def session_endpoints(backend): + """ + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` -class Sessions(Resource): - def post(self): - """ - Generates a sessionID if the user has correct credentials - :return: String - SessionID + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated session endpoint class + """ - --- - summary: Login - description: Generates a sessionID if the user has correct credentials - tags: - - Sessions - security: [] - requestBody: - description: User credentials to login with - required: true - content: - application/json: - schema: - type: object - properties: - username: - type: string - password: - type: string - mechanism: - type: string - responses: - 201: - description: Success - returns a session ID - content: - application/json: - schema: - type: object - properties: - sessionID: - type: string - description: Session ID - example: xxxxxx-yyyyyyy-zzzzzz - 400: - description: Bad request. User credentials not provided in request body - 403: - description: Forbidden. User credentials were invalid - """ - if not ( - request.data and "username" in request.json and "password" in request.json - ): - return "Bad request", 400 - # If no mechanism is present in request body, default to simple - if not ("mechanism" in request.json): - request.json["mechanism"] = "simple" - try: - return {"sessionID": backend.login(request.json)}, 201 - except AuthenticationError: - return "Forbidden", 403 + class Sessions(Resource): + def post(self): + """ + Generates a sessionID if the user has correct credentials + :return: String - SessionID + + --- + summary: Login + description: Generates a sessionID if the user has correct credentials + tags: + - Sessions + security: [] + requestBody: + description: User credentials to login with + required: true + content: + application/json: + schema: + type: object + properties: + username: + type: string + password: + type: string + mechanism: + type: string + responses: + 201: + description: Success - returns a session ID + content: + application/json: + schema: + type: object + properties: + sessionID: + type: string + description: Session ID + example: xxxxxx-yyyyyyy-zzzzzz + 400: + description: Bad request. User credentials not provided in request body + 403: + description: Forbidden. User credentials were invalid + """ + if not ( + request.data + and "username" in request.json + and "password" in request.json + ): + return "Bad request", 400 + # If no mechanism is present in request body, default to simple + if not ("mechanism" in request.json): + request.json["mechanism"] = "simple" + try: + return {"sessionID": backend.login(request.json)}, 201 + except AuthenticationError: + return "Forbidden", 403 - def delete(self): - """ - Deletes a users sessionID when they logout - :return: Blank response, 200 - --- - summary: Delete session - description: Deletes a users sessionID when they logout - tags: - - Sessions - responses: - 200: - description: Success - User's session was successfully deleted - 400: - description: Bad request - something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: Not Found - Unable to find session ID - """ - backend.logout(get_session_id_from_auth_header()) - return "", 200 + def delete(self): + """ + Deletes a users sessionID when they logout + :return: Blank response, 200 + --- + summary: Delete session + description: Deletes a users sessionID when they logout + tags: + - Sessions + responses: + 200: + description: Success - User's session was successfully deleted + 400: + description: Bad request - something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: Not Found - Unable to find session ID + """ + backend.logout(get_session_id_from_auth_header()) + return "", 200 - def get(self): - """ - Gives details of a users session - :return: String: Details of the session, 200 - --- - summary: Get session details - description: Gives details of a user's session - tags: - - Sessions - responses: - 200: - description: Success - a user's session details - content: - application/json: - schema: - type: object - properties: - ID: + def get(self): + """ + Gives details of a users session + :return: String: Details of the session, 200 + --- + summary: Get session details + description: Gives details of a user's session + tags: + - Sessions + responses: + 200: + description: Success - a user's session details + content: + application/json: + schema: + type: object + properties: + ID: + type: string + description: The session ID + example: xxxxxx-yyyyyyy-zzzzzz + EXPIREDATETIME: + type: string + format: datetime + description: When this session expires + example: "2017-07-21T17:32:28Z" + USERNAME: + type: string + description: Username associated with this session + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + """ + return backend.get_session_details(get_session_id_from_auth_header()), 200 + + def put(self): + """ + Refreshes a users session + :return: String: The session ID that has been refreshed, 200 + --- + summary: Refresh session + description: Refreshes a users session + tags: + - Sessions + responses: + 200: + description: Success - the user's session ID that has been refreshed + content: + application/json: + schema: type: string - description: The session ID + description: Session ID example: xxxxxx-yyyyyyy-zzzzzz - EXPIREDATETIME: - type: string - format: datetime - description: When this session expires - example: "2017-07-21T17:32:28Z" - USERNAME: - type: string - description: Username associated with this session - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - """ - return backend.get_session_details(get_session_id_from_auth_header()), 200 + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + """ + return backend.refresh(get_session_id_from_auth_header()), 200 - def put(self): - """ - Refreshes a users session - :return: String: The session ID that has been refreshed, 200 - --- - summary: Refresh session - description: Refreshes a users session - tags: - - Sessions - responses: - 200: - description: Success - the user's session ID that has been refreshed - content: - application/json: - schema: - type: string - description: Session ID - example: xxxxxx-yyyyyyy-zzzzzz - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - """ - return backend.refresh(get_session_id_from_auth_header()), 200 + return Sessions diff --git a/datagateway_api/src/resources/table_endpoints/table_endpoints.py b/datagateway_api/src/resources/table_endpoints/table_endpoints.py index 88637bcf..94d62463 100644 --- a/datagateway_api/src/resources/table_endpoints/table_endpoints.py +++ b/datagateway_api/src/resources/table_endpoints/table_endpoints.py @@ -1,214 +1,277 @@ from flask_restful import Resource -from datagateway_api.common.config import config from datagateway_api.common.helpers import ( get_filters_from_query_string, get_session_id_from_auth_header, ) -backend = config.get_backend_type() - - -class InstrumentsFacilityCycles(Resource): - def get(self, id_): - """ - --- - summary: Get an Instrument's FacilityCycles - description: Given an Instrument id get facility cycles where the instrument has - investigations that occur within that cycle, subject to the given filters - tags: - - FacilityCycles - parameters: - - in: path - required: true - name: id - description: The id of the instrument to retrieve the facility cycles of - schema: - type: integer - - WHERE_FILTER - - ORDER_FILTER - - LIMIT_FILTER - - SKIP_FILTER - - DISTINCT_FILTER - - INCLUDE_FILTER - responses: - 200: - description: Success - returns a list of the instrument's facility - cycles that satisfy the filters - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/FACILITYCYCLE' - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_facility_cycles_for_instrument_with_filters( - get_session_id_from_auth_header(), id_, get_filters_from_query_string(), - ), - 200, - ) - - -class InstrumentsFacilityCyclesCount(Resource): - def get(self, id_): - """ - --- - summary: Count an Instrument's FacilityCycles - description: Return the count of the Facility Cycles that have investigations - that occur within that cycle on the specified instrument that would be - retrieved given the filters provided - tags: - - FacilityCycles - parameters: - - in: path - required: true - name: id - description: The id of the instrument to count the facility cycles of - schema: - type: integer - - WHERE_FILTER - - DISTINCT_FILTER - responses: - 200: - description: Success - The count of the instrument's facility cycles - that satisfy the filters - content: - application/json: - schema: - type: integer - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_facility_cycles_for_instrument_count_with_filters( - get_session_id_from_auth_header(), id_, get_filters_from_query_string(), - ), - 200, - ) - - -class InstrumentsFacilityCyclesInvestigations(Resource): - def get(self, instrument_id, cycle_id): - """ - --- - summary: Get the investigations for a given Facility Cycle & Instrument - description: Given an Instrument id and Facility Cycle id, get the - investigations that occur within that cycle on that instrument, subject to - the given filters - tags: - - Investigations - parameters: - - in: path - required: true - name: instrument_id - description: The id of the instrument to retrieve the investigations of - schema: - type: integer - - in: path - required: true - name: cycle_id - description: The id of the facility cycle to retrieve the investigations - schema: - type: integer - - WHERE_FILTER - - ORDER_FILTER - - LIMIT_FILTER - - SKIP_FILTER - - DISTINCT_FILTER - - INCLUDE_FILTER - responses: - 200: - description: Success - returns a list of the investigations for the - given instrument and facility cycle that satisfy the filters - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/INVESTIGATION' - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_investigations_for_instrument_facility_cycle_with_filters( - get_session_id_from_auth_header(), - instrument_id, - cycle_id, - get_filters_from_query_string(), - ), - 200, - ) - - -class InstrumentsFacilityCyclesInvestigationsCount(Resource): - def get(self, instrument_id, cycle_id): - """ - --- - summary: Count investigations for a given Facility Cycle & Instrument - description: Given an Instrument id and Facility Cycle id, get the number of - investigations that occur within that cycle on that instrument, subject to - the given filters - tags: - - Investigations - parameters: - - in: path - required: true - name: instrument_id - description: The id of the instrument to retrieve the investigations of - schema: - type: integer - - in: path - required: true - name: cycle_id - description: The id of the facility cycle to retrieve the investigations - schema: - type: integer - - WHERE_FILTER - - DISTINCT_FILTER - responses: - 200: - description: Success - The count of the investigations for the given - instrument and facility cycle that satisfy the filters - content: - application/json: - schema: - type: integer - 400: - description: Bad request - Something was wrong with the request - 401: - description: Unauthorized - No session ID found in HTTP Auth. header - 403: - description: Forbidden - The session ID provided is invalid - 404: - description: No such record - Unable to find a record in ICAT - """ - return ( - backend.get_investigation_count_instrument_facility_cycle_with_filters( - get_session_id_from_auth_header(), - instrument_id, - cycle_id, - get_filters_from_query_string(), - ), - 200, - ) + +def instrument_facility_cycles_endpoint(backend): + """ + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class + """ + pass + + class InstrumentsFacilityCycles(Resource): + def get(self, id_): + """ + --- + summary: Get an Instrument's FacilityCycles + description: Given an Instrument id get facility cycles where the instrument + has investigations that occur within that cycle, subject to the given + filters + tags: + - FacilityCycles + parameters: + - in: path + required: true + name: id + description: The id of the instrument to retrieve the facility cycles + of + schema: + type: integer + - WHERE_FILTER + - ORDER_FILTER + - LIMIT_FILTER + - SKIP_FILTER + - DISTINCT_FILTER + - INCLUDE_FILTER + responses: + 200: + description: Success - returns a list of the instrument's facility + cycles that satisfy the filters + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/FACILITYCYCLE' + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_facility_cycles_for_instrument_with_filters( + get_session_id_from_auth_header(), + id_, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCycles + + +def count_instrument_facility_cycles_endpoint(backend): + """ + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class + """ + pass + + class InstrumentsFacilityCyclesCount(Resource): + def get(self, id_): + """ + --- + summary: Count an Instrument's FacilityCycles + description: Return the count of the Facility Cycles that have + investigations that occur within that cycle on the specified instrument + that would be retrieved given the filters provided + tags: + - FacilityCycles + parameters: + - in: path + required: true + name: id + description: The id of the instrument to count the facility cycles of + schema: + type: integer + - WHERE_FILTER + - DISTINCT_FILTER + responses: + 200: + description: Success - The count of the instrument's facility cycles + that satisfy the filters + content: + application/json: + schema: + type: integer + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_facility_cycles_for_instrument_count_with_filters( + get_session_id_from_auth_header(), + id_, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCyclesCount + + +def instrument_investigation_endpoint(backend): + """ + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class + """ + pass + + class InstrumentsFacilityCyclesInvestigations(Resource): + def get(self, instrument_id, cycle_id): + """ + --- + summary: Get the investigations for a given Facility Cycle & Instrument + description: Given an Instrument id and Facility Cycle id, get the + investigations that occur within that cycle on that instrument, subject + to the given filters + tags: + - Investigations + parameters: + - in: path + required: true + name: instrument_id + description: The id of the instrument to retrieve the investigations + of + schema: + type: integer + - in: path + required: true + name: cycle_id + description: The id of the facility cycle to retrieve the + investigations + schema: + type: integer + - WHERE_FILTER + - ORDER_FILTER + - LIMIT_FILTER + - SKIP_FILTER + - DISTINCT_FILTER + - INCLUDE_FILTER + responses: + 200: + description: Success - returns a list of the investigations for the + given instrument and facility cycle that satisfy the filters + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/INVESTIGATION' + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_investigations_for_instrument_facility_cycle_with_filters( + get_session_id_from_auth_header(), + instrument_id, + cycle_id, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCyclesInvestigations + + +def count_instrument_investigation_endpoint(backend): + """ + Generate a flask_restful Resource class using the configured backend. In main.py + these generated classes are registered with the api e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")` + + :param backend: The backend instance used for processing requests + :type backend: :class:`DatabaseBackend` or :class:`PythonICATBackend` + :return: The generated endpoint class + """ + pass + + class InstrumentsFacilityCyclesInvestigationsCount(Resource): + def get(self, instrument_id, cycle_id): + """ + --- + summary: Count investigations for a given Facility Cycle & Instrument + description: Given an Instrument id and Facility Cycle id, get the number of + investigations that occur within that cycle on that instrument, subject + to the given filters + tags: + - Investigations + parameters: + - in: path + required: true + name: instrument_id + description: The id of the instrument to retrieve the investigations + of + schema: + type: integer + - in: path + required: true + name: cycle_id + description: The id of the facility cycle to retrieve the + investigations + schema: + type: integer + - WHERE_FILTER + - DISTINCT_FILTER + responses: + 200: + description: Success - The count of the investigations for the given + instrument and facility cycle that satisfy the filters + content: + application/json: + schema: + type: integer + 400: + description: Bad request - Something was wrong with the request + 401: + description: Unauthorized - No session ID found in HTTP Auth. header + 403: + description: Forbidden - The session ID provided is invalid + 404: + description: No such record - Unable to find a record in ICAT + """ + return ( + backend.get_investigation_count_instrument_facility_cycle_with_filters( + get_session_id_from_auth_header(), + instrument_id, + cycle_id, + get_filters_from_query_string(), + ), + 200, + ) + + return InstrumentsFacilityCyclesInvestigationsCount From 0c59b58b27e44c4258986602779579e7507c13e4 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 7 Dec 2020 15:01:03 +0000 Subject: [PATCH 097/109] #150: Add ISIS endpoint testing for DB backend --- test/db/conftest.py | 3 +- test/db/endpoints/test_table_endpoints_db.py | 133 ++++++++++++++++++ .../endpoints/test_table_endpoints_icat.py | 2 +- 3 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 test/db/endpoints/test_table_endpoints_db.py diff --git a/test/db/conftest.py b/test/db/conftest.py index 6f8fc208..24523f41 100644 --- a/test/db/conftest.py +++ b/test/db/conftest.py @@ -87,6 +87,7 @@ def isis_specific_endpoint_data_db(): ) facility_cycle.FACILITY_ID = 1 set_meta_attributes(facility_cycle) + insert_row_into_table(FACILITYCYCLE, facility_cycle) investigation = create_investigation_db_data() @@ -94,7 +95,6 @@ def isis_specific_endpoint_data_db(): instrument.NAME = "Test Instrument for DataGateway API Endpoint Testing (DB)" instrument.FACILITY_ID = 1 set_meta_attributes(instrument) - insert_row_into_table(INSTRUMENT, instrument) investigation_instrument = INVESTIGATIONINSTRUMENT() @@ -106,6 +106,7 @@ def isis_specific_endpoint_data_db(): yield (instrument.ID, facility_cycle, investigation) + delete_row_by_id(INVESTIGATIONINSTRUMENT, investigation_instrument.ID) delete_row_by_id(FACILITYCYCLE, facility_cycle.ID) delete_row_by_id(INVESTIGATION, investigation.ID) delete_row_by_id(INSTRUMENT, instrument.ID) diff --git a/test/db/endpoints/test_table_endpoints_db.py b/test/db/endpoints/test_table_endpoints_db.py new file mode 100644 index 00000000..64f5f636 --- /dev/null +++ b/test/db/endpoints/test_table_endpoints_db.py @@ -0,0 +1,133 @@ +class TestDBTableEndpoints: + """ + This class tests the endpoints defined in table_endpoints.py, commonly referred to + as the ISIS specific endpoints + """ + + def test_valid_get_facility_cycles_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + isis_specific_endpoint_data_db, + ): + + print(int(isis_specific_endpoint_data_db[0])) + test_response = flask_test_app_db.get( + f"/instruments/{int(isis_specific_endpoint_data_db[0])}/facilitycycles", + headers=valid_db_credentials_header, + ) + + assert test_response.json == [isis_specific_endpoint_data_db[1].to_dict()] + + def test_invalid_get_facility_cycles_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + final_instrument_result = flask_test_app_db.get( + '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, + ) + final_instrument_id = final_instrument_result.json["ID"] + + test_response = flask_test_app_db.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles", + headers=valid_db_credentials_header, + ) + + assert test_response.json == [] + + def test_valid_get_facility_cycles_count_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + isis_specific_endpoint_data_db, + ): + test_response = flask_test_app_db.get( + f"/instruments/{isis_specific_endpoint_data_db[0]}/facilitycycles/count", + headers=valid_db_credentials_header, + ) + + assert test_response.json == 1 + + def test_invalid_get_facility_cycles_count_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + final_instrument_result = flask_test_app_db.get( + '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, + ) + final_instrument_id = final_instrument_result.json["ID"] + + test_response = flask_test_app_db.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles/count", + headers=valid_db_credentials_header, + ) + + assert test_response.json == 0 + + def test_valid_get_investigations_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + isis_specific_endpoint_data_db, + ): + test_response = flask_test_app_db.get( + f"/instruments/{isis_specific_endpoint_data_db[0]}/facilitycycles/" + f"{isis_specific_endpoint_data_db[1].to_dict()['ID']}/investigations", + headers=valid_db_credentials_header, + ) + + assert test_response.json == [isis_specific_endpoint_data_db[2].to_dict()] + + def test_invalid_get_investigations_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + final_instrument_result = flask_test_app_db.get( + '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, + ) + final_instrument_id = final_instrument_result.json["ID"] + final_facilitycycle_result = flask_test_app_db.get( + '/facilitycycles/findone?order="ID DESC"', + headers=valid_db_credentials_header, + ) + final_facilitycycle_id = final_facilitycycle_result.json["ID"] + + test_response = flask_test_app_db.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles/" + f"{final_facilitycycle_id + 100}/investigations", + headers=valid_db_credentials_header, + ) + + assert test_response.json == [] + + def test_valid_get_investigations_count_with_filters( + self, + flask_test_app_db, + valid_db_credentials_header, + isis_specific_endpoint_data_db, + ): + test_response = flask_test_app_db.get( + f"/instruments/{isis_specific_endpoint_data_db[0]}/facilitycycles/" + f"{isis_specific_endpoint_data_db[1].to_dict()['ID']}/investigations/count", + headers=valid_db_credentials_header, + ) + + assert test_response.json == 1 + + def test_invalid_get_investigations_count_with_filters( + self, flask_test_app_db, valid_db_credentials_header, + ): + final_instrument_result = flask_test_app_db.get( + '/instruments/findone?order="id DESC"', headers=valid_db_credentials_header, + ) + final_instrument_id = final_instrument_result.json["ID"] + final_facilitycycle_result = flask_test_app_db.get( + '/facilitycycles/findone?order="ID DESC"', + headers=valid_db_credentials_header, + ) + final_facilitycycle_id = final_facilitycycle_result.json["ID"] + + test_response = flask_test_app_db.get( + f"/instruments/{final_instrument_id + 100}/facilitycycles/" + f"{final_facilitycycle_id + 100}/investigations/count", + headers=valid_db_credentials_header, + ) + + assert test_response.json == 0 diff --git a/test/icat/endpoints/test_table_endpoints_icat.py b/test/icat/endpoints/test_table_endpoints_icat.py index 6d607537..af1e9581 100644 --- a/test/icat/endpoints/test_table_endpoints_icat.py +++ b/test/icat/endpoints/test_table_endpoints_icat.py @@ -1,7 +1,7 @@ from test.icat.test_query import prepare_icat_data_for_assertion -class TestTableEndpoints: +class TestICATableEndpoints: """ This class tests the endpoints defined in table_endpoints.py, commonly referred to as the ISIS specific endpoints From 40392333b159de0d3c2034e8ede2ab46d705a2dd Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 8 Dec 2020 12:49:01 +0000 Subject: [PATCH 098/109] #150: Add documentation for running the tests --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec1f9cb9..7bc0c7ca 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,9 @@ Currently, the following Nox sessions have been created: - `safety` - this uses [safety](https://github.com/pyupio/safety) to check the dependencies (pulled directly from Poetry) for any known vulnerabilities. This session gives the output in a full ASCII style report. +- `tests` - this uses [pytest](https://docs.pytest.org/en/stable/) to execute the + automated tests in `test/`, tests for the database and ICAT backends, and non-backend + specific tests. More details [here](#running-tests). ### Pre Commit (Automated Checks during Git Commit) To make use of Git's ability to run custom hooks, [pre-commit](https://pre-commit.com/) @@ -234,7 +237,7 @@ provided in the base directory of this repository). Copy `config.json.example` t `config.json` and set the values as needed. Ideally, the API would be run with: -`poetry run python -m src.main` +`poetry run python -m datagateway_api.src.main` However it can be run with the flask run command as shown below: @@ -378,4 +381,52 @@ class DataCollectionDatasets(Resource): ## Running Tests -To run the tests use `python -m unittest discover` +To run the tests use `nox -s tests`. The repository contains a variety of tests, to test +the functionality of the API works as intended. The tests are split into 3 main +sections: non-backend specific (testing features such as the date handler), ICAT backend +tests (containing tests for backend specific components, including tests for the +different types of endpoints) and Database Backend tests (like the ICAT backend tests, +but covering only the most used aspects of the API). + +The configuration file (`config.json`) contains two options that will be used during the +testing of the API. Set `test_user_credentials` and `test_mechanism` appropriately for +your test environment, using `config.json.example` as a reference. The tests require a +connection to an instance of ICAT, so set the rest of the config as needed. + +By default, this will execute the repo's tests in +Python 3.6, 3.7 and 3.8. For most cases, running the tests in a single Python version +will be sufficient: + +```bash +nox -p 3.6 -s tests +``` + +This repository also utilises [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/) +to check how much of the codebase is covered by the tests in `test/`: + +```bash +nox -p 3.6 -s tests -- --cov-report term --cov=./datagateway_api +``` + +With `pytest`, you can output the duration for each test, useful for showing the slower +tests in the collection (sortest from slowest to fastest). The test duration is split +into setup, call and teardown to more easily understand where the tests are being slowed +down: + +```bash +nox -p 3.6 -s tests -- --durations=0 +``` + +To test a specific test class (or even a specific test function), use a double colon to +denote a each level down beyond the filename: + +```bash +# Test a specific file +nox -p 3.6 -s tests -- test/icat/test_query.py + +# Test a specific test class +nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery + +# Test a specific test function +nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery::test_valid_query_exeuction +``` From 96f5aecc5fdf37917899cf2c67aa42b7d25f94e5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 8 Dec 2020 18:38:05 +0000 Subject: [PATCH 099/109] #190: Make small changes to existing docs --- README.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7aa49236..87ed07ef 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # datagateway-api -ICAT API to interface with the Data Gateway +Flask-based API that fetches data from an ICAT instance, to interface with +[DataGateway](https://github.com/ral-facilities/datagateway) ## Contents - [datagateway-api](#datagateway-api) @@ -268,17 +269,20 @@ Once this is set the API can be run with `flask run` while inside the root direc the project. The `flask run` command gets installed with flask. Examples shown: -Unix + +Unix: ```bash $ export FLASK_APP=src/main.py $ flask run ``` -CMD + +CMD: ```CMD > set FLASK_APP=src/main.py > flask run ``` -PowerShell + +PowerShell: ```powershell > $env:FLASK_APP = "src/main.py" > flask run @@ -353,14 +357,14 @@ This is illustrated below. │ └── test_helpers.py └── util └── icat_db_generator.py - ````` +````` The directory tree can be generated using the following command: `git ls-tree -r --name-only HEAD | grep -v __init__.py | tree --fromfile` -#### Main +### Main `main.py` is where the flask_restful api is set up. This is where each endpoint resource class is generated and mapped to an endpoint. @@ -370,7 +374,7 @@ api.add_resource(get_endpoint(entity_name, endpoints[entity_name]), f"/{entity_n ``` -#### Endpoints +### Endpoints The logic for each endpoint are within `/src/resources`. They are split into entities, non_entities and table_endpoints. The entities package contains `entities_map` which maps entity names to their sqlalchemy model. The `entity_endpoint` module contains the @@ -379,7 +383,7 @@ endpoint classes that are table specific. Finally, non_entities contains the ses endpoint. -#### Mapped classes +### Mapped classes The classes mapped from the database are stored in `/common/database/models.py`. Each model was automatically generated using sqlacodegen. A class `EntityHelper` is defined so that each model may inherit two methods `to_dict()` and @@ -397,25 +401,25 @@ example: `python -m util.icat_db_generator -s 4 -y 10` Would set the seed to 4 a generate 10 years of data. -#### Querying and filtering: +### Querying and filtering: The querying and filtering logic is located in `/common/database_helpers.py`. In this module the abstract `Query` and `QueryFilter` classes are defined as well as their implementations. The functions that are used by various endpoints to query the database are also in this module. -#### Class diagrams for this module: +### Class diagrams for this module: ![image](https://user-images.githubusercontent.com/44777678/67954353-ba69ef80-fbe8-11e9-81e3-0668cea3fa35.png) ![image](https://user-images.githubusercontent.com/44777678/67954834-7fb48700-fbe9-11e9-96f3-ffefc7277ebd.png) -#### Authentication +### Authentication Each request requires a valid session ID to be provided in the Authorization header. This header should take the form of `{"Authorization":"Bearer "}` A session ID can be obtained by sending a post request to `/sessions/`. All endpoint methods that require a session id are decorated with `@requires_session_id` -#### Generating the swagger spec: `openapi.yaml` +### Generating the swagger spec: `openapi.yaml` When the config option `generate_swagger` is set to true in `config.json`, a YAML file defining the API using OpenAPI standards will be created at `src/swagger/openapi.yaml`. [apispec](https://apispec.readthedocs.io/en/latest/) is used From 8b2472514df420ea6546b860e95394dfd22ad1e9 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 9 Dec 2020 15:24:36 +0000 Subject: [PATCH 100/109] #190: Update README.md for ICAT backend --- README.md | 530 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 392 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 87ed07ef..ac911b3c 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,56 @@ -# datagateway-api -Flask-based API that fetches data from an ICAT instance, to interface with -[DataGateway](https://github.com/ral-facilities/datagateway) - -## Contents -- [datagateway-api](#datagateway-api) - - [Contents](#contents) - - [Creating Dev Environment and API Setup](#creating-dev-environment-and-api-setup) - - [Running DataGateway API](#running-datagateway-api) - - [Project structure](#project-structure) - - [Main](#main) - - [Endpoints](#endpoints) - - [Mapped classes](#mapped-classes) - - [Database Generator](#database-generator) - - [Class Diagrams](#class-diagrams-for-this-module) - - [Querying and filtering](#querying-and-filtering) - - [Swagger Generation](#generating-the-swagger-spec-openapiyaml) - - [Authentication](#authentication) +# DataGateway API +This is a Flask-based API that fetches data from an ICAT instance, to interface with +[DataGateway](https://github.com/ral-facilities/datagateway). This API uses two ways +for data collection/manipulation, using a Python-based ICAT API wrapper or using +sqlalchemy to communicate directly with ICAT's database. + + + + +# Contents +- [Creating Dev Environment and API Setup](#creating-dev-environment-and-api-setup) + - [Python Version Management (pyenv)](#python-version-management-(pyenv)) + - [API Dependency Management (Poetry)](#api-dependency-management-(poetry)) + - [Automated Testing & Other Development Helpers (Nox)](#automated-testing-&-other-development-helpers-(nox)) + - [Automated Checks during Git Commit (Pre Commit)](#automated-checks-during-git-commit-(pre-commit)) + - [Summary](#summary) +- [Running DataGateway API](#running-datagateway-api) + - [API Startup](#api-startup) + - [Authentication](#authentication) + - [Swagger Interface](#swagger-interface) +- [Running Tests](#running-tests) +- [Project Structure](#project-structure) + - [Main](#main) + - [Endpoints](#endpoints) + - [Logging](#logging) + - [Date Handler](#date-handler) + - [Exceptions & Flask Error Handling](#exceptions-&-flask-error-handling) + - [Filtering](#filtering) + - [Backends](#backends) + - [Abstract Backend Class](#abstract-backend-class) + - [Creating a Backend](#creating-a-backend) + - [Database Backend](#database-backend) + - [Mapped Classes](#mapped-classes) + - [Python ICAT Backend](#python-icat-backend) + - [ICATQuery](#icatquery) + - [Generating the OpenAPI Specification](#generating-the-openapi-specification) +- [Utilities](#utilities) - [Database Generator](#database-generator) - - [Running Tests](#running-tests) + - [Postman Collection](#postman-collection) +- [Updating README](#updating-readme) -## Creating Dev Environment and API Setup + + +# Creating Dev Environment and API Setup The recommended development environment for this API has taken lots of inspiration from the [Hypermodern Python](https://cjolowicz.github.io/posts/hypermodern-python-01-setup/) guide found online. It is assumed the commands shown in this part of the README are executed in the root directory of this repo once it has been cloned to your local machine. -### pyenv (Python Version Management) + +## Python Version Management (pyenv) To start, install [pyenv](https://github.com/pyenv/pyenv). There is a Windows version of this tool ([pyenv-win](https://github.com/pyenv-win/pyenv-win)), however this is currently untested on this repo. This is used to manage the various versions of Python @@ -86,7 +109,8 @@ currently listed in `.gitignore`): pyenv local 3.6.8 3.7.7 3.8.2 ``` -### Poetry (API Dependency Management) + +## API Dependency Management (Poetry) To maintain records of the API's dependencies, [Poetry](https://github.com/python-poetry/poetry) is used. To install, use the following command: @@ -121,7 +145,8 @@ intricacies of this command: poetry add [PACKAGE-NAME] ``` -### Nox (Automated Testing & Other Code Changes) + +## Automated Testing & Other Development Helpers (Nox) When developing new features for the API, there are a number of Nox sessions that can be used to lint/format/test the code in the included `noxfile.py`. To install Nox, use Pip as shown below. Nox is not listed as a Poetry dependency because this has the potential @@ -157,7 +182,7 @@ Currently, the following Nox sessions have been created: gives the output in a full ASCII style report. - `tests` - this uses [pytest](https://docs.pytest.org/en/stable/) to execute the automated tests in `test/`, tests for the database and ICAT backends, and non-backend - specific tests. More details [here](#running-tests). + specific tests. More details about the tests themselves [here](#running-tests). Each Nox session builds an environment using the repo's dependencies (defined using Poetry) using `install_with_constraints()`. This stores the dependencies in a @@ -174,7 +199,7 @@ nox -s lint -- util datagateway_api --tmpdir /root ``` -### Pre Commit (Automated Checks during Git Commit) +## Automated Checks during Git Commit (Pre Commit) To make use of Git's ability to run custom hooks, [pre-commit](https://pre-commit.com/) is used. Like Nox, Pip is used to install this tool: @@ -198,8 +223,8 @@ command: pre-commit run --all-files ``` -### Summary +## Summary As a summary, these are the steps needed to create a dev environment for this repo compressed into a single code block: @@ -250,23 +275,34 @@ pre-commit install ``` -## Running DataGateway API + + +# Running DataGateway API Depending on the backend you want to use (either `db` or `python_icat`) the connection URL for the backend needs to be set. These are set in `config.json` (an example file is provided in the base directory of this repository). Copy `config.json.example` to `config.json` and set the values as needed. -Ideally, the API would be run with: -`poetry run python -m datagateway_api.src.main` -However it can be run with the flask run command as shown below: +By default, the API will run on `http://localhost:5000` and all requests are made here +e.g. `http://localhost:5000/sessions`. + +## API Startup +Ideally, the API would be run using the following command: + +```bash +poetry run python -m datagateway_api.src.main +``` + +However, it can also be run with the `flask run` command (installed with Flask) as shown +below: **Warning: the host, port and debug config options will not be respected when the API is run this way** -To use `flask run`, the enviroment variable `FLASK_APP` should be set to `src/main.py`. -Once this is set the API can be run with `flask run` while inside the root directory of -the project. The `flask run` command gets installed with flask. +To use `flask run`, the enviroment variable `FLASK_APP` should be set to +`datagateway_api/src/main.py`. Once this is set, the API can be run with `flask run` +while inside the root directory of the project. Examples shown: @@ -290,16 +326,91 @@ PowerShell: More information can be found [here](http://flask.pocoo.org/docs/1.0/cli/). -By default the api will run on `http://localhost:5000` and all requests are made here -e.g. `http://localhost:5000/sessions` -## Project structure -The project consists of 3 main packages: common, src and test. common contains modules -shared across test and src such as the database mapping classes. src contains the api -resources and their http method definitions, and test contains tests for each endpoint. +## Authentication +Each request requires a valid session ID to be provided in the Authorization header. +This header should take the form of `{"Authorization":"Bearer "}` A session +ID can be obtained by sending a POST request to `/sessions`. All endpoint methods that +require a session id are decorated with `@requires_session_id`. + -This is illustrated below. +## Swagger Interface +If you go to the API's base path in your browser (`http://localhost:5000` by default), a +representation of the API will be shown using +[Swagger UI](https://swagger.io/tools/swagger-ui/). This uses an OpenAPI specfication to +visualise and allow users to easily interact with the API without building their own +requests. It's great for gaining an understanding in what endpoints are available and +what inputs the requests can receive, all from an interactive interface. +This specification is built with the Database Backend in mind (attribute names on +example outputs are capitalised for example), however the Swagger interface can also be +used with the Python ICAT Backend. More details on how the API's OpenAPI specification +is built can be found [here](TODO INSERT HYPERLINK). + + + + +# Running Tests +To run the tests use `nox -s tests`. The repository contains a variety of tests, to test +the functionality of the API works as intended. The tests are split into 3 main +sections: non-backend specific (testing features such as the date handler), ICAT backend +tests (containing tests for backend specific components, including tests for the +different types of endpoints) and Database Backend tests (like the ICAT backend tests, +but covering only the most used aspects of the API). + +The configuration file (`config.json`) contains two options that will be used during the +testing of the API. Set `test_user_credentials` and `test_mechanism` appropriately for +your test environment, using `config.json.example` as a reference. The tests require a +connection to an instance of ICAT, so set the rest of the config as needed. + +By default, this will execute the repo's tests in +Python 3.6, 3.7 and 3.8. For most cases, running the tests in a single Python version +will be sufficient: + +```bash +nox -p 3.6 -s tests +``` + +This repository also utilises [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/) +to check how much of the codebase is covered by the tests in `test/`: + +```bash +nox -p 3.6 -s tests -- --cov-report term --cov=./datagateway_api +``` + +With `pytest`, you can output the duration for each test, useful for showing the slower +tests in the collection (sortest from slowest to fastest). The test duration is split +into setup, call and teardown to more easily understand where the tests are being slowed +down: + +```bash +nox -p 3.6 -s tests -- --durations=0 +``` + +To test a specific test class (or even a specific test function), use a double colon to +denote a each level down beyond the filename: + +```bash +# Test a specific file +nox -p 3.6 -s tests -- test/icat/test_query.py + +# Test a specific test class +nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery + +# Test a specific test function +nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery::test_valid_query_exeuction +``` + + + + +# Project Structure +The project consists of 3 main packages: `datagateway_api.common`, +`datagateway_api.src`, and `test`. `datagateway_api.common` contains modules for the +Database and Python ICAT Backends as well as code to deal with query filters. +`datagateway_api.src` contains the API resources and their HTTP method definitions (e.g. +GET, POST). `test` contains automated tests written using Pytest. A directory tree is +illustrated below: ````` . @@ -331,7 +442,8 @@ This is illustrated below. │ │ │ ├── filters.py │ │ │ ├── helpers.py │ │ │ └── query.py -│ │ └── logger_setup.py +│ │ ├── logger_setup.py +│ │ └── query_filter.py │ └── src │ ├── main.py │ ├── resources @@ -351,79 +463,204 @@ This is illustrated below. ├── postman_collection_icat.json ├── pyproject.toml ├── test +│ ├── conftest.py +│ ├── db +│ │ ├── conftest.py +│ │ ├── endpoints +│ │ │ ├── test_count_with_filters_db.py +│ │ │ ├── test_findone_db.py +│ │ │ ├── test_get_by_id_db.py +│ │ │ ├── test_get_with_filters.py +│ │ │ └── test_table_endpoints_db.py +│ │ ├── test_entity_helper.py +│ │ ├── test_query_filter_factory.py +│ │ └── test_requires_session_id.py +│ ├── icat +│ │ ├── conftest.py +│ │ ├── endpoints +│ │ │ ├── test_count_with_filters_icat.py +│ │ │ ├── test_create_icat.py +│ │ │ ├── test_delete_by_id_icat.py +│ │ │ ├── test_findone_icat.py +│ │ │ ├── test_get_by_id_icat.py +│ │ │ ├── test_get_with_filters_icat.py +│ │ │ ├── test_table_endpoints_icat.py +│ │ │ ├── test_update_by_id_icat.py +│ │ │ └── test_update_multiple_icat.py +│ │ ├── filters +│ │ │ ├── test_distinct_filter.py +│ │ │ ├── test_include_filter.py +│ │ │ ├── test_limit_filter.py +│ │ │ ├── test_order_filter.py +│ │ │ ├── test_skip_filter.py +│ │ │ └── test_where_filter.py +│ │ ├── test_filter_order_handler.py +│ │ ├── test_query.py +│ │ └── test_session_handling.py +│ ├── test_backends.py │ ├── test_base.py -│ ├── test_database_helpers.py -│ ├── test_entityHelper.py -│ └── test_helpers.py +│ ├── test_config.py +│ ├── test_date_handler.py +│ ├── test_endpoint_rules.py +│ ├── test_get_filters_from_query.py +│ ├── test_get_session_id_from_auth_header.py +│ ├── test_is_valid_json.py +│ └── test_queries_records.py └── util └── icat_db_generator.py ````` -The directory tree can be generated using the following command: - - `git ls-tree -r --name-only HEAD | grep -v __init__.py | tree --fromfile` - -### Main -`main.py` is where the flask_restful api is set up. This is where each endpoint resource +## Main +`main.py` is where the flask_restful API is set up. This is where each endpoint resource class is generated and mapped to an endpoint. Example: ```python -api.add_resource(get_endpoint(entity_name, endpoints[entity_name]), f"/{entity_name.lower()}") +api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") ``` -### Endpoints -The logic for each endpoint are within `/src/resources`. They are split into entities, -non_entities and table_endpoints. The entities package contains `entities_map` which -maps entity names to their sqlalchemy model. The `entity_endpoint` module contains the -function that is used to generate endpoints at start up. `table_endpoints` contains the -endpoint classes that are table specific. Finally, non_entities contains the session -endpoint. - - -### Mapped classes -The classes mapped from the database are stored in `/common/database/models.py`. Each -model was automatically generated using sqlacodegen. A class `EntityHelper` is defined -so that each model may inherit two methods `to_dict()` and -`update_from_dict(dictionary)`, both used for returning entities and updating them, in a -form easily converted to JSON. - - -## Database Generator -There is a tool to generate mock data into the database. It is located in -`util/icat_db_generator.py`. By default it will generate 20 years worth of data (approx -70,000 entities). The script makes use of `random` and `Faker` and is seeded with a seed -of 1. The seed and number of years of data generated can be changed by using the arg -flags `-s` or `--seed` for the seed, and `-y` or `--years` for the number of years. For -example: `python -m util.icat_db_generator -s 4 -y 10` Would set the seed to 4 and -generate 10 years of data. - - -### Querying and filtering: -The querying and filtering logic is located in `/common/database_helpers.py`. In this -module the abstract `Query` and `QueryFilter` classes are defined as well as their -implementations. The functions that are used by various endpoints to query the database -are also in this module. - - -### Class diagrams for this module: -![image](https://user-images.githubusercontent.com/44777678/67954353-ba69ef80-fbe8-11e9-81e3-0668cea3fa35.png) -![image](https://user-images.githubusercontent.com/44777678/67954834-7fb48700-fbe9-11e9-96f3-ffefc7277ebd.png) - - -### Authentication -Each request requires a valid session ID to be provided in the Authorization header. -This header should take the form of `{"Authorization":"Bearer "}` A session -ID can be obtained by sending a post request to `/sessions/`. All endpoint methods that -require a session id are decorated with `@requires_session_id` - -### Generating the swagger spec: `openapi.yaml` +## Endpoints +The logic for each endpoint is within `/src/resources`. They are split into entities, +non_entities and table_endpoints. + +The entities package contains `entity_map` which +maps entity names to their field name used in backend-specific code. The Database +Backend uses this for its mapped classes (explained below) and the Python ICAT Backend +uses this for interacting with ICAT objects within Python ICAT. In most instances, the +dictionary found in `entity_map.py` is simply mapping the plural entity name (used to +build the entity endpoints) to the singular version. The `entity_endpoint` module +contains the function that is used to generate endpoints at start up. `table_endpoints` +contains the endpoint classes that are table specific (currently these are the ISIS +specific endpoints required for their use cases). Finally, `non_entities` contains the +session endpoint for session handling. + + +## Logging +Logging configuration can be found in `datagateway_api.common.logger_setup.py`. This +contains a typical dictionary-based config for the standard Python `logging` library +that rotates files after they become 5MB in size. + +The default logging location is in the root directory of this repo. This location (and +filename) can be changed by editing the `log_location` value in `config.json`. The log +level (set to `WARN` by default) can also be changed using the appropriate value in that +file. + + +## Date Handler +This is a class containing static methods to deal with dates within the API. The date +handler can be used to convert dates between string and datetime objects (using a format +agreed in `datagateway_api.common.constants`) and uses a parser from `dateutil` to +detect if an input contains a date. This is useful for determining if a JSON value given +in a request body is a date, at which point it can be converted to a datetime object, +ready for storing in ICAT. The handler is currently only used in the Python ICAT +Backend, however this is non-backend specific class. + + +## Exceptions & Flask Error Handling +Exceptions custom to DataGateway API are defined in `datagateway_api.common.exceptions`. +Each exception has a status code and a default message (which can be changed when +raising the exception in code). None of them are backend specific, however some are only +used in a single backend because their meaning becomes irrelevant anywhere else. + +When the API is setup in `main.py`, a custom API object is created (inheriting +flask_restful's `Api` object) so `handle_error()` can be overridden. A previous +iteration of the API registered a error handler with the `Api` object, however this +meant DataGateway API's custom error handling only worked as intended in debug mode (as +detailed in a +[GitHub issue](https://github.com/ral-facilities/datagateway-api/issues/147)). This +solution prevents any exception returning a 500 status code (no matter the defined +status code in `exceptions.py`) in production mode. This is explained in a +[Stack Overflow answer](https://stackoverflow.com/a/43534068). + + +## Filtering +Filters available for use in the API are defined in `datagateway_api.common.filters`. +These filters are all based from `QueryFilter`, an asbtract class to define any filter +for the API. Precedence is used to prioritise in which order filters should be applied, +but is only needed for the Database Backend. + +Filtering logic is located in `datagateway_api.common.helpers`. +`get_filters_from_query_string()` uses the request query parameters to form filters to +be used within the API. A `QueryFilterFactory` is used to build filters for the correct +backend and the static method within this class is called in +`get_filters_from_query_string()`. + + +## Backends +As described at the top of this file, there are currently two ways that the API +creates/fetches/updates/deletes data from ICAT. The intention is each backend allows a +different method to communicate with ICAT, but results in a very similarly behaving +DataGateway API. + + +### Abstract Backend Class +The abstract class can be found in `datagateway_api.common.backend` and contains all the +abstract methods that should be found in a class which implements `Backend`. The typical +architecture across both backends is that the implemented functions call a helper +function to process the request and the result of that is returned to the user. + +Each backend module contains the following files which offer similar functionality, +implemented in their own ways: +- `backend.py` - Implemented version of `datagateway_api.common.backend` +- `filters.py` - Inherited versions of each filter defined in + `datagateway_api.common.filters` +- `helpers.py` - Helper functions that are called in `backend.py` + + +### Creating a Backend +A function inside `datagateway_api.common.backends` creates an instance of a backend +using input to that function to decide which backend to create. This function is called +in `main.py` which uses the backend type set in `config.json`, or a config value in the +Flask app if it's set (this config option is only used in the tests however). The +backend object is then parsed into the endpoint classes so the correct backend can be +used. + + +## Database Backend +The Database Backend uses [SQLAlchemy](https://www.sqlalchemy.org/) to interface +directly with the database for an instance of ICAT. This backend favours speed over +thoroughness, allowing no control over which users can access a particular piece of +data. + + +### Mapped Classes +The classes mapped from the database (as described [above](#endpoints)) are stored in +`/common/database/models.py`. Each model was automatically generated using sqlacodegen. +A class `EntityHelper` is defined so that each model may inherit two methods `to_dict()` +and `update_from_dict(dictionary)`, both used for returning entities and updating them, +in a form easily converted to JSON. + + +## Python ICAT Backend +Sometimes referred to as the ICAT Backend, this uses +[python-icat](https://python-icat.readthedocs.io/en/stable/) to interact with ICAT data. +The Python-based API wrapper allows ICAT Server to be accessed using the SOAP interface. +Python ICAT allows control over which users can access a particular piece of data, with +the API supporting multiple authentication mechanisms. Meta attributes such as `modId` +are dealt by Python ICAT, rather than the API. + + +### ICATQuery +The ICATQuery classed is in `datagateway_api.common.icat.query`. This class stores a +query created with Python ICAT +[documentation for the query](https://python-icat.readthedocs.io/en/stable/query.html). +The `execute_query()` function executes the query and returns either results in either a +JSON format, or a list of +[Python ICAT entity's](https://python-icat.readthedocs.io/en/stable/entity.html) (this +is defined using the `return_json_formattable` flag). Other functions within that class +are used within `execute_query()`. + + +## Generating the OpenAPI Specification When the config option `generate_swagger` is set to true in `config.json`, a YAML file defining the API using OpenAPI standards will be created at -`src/swagger/openapi.yaml`. [apispec](https://apispec.readthedocs.io/en/latest/) is used -to help with this, with an `APISpec()` object created in `src/main.py` which is added to +`src/swagger/openapi.yaml`. This option should be disabled in production to avoid any +issues with read-only directories. + +[apispec](https://apispec.readthedocs.io/en/latest/) is used to help with this, with an +`APISpec()` object created in `src/main.py` which endpoint specifications are added to (using `APISpec.path()`) when the endpoints are created for Flask. These paths are iterated over and ordered alphabetically, to ensure `openapi.yaml` only changes if there have been changes to the Swagger docs of the API; without that code, Git will detect @@ -432,58 +669,75 @@ contents of the `APISpec` object are written to a YAML file and is used when the goes to the configured (root) page in their browser. The endpoint related files in `src/resources/` contain `__doc__` which have the Swagger -docs for each type of endpoint. `src/resources/swagger/` contain code to aid Swagger doc +docs for each type of endpoint. For non-entity and table endpoints, the Swagger docs are +contained in the docstrings. `src/resources/swagger/` contain code to aid Swagger doc generation, with a plugin (`RestfulPlugin`) created for `apispec` to extract Swagger documentation from `flask-restful` functions. -## Running Tests -To run the tests use `nox -s tests`. The repository contains a variety of tests, to test -the functionality of the API works as intended. The tests are split into 3 main -sections: non-backend specific (testing features such as the date handler), ICAT backend -tests (containing tests for backend specific components, including tests for the -different types of endpoints) and Database Backend tests (like the ICAT backend tests, -but covering only the most used aspects of the API). -The configuration file (`config.json`) contains two options that will be used during the -testing of the API. Set `test_user_credentials` and `test_mechanism` appropriately for -your test environment, using `config.json.example` as a reference. The tests require a -connection to an instance of ICAT, so set the rest of the config as needed. -By default, this will execute the repo's tests in -Python 3.6, 3.7 and 3.8. For most cases, running the tests in a single Python version -will be sufficient: +# Utilities +Within the repository, there are some useful files which can help with using the API. -```bash -nox -p 3.6 -s tests -``` -This repository also utilises [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/) -to check how much of the codebase is covered by the tests in `test/`: +## Database Generator +There is a tool to generate mock data into ICAT's database. It is located in +`util/icat_db_generator.py`. By default it will generate 20 years worth of data (approx +70,000 entities). The script makes use of `random` and `Faker` and is seeded with a seed +of 1. The seed and number of years of data generated can be changed by using the arg +flags `-s` or `--seed` for the seed, and `-y` or `--years` for the number of years. For +example: `python -m util.icat_db_generator -s 4 -y 10` Would set the seed to 4 and +generate 10 years of data. -```bash -nox -p 3.6 -s tests -- --cov-report term --cov=./datagateway_api -``` +This uses code from the API's Database Backend, so a suitable `DB_URL` should be +configured in `config.json`. -With `pytest`, you can output the duration for each test, useful for showing the slower -tests in the collection (sortest from slowest to fastest). The test duration is split -into setup, call and teardown to more easily understand where the tests are being slowed -down: -```bash -nox -p 3.6 -s tests -- --durations=0 -``` +## Postman Collection +With a handful of endpoints associated with each entity, there are hundreds of endpoints +for this API. A Postman collection is stored in the root directory of this repository, +containing over 300 requests, with each type of endpoint for every entity as well as the +table and session endpoints. The exported collection is in v2.1 format and is currently +the recommended export version for Postman. -To test a specific test class (or even a specific test function), use a double colon to -denote a each level down beyond the filename: +This collection is mainly based around the Python ICAT Backend (request bodies for +creating and updating data uses camelCase attribute names as accepted by that backend) +but can easily be adapted for using the Database Backend if needed (changing attribute +names to uppercase for example). The collection also contains a login request specially +for the Database Backend, as logging in using that backend is slightly different to +logging in via the Python ICAT Backend. -```bash -# Test a specific file -nox -p 3.6 -s tests -- test/icat/test_query.py +The repo's collection can be easily imported into your Postman installation by opening +Postman and selecting File > Import... and choosing the Postman collection from your +cloned DataGateway API repository. -# Test a specific test class -nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery -# Test a specific test function -nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery::test_valid_query_exeuction + + +# Updating README +Like the codebase, this README file follows a 88 character per line formatting approach. +This isn't always possible with URLs and codeblocks, but the vast majority of the file +should follow this approach. Most IDEs can be configured to include a guideline to show +where this point is. To do this in VS Code, insert the following line into +`settings.json`: + +```json +"editor.rulers": [ + 88 +] ``` + +Before a heading with a single hash, a four line gap should be given to easily indicate +separation between two sections. Before every other heading (i.e. headings with two or +more hashes), a two line gap should be given. This helps to denote a new heading rather +than just a new paragraph. While sections can be easily distinguished in a colourful +IDE, the multi-line spacing can be much easier to identify on an editor that doesn't use +colours. + +The directory tree found in the [project structure](#project-structure) can be generated +using the following command: + + ```bash + git ls-tree -r --name-only HEAD | grep -v __init__.py | tree --fromfile + ``` From f2f7f55eff5ea86202471ea7b709d24747d7a606 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 9 Dec 2020 15:30:52 +0000 Subject: [PATCH 101/109] #165: Add a note to remind users to have user's Python added to PATH --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ac911b3c..f4bab49f 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,9 @@ used to lint/format/test the code in the included `noxfile.py`. To install Nox, as shown below. Nox is not listed as a Poetry dependency because this has the potential to cause issues if Nox was executed inside Poetry (see [here](https://medium.com/@cjolowicz/nox-is-a-part-of-your-global-developer-environment-like-poetry-pre-commit-pyenv-or-pipx-1cdeba9198bd) -for more detailed reasoning). If you do choose to install these packages within a +for more detailed reasoning). When using the `--user` option, ensure your user's Python +installation is added to the system `PATH` variable, remembering to reboot your system +if you need to change the `PATH`. If you do choose to install these packages within a virtual environment, you do not need the `--user` option: ```bash From 60c56f62ec877d1d6f934b24173b9cd40b9b06f8 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 9 Dec 2020 15:47:14 +0000 Subject: [PATCH 102/109] #190: Rename query_filter_factory.py --- datagateway_api/common/helpers.py | 2 +- .../common/{query_filter.py => query_filter_factory.py} | 0 test/db/test_query_filter_factory.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename datagateway_api/common/{query_filter.py => query_filter_factory.py} (100%) diff --git a/datagateway_api/common/helpers.py b/datagateway_api/common/helpers.py index 0a86dd33..a6562394 100644 --- a/datagateway_api/common/helpers.py +++ b/datagateway_api/common/helpers.py @@ -13,7 +13,7 @@ FilterError, MissingCredentialsError, ) -from datagateway_api.common.query_filter import QueryFilterFactory +from datagateway_api.common.query_filter_factory import QueryFilterFactory log = logging.getLogger() diff --git a/datagateway_api/common/query_filter.py b/datagateway_api/common/query_filter_factory.py similarity index 100% rename from datagateway_api/common/query_filter.py rename to datagateway_api/common/query_filter_factory.py diff --git a/test/db/test_query_filter_factory.py b/test/db/test_query_filter_factory.py index dfc6f745..f5d3c682 100644 --- a/test/db/test_query_filter_factory.py +++ b/test/db/test_query_filter_factory.py @@ -8,7 +8,7 @@ DatabaseSkipFilter, DatabaseWhereFilter, ) -from datagateway_api.common.query_filter import QueryFilterFactory +from datagateway_api.common.query_filter_factory import QueryFilterFactory class TestQueryFilterFactory: From 93d09807b6c631e6607a94d2964b5e0bf45b8018 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 18 Dec 2020 11:35:57 +0000 Subject: [PATCH 103/109] #190: Make change to project tree based on previous commit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4bab49f..81577859 100644 --- a/README.md +++ b/README.md @@ -445,7 +445,7 @@ illustrated below: │ │ │ ├── helpers.py │ │ │ └── query.py │ │ ├── logger_setup.py -│ │ └── query_filter.py +│ │ └── query_filter_factory.py │ └── src │ ├── main.py │ ├── resources From 135965c02c1384d1350fe4052c759041b561a273 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 4 Jan 2021 10:45:20 +0000 Subject: [PATCH 104/109] #165: Remove details regarding tmpdir cmd option - This has been removed in a different branch so I've removed the documented details about it in this branch and adjusted it according to the new solution found --- README.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 81577859..ebb7b572 100644 --- a/README.md +++ b/README.md @@ -189,16 +189,10 @@ Currently, the following Nox sessions have been created: Each Nox session builds an environment using the repo's dependencies (defined using Poetry) using `install_with_constraints()`. This stores the dependencies in a `requirements.txt`-like format temporarily during this process, using the OS' default -temporary location. This could result in permissions issues (this has been seen by a -colleague on Windows), so adding the `--tmpdir [DIRECTORY PATH]` allows the user to -define where this file should be stored. Due to Nox session being initiated in the -command line, this argument needs to be a positional argument (denoted by the `--` in -the Nox command). This argument is optional, but **must** be the final argument avoid -interference with Nox's argument parsing. An example: - -```bash -nox -s lint -- util datagateway_api --tmpdir /root -``` +temporary location. These files are manually deleted in `noxfile.py` (as opposed to +being automatically removed by Python) to minimise any potential permission-related +issues as documented +[here](https://github.com/bravoserver/bravo/issues/111#issuecomment-826990). ## Automated Checks during Git Commit (Pre Commit) From baff80dd02f6a6f40d217ac5b0aff438d02fb123 Mon Sep 17 00:00:00 2001 From: Matthew Richards <32678030+MRichards99@users.noreply.github.com> Date: Mon, 4 Jan 2021 11:56:38 +0000 Subject: [PATCH 105/109] Update datagateway_api/common/backends.py Co-authored-by: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> --- datagateway_api/common/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/common/backends.py b/datagateway_api/common/backends.py index ed19a93a..e74d941b 100644 --- a/datagateway_api/common/backends.py +++ b/datagateway_api/common/backends.py @@ -16,7 +16,7 @@ def create_backend(backend_type): :param backend_type: The type of backend that should be created and used for the API :type backend_type: :class:`str` - :return: Either an instance of `common.dataase.backend.DatabaseBackend` or + :return: Either an instance of `common.database.backend.DatabaseBackend` or `common.icat.backend.PythonICATBackend` """ From e826e207c439da7a6597df1fe30fc32bf3497292 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 4 Jan 2021 13:39:29 +0000 Subject: [PATCH 106/109] #150: Make requested changes on PR review --- datagateway_api/common/backends.py | 2 - datagateway_api/common/config.py | 1 - datagateway_api/common/constants.py | 3 ++ test/db/conftest.py | 21 +++++++- test/db/endpoints/test_get_by_id_db.py | 1 - test/db/endpoints/test_get_with_filters.py | 3 +- test/db/endpoints/test_table_endpoints_db.py | 48 +++++------------- test/db/test_entity_helper.py | 43 ++++++++-------- test/icat/conftest.py | 17 +++++++ test/icat/endpoints/test_create_icat.py | 17 ++----- .../endpoints/test_get_with_filters_icat.py | 6 +-- .../endpoints/test_table_endpoints_icat.py | 50 +++++-------------- test/icat/endpoints/test_update_by_id_icat.py | 15 ++---- test/icat/test_query.py | 20 +------- test/icat/test_session_handling.py | 27 +++++----- test/test_base.py | 21 -------- 16 files changed, 108 insertions(+), 187 deletions(-) delete mode 100644 test/test_base.py diff --git a/datagateway_api/common/backends.py b/datagateway_api/common/backends.py index ed19a93a..d9f92be3 100644 --- a/datagateway_api/common/backends.py +++ b/datagateway_api/common/backends.py @@ -1,6 +1,5 @@ import sys -from datagateway_api.common.backend import Backend from datagateway_api.common.database.backend import DatabaseBackend from datagateway_api.common.icat.backend import PythonICATBackend @@ -27,6 +26,5 @@ def create_backend(backend_type): else: # Might turn to a warning so the abstract class can be tested? sys.exit(f"Invalid config value '{backend_type}' for config option backend") - backend = Backend() return backend diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index e5d55ec4..9b5090a4 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -14,7 +14,6 @@ def __init__(self, path=Path(__file__).parent.parent.parent / "config.json"): self.path = path with open(self.path) as target: self.config = json.load(target) - target.close() def get_backend_type(self): try: diff --git a/datagateway_api/common/constants.py b/datagateway_api/common/constants.py index f6e502e6..ce1ae9ac 100644 --- a/datagateway_api/common/constants.py +++ b/datagateway_api/common/constants.py @@ -1,3 +1,5 @@ +from datetime import datetime + from datagateway_api.common.config import config @@ -6,3 +8,4 @@ class Constants: ACCEPTED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" PYTHON_ICAT_DISTNCT_CONDITION = "!= null" ICAT_PROPERTIES = config.get_icat_properties() + TEST_MOD_CREATE_DATETIME = datetime(2000, 1, 1) diff --git a/test/db/conftest.py b/test/db/conftest.py index 24523f41..6977104e 100644 --- a/test/db/conftest.py +++ b/test/db/conftest.py @@ -3,6 +3,7 @@ import pytest +from datagateway_api.common.constants import Constants from datagateway_api.common.database.helpers import ( delete_row_by_id, insert_row_into_table, @@ -17,8 +18,8 @@ def set_meta_attributes(entity): db_meta_attributes = { - "CREATE_TIME": datetime(2000, 1, 1), - "MOD_TIME": datetime(2000, 1, 1), + "CREATE_TIME": Constants.TEST_MOD_CREATE_DATETIME, + "MOD_TIME": Constants.TEST_MOD_CREATE_DATETIME, "CREATE_ID": "test create id", "MOD_ID": "test mod id", } @@ -110,3 +111,19 @@ def isis_specific_endpoint_data_db(): delete_row_by_id(FACILITYCYCLE, facility_cycle.ID) delete_row_by_id(INVESTIGATION, investigation.ID) delete_row_by_id(INSTRUMENT, instrument.ID) + + +@pytest.fixture() +def final_instrument_id(flask_test_app_db, valid_db_credentials_header): + final_instrument_result = flask_test_app_db.get( + '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, + ) + return final_instrument_result.json["ID"] + + +@pytest.fixture() +def final_facilitycycle_id(flask_test_app_db, valid_db_credentials_header): + final_facilitycycle_result = flask_test_app_db.get( + '/facilitycycles/findone?order="ID DESC"', headers=valid_db_credentials_header, + ) + return final_facilitycycle_result.json["ID"] diff --git a/test/db/endpoints/test_get_by_id_db.py b/test/db/endpoints/test_get_by_id_db.py index 8bb4ded5..3c2932a7 100644 --- a/test/db/endpoints/test_get_by_id_db.py +++ b/test/db/endpoints/test_get_by_id_db.py @@ -11,7 +11,6 @@ def test_valid_get_with_id( ' Testing (DB)"}}', headers=valid_db_credentials_header, ) - print(investigation_data.json) test_data_id = investigation_data.json[0]["ID"] test_response = flask_test_app_db.get( diff --git a/test/db/endpoints/test_get_with_filters.py b/test/db/endpoints/test_get_with_filters.py index 1d0d7018..da25b894 100644 --- a/test/db/endpoints/test_get_with_filters.py +++ b/test/db/endpoints/test_get_with_filters.py @@ -24,7 +24,6 @@ def test_valid_no_results_get_with_filters( 'testing purposes..."}}', headers=valid_db_credentials_header, ) - print(test_response.json) assert test_response.json == [] @@ -39,7 +38,7 @@ def test_valid_get_with_filters_distinct( ) expected = [ - {"TITLE": f"Title for DataGateway API Testing (DB) {i}" for i in range(5)}, + {"TITLE": f"Title for DataGateway API Testing (DB) {i}"} for i in range(5) ] for title in expected: diff --git a/test/db/endpoints/test_table_endpoints_db.py b/test/db/endpoints/test_table_endpoints_db.py index 64f5f636..45a95da5 100644 --- a/test/db/endpoints/test_table_endpoints_db.py +++ b/test/db/endpoints/test_table_endpoints_db.py @@ -10,8 +10,6 @@ def test_valid_get_facility_cycles_with_filters( valid_db_credentials_header, isis_specific_endpoint_data_db, ): - - print(int(isis_specific_endpoint_data_db[0])) test_response = flask_test_app_db.get( f"/instruments/{int(isis_specific_endpoint_data_db[0])}/facilitycycles", headers=valid_db_credentials_header, @@ -20,13 +18,8 @@ def test_valid_get_facility_cycles_with_filters( assert test_response.json == [isis_specific_endpoint_data_db[1].to_dict()] def test_invalid_get_facility_cycles_with_filters( - self, flask_test_app_db, valid_db_credentials_header, + self, flask_test_app_db, valid_db_credentials_header, final_instrument_id, ): - final_instrument_result = flask_test_app_db.get( - '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, - ) - final_instrument_id = final_instrument_result.json["ID"] - test_response = flask_test_app_db.get( f"/instruments/{final_instrument_id + 100}/facilitycycles", headers=valid_db_credentials_header, @@ -48,13 +41,8 @@ def test_valid_get_facility_cycles_count_with_filters( assert test_response.json == 1 def test_invalid_get_facility_cycles_count_with_filters( - self, flask_test_app_db, valid_db_credentials_header, + self, flask_test_app_db, valid_db_credentials_header, final_instrument_id, ): - final_instrument_result = flask_test_app_db.get( - '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, - ) - final_instrument_id = final_instrument_result.json["ID"] - test_response = flask_test_app_db.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/count", headers=valid_db_credentials_header, @@ -77,18 +65,12 @@ def test_valid_get_investigations_with_filters( assert test_response.json == [isis_specific_endpoint_data_db[2].to_dict()] def test_invalid_get_investigations_with_filters( - self, flask_test_app_db, valid_db_credentials_header, + self, + flask_test_app_db, + valid_db_credentials_header, + final_instrument_id, + final_facilitycycle_id, ): - final_instrument_result = flask_test_app_db.get( - '/instruments/findone?order="ID DESC"', headers=valid_db_credentials_header, - ) - final_instrument_id = final_instrument_result.json["ID"] - final_facilitycycle_result = flask_test_app_db.get( - '/facilitycycles/findone?order="ID DESC"', - headers=valid_db_credentials_header, - ) - final_facilitycycle_id = final_facilitycycle_result.json["ID"] - test_response = flask_test_app_db.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations", @@ -112,18 +94,12 @@ def test_valid_get_investigations_count_with_filters( assert test_response.json == 1 def test_invalid_get_investigations_count_with_filters( - self, flask_test_app_db, valid_db_credentials_header, + self, + flask_test_app_db, + valid_db_credentials_header, + final_instrument_id, + final_facilitycycle_id, ): - final_instrument_result = flask_test_app_db.get( - '/instruments/findone?order="id DESC"', headers=valid_db_credentials_header, - ) - final_instrument_id = final_instrument_result.json["ID"] - final_facilitycycle_result = flask_test_app_db.get( - '/facilitycycles/findone?order="ID DESC"', - headers=valid_db_credentials_header, - ) - final_facilitycycle_id = final_facilitycycle_result.json["ID"] - test_response = flask_test_app_db.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations/count", diff --git a/test/db/test_entity_helper.py b/test/db/test_entity_helper.py index 5f8e9c0e..bba41776 100644 --- a/test/db/test_entity_helper.py +++ b/test/db/test_entity_helper.py @@ -1,7 +1,6 @@ -import datetime - import pytest +from datagateway_api.common.constants import Constants from datagateway_api.common.database.models import ( DATAFILE, DATAFILEFORMAT, @@ -28,12 +27,12 @@ def datafile_entity(dataset_entity): datafile.DATASET = dataset_entity datafile.DATAFILEFORMAT = datafileformat datafile.NAME = "test name" - datafile.MOD_TIME = datetime.datetime(2000, 1, 1) - datafile.CREATE_TIME = datetime.datetime(2000, 1, 1) + datafile.MOD_TIME = Constants.TEST_MOD_CREATE_DATETIME + datafile.CREATE_TIME = Constants.TEST_MOD_CREATE_DATETIME datafile.CHECKSUM = "test checksum" datafile.FILESIZE = 64 - datafile.DATAFILEMODTIME = datetime.datetime(2000, 1, 1) - datafile.DATAFILECREATETIME = datetime.datetime(2000, 1, 1) + datafile.DATAFILEMODTIME = Constants.TEST_MOD_CREATE_DATETIME + datafile.DATAFILECREATETIME = Constants.TEST_MOD_CREATE_DATETIME datafile.DATASET_ID = 1 datafile.DOI = "test doi" datafile.DESCRIPTION = "test description" @@ -50,18 +49,18 @@ def test_valid_to_dict(self, datafile_entity): "ID": 1, "LOCATION": "test location", "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "MOD_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), "CHECKSUM": "test checksum", "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILEMODTIME": str(Constants.TEST_MOD_CREATE_DATETIME), + "DATAFILECREATETIME": str(Constants.TEST_MOD_CREATE_DATETIME), "DATASET_ID": 1, "DOI": "test doi", "DESCRIPTION": "test description", "CREATE_ID": "test create id", "MOD_ID": "test mod id", "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + "CREATE_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), } test_data = datafile_entity.to_dict() @@ -76,18 +75,18 @@ def test_valid_to_dict(self, datafile_entity): "ID": 1, "LOCATION": "test location", "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "MOD_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), "CHECKSUM": "test checksum", "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILEMODTIME": str(Constants.TEST_MOD_CREATE_DATETIME), + "DATAFILECREATETIME": str(Constants.TEST_MOD_CREATE_DATETIME), "DATASET_ID": 1, "DOI": "test doi", "DESCRIPTION": "test description", "CREATE_ID": "test create id", "MOD_ID": "test mod id", "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + "CREATE_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), "DATASET": { "ID": None, "CREATE_TIME": None, @@ -114,18 +113,18 @@ def test_valid_to_dict(self, datafile_entity): "ID": 1, "LOCATION": "test location", "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "MOD_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), "CHECKSUM": "test checksum", "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILEMODTIME": str(Constants.TEST_MOD_CREATE_DATETIME), + "DATAFILECREATETIME": str(Constants.TEST_MOD_CREATE_DATETIME), "DATASET_ID": 1, "DOI": "test doi", "DESCRIPTION": "test description", "CREATE_ID": "test create id", "MOD_ID": "test mod id", "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + "CREATE_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), "DATASET": { "ID": None, "CREATE_TIME": None, @@ -180,18 +179,18 @@ def test_valid_update_from_dict(self, datafile_entity): "ID": 1, "LOCATION": "test location", "NAME": "test name", - "MOD_TIME": str(datetime.datetime(2000, 1, 1)), + "MOD_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), "CHECKSUM": "test checksum", "FILESIZE": 64, - "DATAFILEMODTIME": str(datetime.datetime(2000, 1, 1)), - "DATAFILECREATETIME": str(datetime.datetime(2000, 1, 1)), + "DATAFILEMODTIME": str(Constants.TEST_MOD_CREATE_DATETIME), + "DATAFILECREATETIME": str(Constants.TEST_MOD_CREATE_DATETIME), "DATASET_ID": 1, "DOI": "test doi", "DESCRIPTION": "test description", "CREATE_ID": "test create id", "MOD_ID": "test mod id", "DATAFILEFORMAT_ID": 1, - "CREATE_TIME": str(datetime.datetime(2000, 1, 1)), + "CREATE_TIME": str(Constants.TEST_MOD_CREATE_DATETIME), } datafile.update_from_dict(test_dict_data) diff --git a/test/icat/conftest.py b/test/icat/conftest.py index 38bf6b01..2549dd83 100644 --- a/test/icat/conftest.py +++ b/test/icat/conftest.py @@ -134,3 +134,20 @@ def isis_specific_endpoint_data(icat_client): icat_client.delete(instrument) except ICATNoObjectError as e: print(e) + + +@pytest.fixture() +def final_instrument_id(flask_test_app_icat, valid_icat_credentials_header): + final_instrument_result = flask_test_app_icat.get( + '/instruments/findone?order="id DESC"', headers=valid_icat_credentials_header, + ) + return final_instrument_result.json["id"] + + +@pytest.fixture() +def final_facilitycycle_id(flask_test_app_icat, valid_icat_credentials_header): + final_facilitycycle_result = flask_test_app_icat.get( + '/facilitycycles/findone?order="id DESC"', + headers=valid_icat_credentials_header, + ) + return final_facilitycycle_result.json["id"] diff --git a/test/icat/endpoints/test_create_icat.py b/test/icat/endpoints/test_create_icat.py index b00d0bb8..79e4e198 100644 --- a/test/icat/endpoints/test_create_icat.py +++ b/test/icat/endpoints/test_create_icat.py @@ -7,7 +7,7 @@ def test_valid_create_data( ): create_investigations_json = [ { - "name": "Test Data for API Testing, Data Creation 1", + "name": f"Test Data for API Testing, Data Creation {i}", "title": "Test data for the Python ICAT Backend on DataGateway API", "summary": "Test data for DataGateway API testing", "releaseDate": "2020-03-03 08:00:08", @@ -17,19 +17,8 @@ def test_valid_create_data( "doi": "DataGateway API Test DOI", "facility": 1, "type": 1, - }, - { - "name": "Test Data for API Testing, Data Creation 2", - "title": "Test data for the Python ICAT Backend on DataGateway API", - "summary": "Test data for DataGateway API testing", - "releaseDate": "2020-03-03 08:00:08", - "startDate": "2020-02-02 09:00:09", - "endDate": "2020-02-03 10:00:10", - "visitId": "Data Creation Visit", - "doi": "DataGateway API Test DOI", - "facility": 1, - "type": 1, - }, + } + for i in range(2) ] test_response = flask_test_app_icat.post( diff --git a/test/icat/endpoints/test_get_with_filters_icat.py b/test/icat/endpoints/test_get_with_filters_icat.py index 900fa73d..b05e8160 100644 --- a/test/icat/endpoints/test_get_with_filters_icat.py +++ b/test/icat/endpoints/test_get_with_filters_icat.py @@ -41,10 +41,8 @@ def test_valid_get_with_filters_distinct( ) expected = [ - { - "title": f"Test data for the Python ICAT Backend on DataGateway API {i}" - for i in range(5) - }, + {"title": f"Test data for the Python ICAT Backend on DataGateway API {i}"} + for i in range(5) ] for title in expected: diff --git a/test/icat/endpoints/test_table_endpoints_icat.py b/test/icat/endpoints/test_table_endpoints_icat.py index af1e9581..a62b3c63 100644 --- a/test/icat/endpoints/test_table_endpoints_icat.py +++ b/test/icat/endpoints/test_table_endpoints_icat.py @@ -23,14 +23,8 @@ def test_valid_get_facility_cycles_with_filters( assert response_json == isis_specific_endpoint_data[1] def test_invalid_get_facility_cycles_with_filters( - self, flask_test_app_icat, valid_icat_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, final_instrument_id, ): - final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', - headers=valid_icat_credentials_header, - ) - final_instrument_id = final_instrument_result.json["id"] - test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles", headers=valid_icat_credentials_header, @@ -52,14 +46,8 @@ def test_valid_get_facility_cycles_count_with_filters( assert test_response.json == 1 def test_invalid_get_facility_cycles_count_with_filters( - self, flask_test_app_icat, valid_icat_credentials_header, + self, flask_test_app_icat, valid_icat_credentials_header, final_instrument_id, ): - final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', - headers=valid_icat_credentials_header, - ) - final_instrument_id = final_instrument_result.json["id"] - test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/count", headers=valid_icat_credentials_header, @@ -84,19 +72,12 @@ def test_valid_get_investigations_with_filters( assert response_json == isis_specific_endpoint_data[3] def test_invalid_get_investigations_with_filters( - self, flask_test_app_icat, valid_icat_credentials_header, + self, + flask_test_app_icat, + valid_icat_credentials_header, + final_instrument_id, + final_facilitycycle_id, ): - final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', - headers=valid_icat_credentials_header, - ) - final_instrument_id = final_instrument_result.json["id"] - final_facilitycycle_result = flask_test_app_icat.get( - '/facilitycycles/findone?order="id DESC"', - headers=valid_icat_credentials_header, - ) - final_facilitycycle_id = final_facilitycycle_result.json["id"] - test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations", @@ -120,19 +101,12 @@ def test_valid_get_investigations_count_with_filters( assert test_response.json == 1 def test_invalid_get_investigations_count_with_filters( - self, flask_test_app_icat, valid_icat_credentials_header, + self, + flask_test_app_icat, + valid_icat_credentials_header, + final_instrument_id, + final_facilitycycle_id, ): - final_instrument_result = flask_test_app_icat.get( - '/instruments/findone?order="id DESC"', - headers=valid_icat_credentials_header, - ) - final_instrument_id = final_instrument_result.json["id"] - final_facilitycycle_result = flask_test_app_icat.get( - '/facilitycycles/findone?order="id DESC"', - headers=valid_icat_credentials_header, - ) - final_facilitycycle_id = final_facilitycycle_result.json["id"] - test_response = flask_test_app_icat.get( f"/instruments/{final_instrument_id + 100}/facilitycycles/" f"{final_facilitycycle_id + 100}/investigations/count", diff --git a/test/icat/endpoints/test_update_by_id_icat.py b/test/icat/endpoints/test_update_by_id_icat.py index bd6d4727..af97f54e 100644 --- a/test/icat/endpoints/test_update_by_id_icat.py +++ b/test/icat/endpoints/test_update_by_id_icat.py @@ -8,15 +8,11 @@ def test_valid_update_with_id( valid_icat_credentials_header, single_investigation_test_data, ): - expected_doi = "Test Data Identifier" - expected_summary = "Test Summary" - update_data_json = { - "doi": expected_doi, - "summary": expected_summary, + "doi": "Test Data Identifier", + "summary": "Test Summary", } - single_investigation_test_data[0]["doi"] = expected_doi - single_investigation_test_data[0]["summary"] = expected_summary + single_investigation_test_data[0].update(update_data_json) test_response = flask_test_app_icat.patch( f"/investigations/{single_investigation_test_data[0]['id']}", @@ -37,10 +33,7 @@ def test_invalid_update_with_id( # DOI cannot be over 255 characters, which this string is invalid_update_json = { - "doi": "__________________________________________________________________" - "_________________________________________________________________________" - "_________________________________________________________________________" - "_________________________________________________________________________", + "doi": "_" * 256, } test_response = flask_test_app_icat.patch( diff --git a/test/icat/test_query.py b/test/icat/test_query.py index 6ee3e53d..6def14b8 100644 --- a/test/icat/test_query.py +++ b/test/icat/test_query.py @@ -46,7 +46,7 @@ def prepare_icat_data_for_assertion(data, remove_id=False): class TestICATQuery: def test_valid_query_creation(self, icat_client): - # Paramitise and add inputs for conditions, aggregate and includes + # Paramatise and add inputs for conditions, aggregate and includes test_query = ICATQuery(icat_client, "User") assert test_query.query.entity == icat_client.getEntityClass("User") @@ -80,12 +80,6 @@ def test_invalid_query_execution(self, icat_client): with pytest.raises(PythonICATError): test_query.execute_query(icat_client) - def test_valid_count_query_execution(self, icat_client): - pass - - def test_valid_distinct_query_execution(self, icat_client): - pass - def test_json_format_execution_output( self, icat_client, single_investigation_test_data, ): @@ -100,17 +94,6 @@ def test_json_format_execution_output( assert query_output_json == single_investigation_test_data - # gap in function testing - - def test_valid_entity_to_dict_conversion(self, icat_client): - # Want just a typical entity and an entity with an entity list in it - pass - - def test_valid_distinct_attribute_mapping(self): - pass - - # another gap - def test_include_fields_list_flatten(self, icat_client): included_field_set = { "investigationUsers.investigation.datasets", @@ -120,7 +103,6 @@ def test_include_fields_list_flatten(self, icat_client): } test_query = ICATQuery(icat_client, "User") - flat_list = test_query.flatten_query_included_fields(included_field_set) assert flat_list == [ diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index c01fd33e..2018ca01 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -8,9 +8,6 @@ class TestSessionHandling: - def test_session_id_decorator(self): - pass - def test_get_valid_session_details( self, flask_test_app_icat, valid_icat_credentials_header, ): @@ -18,27 +15,28 @@ def test_get_valid_session_details( "/sessions", headers=valid_icat_credentials_header, ) + session_expiry_datetime = datetime.strptime( + session_details.json["EXPIREDATETIME"], "%Y-%m-%d %H:%M:%S.%f", + ) + current_datetime = datetime.now() + time_diff = abs(session_expiry_datetime - current_datetime) + time_diff_minutes = time_diff.seconds / 60 + + # Allows a bit of leeway for slow test execution + assert time_diff_minutes < 120 and time_diff_minutes >= 118 + # Check username is correct assert ( session_details.json["USERNAME"] == f"{config.get_test_mechanism()}/" f"{config.get_test_user_credentials()['username']}" ) + # Check session ID matches the header from the request assert ( session_details.json["ID"] == valid_icat_credentials_header["Authorization"].split()[1] ) - session_expiry_datetime = datetime.strptime( - session_details.json["EXPIREDATETIME"], "%Y-%m-%d %H:%M:%S.%f", - ) - current_datetime = datetime.now() - time_diff = abs(session_expiry_datetime - current_datetime) - time_diff_minutes = time_diff.seconds / 60 - - # Allows a bit of leeway for slow test execution - assert time_diff_minutes < 120 and time_diff_minutes >= 118 - def test_get_invalid_session_details( self, bad_credentials_header, flask_test_app_icat, ): @@ -56,12 +54,13 @@ def test_refresh_session(self, valid_icat_credentials_header, flask_test_app_ica refresh_session = flask_test_app_icat.put( "/sessions", headers=valid_icat_credentials_header, ) - assert refresh_session.status_code == 200 post_refresh_session_details = flask_test_app_icat.get( "/sessions", headers=valid_icat_credentials_header, ) + assert refresh_session.status_code == 200 + assert ( pre_refresh_session_details.json["EXPIREDATETIME"] != post_refresh_session_details.json["EXPIREDATETIME"] diff --git a/test/test_base.py b/test/test_base.py deleted file mode 100644 index 5236c379..00000000 --- a/test/test_base.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest import TestCase - -from flask import Flask - -from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure - - -class FlaskAppTest(TestCase): - """ - The FlaskAppTest Base class sets up a test client to be used to mock requests - """ - - def setUp(self): - app = Flask(__name__) - app.config["TESTING"] = True - app.config["TESTING"] = True - app.config["TEST_BACKEND"] = "db" - - api, spec = create_app_infrastructure(app) - create_api_endpoints(app, api, spec) - self.app = app.test_client() From f320600cade44ea7ecfe4d3264a05b9dcc1da269 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 4 Jan 2021 15:42:09 +0000 Subject: [PATCH 107/109] #190: Make requested changes from PR linked to this issue --- README.md | 38 +++++++++++++++++++------------------- test/conftest.py | 1 - 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ebb7b572..07f37793 100644 --- a/README.md +++ b/README.md @@ -274,54 +274,54 @@ pre-commit install # Running DataGateway API -Depending on the backend you want to use (either `db` or `python_icat`) the connection -URL for the backend needs to be set. These are set in `config.json` (an example file is -provided in the base directory of this repository). Copy `config.json.example` to -`config.json` and set the values as needed. +Depending on the backend you want to use (either `db` or `python_icat`, more details +about backends [here](#backends)) the connection URL for the backend needs to be set. +These are set in `config.json` (an example file is provided in the base directory of +this repository). Copy `config.json.example` to `config.json` and set the values as +needed. By default, the API will run on `http://localhost:5000` and all requests are made here e.g. `http://localhost:5000/sessions`. ## API Startup -Ideally, the API would be run using the following command: +Ideally, the API would be run using the following command, the alternative (detailed +below) should only be used for development purposes. ```bash poetry run python -m datagateway_api.src.main ``` -However, it can also be run with the `flask run` command (installed with Flask) as shown -below: +However, it can also be run with the `flask run` command (installed with Flask). To use +`flask run`, the enviroment variable `FLASK_APP` should be set to +`datagateway_api/src/main.py`. Once this is set, the API can be run with `flask run` +while inside the root directory of the project. This shouldn't be used in production, as +detailed in Flask's documentation, this method of running the API is only +["provided for convenience"](https://flask.palletsprojects.com/en/1.1.x/cli/#run-the-development-server). -**Warning: the host, port and debug config options will not be respected when the API is +**WARNING: the host, port and debug config options will not be respected when the API is run this way** -To use `flask run`, the enviroment variable `FLASK_APP` should be set to -`datagateway_api/src/main.py`. Once this is set, the API can be run with `flask run` -while inside the root directory of the project. - -Examples shown: +Examples: Unix: ```bash -$ export FLASK_APP=src/main.py +$ export FLASK_APP=datagateway_api/src/main.py $ flask run ``` CMD: ```CMD -> set FLASK_APP=src/main.py +> set FLASK_APP=datagateway_api/src/main.py > flask run ``` PowerShell: ```powershell -> $env:FLASK_APP = "src/main.py" +> $env:FLASK_APP = "datagateway_api/src/main.py" > flask run ``` -More information can be found [here](http://flask.pocoo.org/docs/1.0/cli/). - ## Authentication Each request requires a valid session ID to be provided in the Authorization header. @@ -341,7 +341,7 @@ what inputs the requests can receive, all from an interactive interface. This specification is built with the Database Backend in mind (attribute names on example outputs are capitalised for example), however the Swagger interface can also be used with the Python ICAT Backend. More details on how the API's OpenAPI specification -is built can be found [here](TODO INSERT HYPERLINK). +is built can be found [here](#generating-the-openapi-specification). diff --git a/test/conftest.py b/test/conftest.py index 9c83d3dd..05313099 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -16,7 +16,6 @@ def bad_credentials_header(): return {"Authorization": "Bearer Invalid"} -# TODO - Implement this in test_session_handling.py @pytest.fixture() def invalid_credentials_header(): return {"Authorization": "Test"} From 73bb72d186a321b81bfba4e02484aed37d81e604 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 5 Jan 2021 10:20:13 +0000 Subject: [PATCH 108/109] #190: Add link to icat.manual repo for tutorials on setting up an ICAT instance --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 07f37793..f60871e9 100644 --- a/README.md +++ b/README.md @@ -277,8 +277,13 @@ pre-commit install Depending on the backend you want to use (either `db` or `python_icat`, more details about backends [here](#backends)) the connection URL for the backend needs to be set. These are set in `config.json` (an example file is provided in the base directory of -this repository). Copy `config.json.example` to `config.json` and set the values as -needed. +this repository). While both `DB_URL` and `ICAT_URL` should have values assigned to them +(for best practice), `DB_URL` will only be used for the database backend, and `ICAT_URL` + will only be used for the Python ICAT backend. Copy `config.json.example` to +`config.json` and set the values as needed. If you need to create an instance of ICAT, +there are a number of markdown-formatted tutorials that can be found on the +[icat.manual](https://github.com/icatproject/icat.manual/tree/master/tutorials) +repository. By default, the API will run on `http://localhost:5000` and all requests are made here e.g. `http://localhost:5000/sessions`. From fb3900b25334aa141c36b33efb17bc71e137c043 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 5 Jan 2021 10:23:10 +0000 Subject: [PATCH 109/109] #190: Add --without-hashes options to poetry export cmd - This commit also adds the "tests" session to the list of Nox sessions --- noxfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 72091cf6..fc0ecf34 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,7 +3,7 @@ import nox # Separating Black away from the rest of the sessions -nox.options.sessions = "lint", "safety" +nox.options.sessions = "lint", "safety", "tests" code_locations = "datagateway_api", "test", "util", "noxfile.py" @@ -14,6 +14,7 @@ def install_with_constraints(session, req_dir=None, *args, **kwargs): "export", "--dev", "--format=requirements.txt", + "--without-hashes", f"--output={requirements.name}", external=True, )