From e4cfb383f846bdec3aed14d9a0837f5d3f629156 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 5 Jan 2022 14:05:01 +0000 Subject: [PATCH 01/41] ci: use older version of `setuptools` to solve Python ICAT build issue --- .github/workflows/ci-build.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 9a1cae61..09d48eda 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -76,6 +76,13 @@ jobs: run: pip install nox==2020.8.22 - name: Install Poetry run: pip install poetry==1.1.9 + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + - name: Install dependencies run: poetry install @@ -207,6 +214,13 @@ jobs: run: cp datagateway_api/config.json.example datagateway_api/config.json - name: Install Poetry run: pip install poetry==1.1.9 + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + - name: Install dependencies run: poetry install @@ -251,6 +265,13 @@ jobs: - name: Create config.json run: cp datagateway_api/config.json.example datagateway_api/config.json + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + - name: Install dependencies run: poetry install From 2b66602cbeae0d74aff77df33f810ff1188fe579 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 5 Jan 2022 14:05:46 +0000 Subject: [PATCH 02/41] build: use older version of `setuptools` when running tests --- noxfile.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/noxfile.py b/noxfile.py index 4a97e124..60f2ddd6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -85,5 +85,16 @@ def safety(session): @nox.session(python=["3.6", "3.7", "3.8", "3.9"], reuse_venv=True) def tests(session): args = session.posargs + # Installing setuptools that will work with `2to3` which is used when building + # `python-icat` < 1.0. 58.0.0 removes support of this tool during builds: + # https://setuptools.pypa.io/en/latest/history.html#v58-0-0 + # Ideally this would be done within `pyproject.toml` but specifying `setuptools` as + # a dependency requires Poetry 1.2: + # https://github.com/python-poetry/poetry/issues/4511#issuecomment-922420457 + # Currently, only a pre-release exists for Poetry 1.2. Testing on the pre-release + # version didn't fix the `2to3` issue when building Python ICAT, perhaps because + # Python ICAT isn't built on the downgraded version for some reason? + session.run("poetry", "run", "pip", "uninstall", "-y", "setuptools") + session.run("poetry", "run", "pip", "install", "setuptools<58.0.0") session.run("poetry", "install", external=True) session.run("pytest", *args) From 4736888bf76cda0dbc00f997443ed565f0f5e760 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:18:39 +0000 Subject: [PATCH 03/41] feat: implement `between` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 9877938a..a20a2e5a 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -88,6 +88,10 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) + elif self.operation == "between": + where_filter = self.create_condition( + self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", + ) else: raise FilterError(f"Bad operation given to where filter: {self.operation}") @@ -116,8 +120,7 @@ def create_condition(attribute_name, operator, value): # distinct filter is used in a request jpql_value = ( f"{value}" - if operator == "in" - or operator == "!=" + if operator in ("in", "not in", "!=", "between") or str(value).startswith("UPPER") or "o." in str(value) else f"'{value}'" From d254642c4cf28168c47c8eacdc8fcbf90e1c1943 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:21:45 +0000 Subject: [PATCH 04/41] validate values supplied with `between` operator #297 --- datagateway_api/src/common/filters.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index c99e26eb..623ec393 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -34,6 +34,18 @@ def __init__(self, field, value, operation): " be in a list format e.g. [1, 2, 3]", ) + if self.operation == "between": + if not isinstance(self.value, list): + raise BadRequestError( + "When using the 'between' operation for a WHERE filter, the values" + " must be in a list format e.g. [1, 2]", + ) + if isinstance(self.value, list) and len(self.value) != 2: + raise BadRequestError( + "When using the 'between' operation for a WHERE filter, the list" + "must contain two values e.g. [1, 2]", + ) + class DistinctFieldFilter(QueryFilter): precedence = 0 From f5525c9230240671744534f34919d9511a118374 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:22:39 +0000 Subject: [PATCH 05/41] test: test `between` operator #297 --- .../icat/filters/test_where_filter.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index bc41a5fa..4cc5ce50 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -23,6 +23,7 @@ class TestICATWhereFilter: pytest.param("gte", 5, ["%s >= '5'"], id="greater than or equal"), pytest.param("in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list"), pytest.param("in", [], ["%s in (NULL)"], id="empty list"), + pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), ], ) def test_valid_operations( @@ -33,9 +34,23 @@ def test_valid_operations( assert icat_query.conditions == {"id": expected_condition_value} - def test_invalid_in_operation(self, icat_query): + @pytest.mark.parametrize( + "operation, value", + [ + pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), + pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), + pytest.param("between", [], id="between - empty list"), + pytest.param( + "between", [1], id="between - list with less than two elements", + ), + pytest.param( + "between", [1, 2, 3], id="between - list with more than two elements", + ), + ], + ) + def test_invalid_operations_raise_bad_request_error(self, operation, value): with pytest.raises(BadRequestError): - PythonICATWhereFilter("id", "1, 2, 3, 4, 5", "in") + PythonICATWhereFilter("id", value, operation) def test_invalid_operation(self, icat_query): test_filter = PythonICATWhereFilter("id", 10, "non") From fc1cf194454a4da60652b1f68df278c4624ddc11 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:25:29 +0000 Subject: [PATCH 06/41] feat: implement `inq` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index a20a2e5a..dd42e815 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -88,6 +88,8 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) + elif self.operation == "inq": + self.operation = "in" elif self.operation == "between": where_filter = self.create_condition( self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", From b0134b8882c2c75f666b9c5516c49e1f06c4d72d Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:27:45 +0000 Subject: [PATCH 07/41] validate values supplied with `inq` operator #297 --- datagateway_api/src/common/filters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index 623ec393..716000c9 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,11 +27,11 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation == "in": + if self.operation in ["in", "inq"]: if not isinstance(self.value, list): raise BadRequestError( - "When using the 'in' operation for a WHERE filter, the values must" - " be in a list format e.g. [1, 2, 3]", + f"When using the {self.operation} operation for a WHERE filter, the" + f" values must be in a list format e.g. [1, 2, 3]", ) if self.operation == "between": From 1b142143fb27750e4745d73a5eeb12ce6267849b Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:33:58 +0000 Subject: [PATCH 08/41] test: test `inq` operator #297 --- .../datagateway_api/icat/filters/test_where_filter.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 4cc5ce50..703857aa 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -21,8 +21,14 @@ class TestICATWhereFilter: pytest.param("lte", 5, ["%s <= '5'"], id="less than or equal"), pytest.param("gt", 5, ["%s > '5'"], id="greater than"), pytest.param("gte", 5, ["%s >= '5'"], id="greater than or equal"), - pytest.param("in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list"), - pytest.param("in", [], ["%s in (NULL)"], id="empty list"), + pytest.param( + "in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (in)", + ), + pytest.param("in", [], ["%s in (NULL)"], id="in empty list (in)"), + pytest.param( + "inq", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (inq)", + ), + pytest.param("inq", [], ["%s in (NULL)"], id="in empty list (inq)"), pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), ], ) @@ -38,6 +44,7 @@ def test_valid_operations( "operation, value", [ pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), + pytest.param("inq", "1, 2, 3, 4, 5", id="in a list (inq)"), pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), pytest.param("between", [], id="between - empty list"), pytest.param( From 00dbba525d5cd86cb5577f3b1621a7042cdd2fa0 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:36:00 +0000 Subject: [PATCH 09/41] feat: implement `nin` operator #297 --- .../src/datagateway_api/icat/filters.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index dd42e815..84d9d333 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -90,6 +90,20 @@ def create_filter(self): where_filter = self.create_condition(self.field, "in", self.value) elif self.operation == "inq": self.operation = "in" + where_filter = self.create_filter() + elif self.operation == "nin": + # Convert self.value into a string with brackets equivalent to tuple format. + # Cannot convert straight to tuple as single element tuples contain a + # trailing comma which Python ICAT/JPQL doesn't accept + self.value = str(self.value).replace("[", "(").replace("]", ")") + + # DataGateway Search can send requests with blank lists. Adding NULL to the + # filter prevents the API from returning a 500. An empty list will be + # returned instead, equivalent to the DB backend + if self.value == "()": + self.value = "(NULL)" + + where_filter = self.create_condition(self.field, "not in", self.value) elif self.operation == "between": where_filter = self.create_condition( self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", From 92b5f36389d16f5c79f0199d7dd1408eda688d7a Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:37:02 +0000 Subject: [PATCH 10/41] validate values supplied with `nin` operator #297 --- datagateway_api/src/common/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index 716000c9..a6936d62 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,7 +27,7 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation in ["in", "inq"]: + if self.operation in ["in", "nin", "inq"]: if not isinstance(self.value, list): raise BadRequestError( f"When using the {self.operation} operation for a WHERE filter, the" From 8d572564fbb48714df5276543259aadeb26cb2f4 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:39:05 +0000 Subject: [PATCH 11/41] test: test `nin` operator #297 --- test/datagateway_api/icat/filters/test_where_filter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 703857aa..464c51a2 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -29,6 +29,10 @@ class TestICATWhereFilter: "inq", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (inq)", ), pytest.param("inq", [], ["%s in (NULL)"], id="in empty list (inq)"), + pytest.param( + "nin", [1, 2, 3, 4], ["%s not in (1, 2, 3, 4)"], id="not in a list", + ), + pytest.param("nin", [], ["%s not in (NULL)"], id="not in empty list"), pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), ], ) @@ -45,6 +49,7 @@ def test_valid_operations( [ pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), pytest.param("inq", "1, 2, 3, 4, 5", id="in a list (inq)"), + pytest.param("nin", "1, 2, 3, 4, 5", id="nin"), pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), pytest.param("between", [], id="between - empty list"), pytest.param( From 9094bbb894ead20a53fadfd0e24b264af29548b9 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:40:09 +0000 Subject: [PATCH 12/41] feat: implement `neq` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 84d9d333..ede6f4b9 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -51,6 +51,9 @@ def create_filter(self): where_filter = self.create_condition(self.field, "=", self.value) elif self.operation == "ne": where_filter = self.create_condition(self.field, "!=", self.value) + elif self.operation == "neq": + self.operation = "ne" + where_filter = self.create_filter() elif self.operation == "like": where_filter = self.create_condition(self.field, "like", f"%{self.value}%") elif self.operation == "ilike": From 5817491ea90dc4dca5868b0e35475e534ad95878 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:40:58 +0000 Subject: [PATCH 13/41] test: test `neq` operator #297 --- test/datagateway_api/icat/filters/test_where_filter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 464c51a2..9a200c77 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -10,7 +10,8 @@ class TestICATWhereFilter: "operation, value, expected_condition_value", [ pytest.param("eq", 5, ["%s = '5'"], id="equal"), - pytest.param("ne", 5, ["%s != 5"], id="not equal"), + pytest.param("ne", 5, ["%s != 5"], id="not equal (ne)"), + pytest.param("neq", 5, ["%s != 5"], id="not equal (neq)"), pytest.param("like", 5, ["%s like '%%5%%'"], id="like"), pytest.param("ilike", 5, ["UPPER(%s) like UPPER('%%5%%')"], id="ilike"), pytest.param("nlike", 5, ["%s not like '%%5%%'"], id="not like"), From bf3fe0ef2ac582d55dbd881edf6a81a93625ce91 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:41:51 +0000 Subject: [PATCH 14/41] feat: implement `regexp` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index ede6f4b9..31452a9e 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -111,6 +111,8 @@ def create_filter(self): where_filter = self.create_condition( self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", ) + elif self.operation == "regexp": + where_filter = self.create_condition(self.field, "regexp", self.value) else: raise FilterError(f"Bad operation given to where filter: {self.operation}") From f25333f25791d927ab4601e942fa5e63bac3df6c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:42:36 +0000 Subject: [PATCH 15/41] test: test `regexp` operator #297 --- test/datagateway_api/icat/filters/test_where_filter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 9a200c77..6abaaf00 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -35,6 +35,7 @@ class TestICATWhereFilter: ), pytest.param("nin", [], ["%s not in (NULL)"], id="not in empty list"), pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), + pytest.param("regexp", "^Test", ["%s regexp '^Test'"], id="regexp"), ], ) def test_valid_operations( From bd15023dc7ed56126540eab49fc34b454544b52c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 25 Jan 2022 15:59:41 +0000 Subject: [PATCH 16/41] refactor: remove duplicated code for `between` operator validation #297 --- datagateway_api/src/common/filters.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index a6936d62..bd0e636c 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,20 +27,13 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation in ["in", "nin", "inq"]: + if self.operation in ["in", "nin", "inq", "between"]: if not isinstance(self.value, list): raise BadRequestError( f"When using the {self.operation} operation for a WHERE filter, the" - f" values must be in a list format e.g. [1, 2, 3]", + f" values must be in a list format e.g. [1, 2]", ) - - if self.operation == "between": - if not isinstance(self.value, list): - raise BadRequestError( - "When using the 'between' operation for a WHERE filter, the values" - " must be in a list format e.g. [1, 2]", - ) - if isinstance(self.value, list) and len(self.value) != 2: + if self.operation == "between" and len(self.value) != 2: raise BadRequestError( "When using the 'between' operation for a WHERE filter, the list" "must contain two values e.g. [1, 2]", From 8b344ef9d7bf04cbe55467c2b6eabcff325a8a33 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Tue, 25 Jan 2022 16:13:15 +0000 Subject: [PATCH 17/41] refactor: merge logic for `ne` and `neq` operators #297 Co-authored-by: Matthew Richards <32678030+MRichards99@users.noreply.github.com> --- datagateway_api/src/datagateway_api/icat/filters.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 31452a9e..2a70a106 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -49,11 +49,8 @@ def create_filter(self): log.info("Creating condition for ICAT where filter") if self.operation == "eq": where_filter = self.create_condition(self.field, "=", self.value) - elif self.operation == "ne": + elif self.operation in ["ne", "neq"]: where_filter = self.create_condition(self.field, "!=", self.value) - elif self.operation == "neq": - self.operation = "ne" - where_filter = self.create_filter() elif self.operation == "like": where_filter = self.create_condition(self.field, "like", f"%{self.value}%") elif self.operation == "ilike": From 3211dfb277e4466895f89af70b770f262b2581c7 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 25 Jan 2022 16:20:10 +0000 Subject: [PATCH 18/41] refactor: merge logic for `in` and `inq` operators #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 2a70a106..03d79efd 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -75,7 +75,7 @@ def create_filter(self): where_filter = self.create_condition(self.field, ">", self.value) elif self.operation == "gte": where_filter = self.create_condition(self.field, ">=", self.value) - elif self.operation == "in": + elif self.operation in ["in", "inq"]: # Convert self.value into a string with brackets equivalent to tuple format. # Cannot convert straight to tuple as single element tuples contain a # trailing comma which Python ICAT/JPQL doesn't accept @@ -88,9 +88,6 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) - elif self.operation == "inq": - self.operation = "in" - where_filter = self.create_filter() elif self.operation == "nin": # Convert self.value into a string with brackets equivalent to tuple format. # Cannot convert straight to tuple as single element tuples contain a From c13c0db07a1a23a662d5ce32d7fb2f4709bc282d Mon Sep 17 00:00:00 2001 From: Matthew Richards <32678030+MRichards99@users.noreply.github.com> Date: Mon, 24 Jan 2022 16:56:05 +0000 Subject: [PATCH 19/41] refactor: update PaNOSC mappings Co-authored-by: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> --- datagateway_api/search_api_mapping.json.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index 57161585..f93c2a04 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -63,12 +63,12 @@ "affiliation": {"Affiliation": "user.dataPublicationUsers.affiliations"} }, "Parameter": { - "base_icat_entity": "InvestigationParameter", + "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", - "name": "name", + "name": "type.name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", - "dataset": {"Dataset": "investigation.investigationInstruments.instrument.datasetInstruments.dataset"}, + "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, "document": {"Document": "investigation"} }, "Person": { From 3802cc9ee71a355d0ad87529f65112e8c3f8b881 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 26 Jan 2022 12:38:09 +0000 Subject: [PATCH 20/41] fix: fix example mapping file --- datagateway_api/search_api_mapping.json.example | 1 - 1 file changed, 1 deletion(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index ebef5f30..f93c2a04 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -71,7 +71,6 @@ "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, "document": {"Document": "investigation"} }, - }, "Person": { "base_icat_entity": "User", "id": "id", From 601c51b29cd29fbad687d83084202996e7eb4714 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 10:55:50 +0000 Subject: [PATCH 21/41] refactor: update PaNOSC `Parameter` mappings in example file #265 --- datagateway_api/search_api_mapping.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index f93c2a04..19a81cfa 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -68,7 +68,7 @@ "name": "type.name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", - "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, + "dataset": {"Dataset": "dataset"}, "document": {"Document": "investigation"} }, "Person": { From 1e8badb9fae458ee5d0be2a67d3774ef9a198e10 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 10:56:31 +0000 Subject: [PATCH 22/41] test: update `Parameter` mappings in mappings fixture #265 --- test/search_api/conftest.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/search_api/conftest.py b/test/search_api/conftest.py index e5bfb508..e8a52212 100644 --- a/test/search_api/conftest.py +++ b/test/search_api/conftest.py @@ -89,15 +89,12 @@ def test_search_api_mappings_data(): "affiliation": {"Affiliation": "user.dataPublicationUsers.affiliations"}, }, "Parameter": { - "base_icat_entity": "InvestigationParameter", + "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", - "name": "name", + "name": "type.name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", - "dataset": { - "Dataset": "investigation.investigationInstruments.instrument." - "datasetInstruments.dataset", - }, + "dataset": {"Dataset": "dataset"}, "document": {"Document": "investigation"}, }, "Person": { From d595cebf354e95cd3d4fa229229594ec131e4441 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 10:57:56 +0000 Subject: [PATCH 23/41] test: fix tests following updates to PaNOSC `Parameter` mappings #265 --- test/search_api/test_models.py | 35 ++++------------------------------ 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/test/search_api/test_models.py b/test/search_api/test_models.py index 2887236b..3c4e04a0 100644 --- a/test/search_api/test_models.py +++ b/test/search_api/test_models.py @@ -498,7 +498,6 @@ def test_from_icat_parameter_entity_without_data_for_related_entities(self): def test_from_icat_parameter_entity_with_investigation_parameter_data(self): expected_entity_data = PARAMETER_PANOSC_DATA.copy() - expected_entity_data["dataset"] = DATASET_PANOSC_DATA expected_entity_data["document"] = DOCUMENT_PANOSC_DATA icat_data = INVESTIGATION_PARAMETER_ICAT_DATA.copy() @@ -506,18 +505,8 @@ def test_from_icat_parameter_entity_with_investigation_parameter_data(self): icat_data["investigation"] = INVESTIGATION_ICAT_DATA.copy() icat_data["investigation"]["type"] = INVESTIGATION_TYPE_ICAT_DATA icat_data["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] - icat_data["investigation"]["investigationInstruments"] = [ - {"instrument": INSTRUMENT_ICAT_DATA.copy()}, - {"instrument": INSTRUMENT_ICAT_DATA.copy()}, - ] - icat_data["investigation"]["investigationInstruments"][0]["instrument"][ - "datasetInstruments" - ] = [{"dataset": DATASET_ICAT_DATA}, {"dataset": DATASET_ICAT_DATA}] - icat_data["investigation"]["investigationInstruments"][1]["instrument"][ - "datasetInstruments" - ] = [] - parameter_entity = models.Parameter.from_icat(icat_data, ["dataset"]) + parameter_entity = models.Parameter.from_icat(icat_data, ["document"]) assert parameter_entity.dict(by_alias=True) == expected_entity_data @@ -613,13 +602,7 @@ def test_from_icat_multiple_and_nested_relations(self): expected_entity_data["members"][0]["affiliation"] = AFFILIATION_PANOSC_DATA expected_entity_data["members"][0]["person"] = PERSON_PANOSC_DATA expected_entity_data["parameters"] = [PARAMETER_PANOSC_DATA.copy()] - expected_entity_data["parameters"][0]["value"] - expected_entity_data["parameters"][0]["dataset"] = DATASET_PANOSC_DATA.copy() - expected_entity_data["parameters"][0]["document"] = DOCUMENT_PANOSC_DATA.copy() - expected_entity_data["parameters"][0]["document"]["keywords"] = [] - expected_entity_data["parameters"][0]["dataset"]["techniques"] = [ - TECHNIQUE_PANOSC_DATA, - ] + expected_entity_data["parameters"][0]["document"] = DOCUMENT_PANOSC_DATA icat_data = INVESTIGATION_ICAT_DATA.copy() icat_data["type"] = INVESTIGATION_TYPE_ICAT_DATA @@ -645,17 +628,7 @@ def test_from_icat_multiple_and_nested_relations(self): icat_data["parameters"][0]["investigation"][ "type" ] = INVESTIGATION_TYPE_ICAT_DATA - dataset_with_techniques_icat = DATASET_ICAT_DATA.copy() - dataset_with_techniques_icat.update( - {"datasetTechniques": [{"technique": TECHNIQUE_ICAT_DATA}]}, - ) - icat_data["parameters"][0]["investigation"]["investigationInstruments"] = [ - { - "instrument": { - "datasetInstruments": [{"dataset": dataset_with_techniques_icat}], - }, - }, - ] + icat_data["parameters"][0]["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] relations = [ "datasets.instrument", @@ -664,7 +637,7 @@ def test_from_icat_multiple_and_nested_relations(self): "datasets.samples", "members.affiliation", "members.person", - "parameters.dataset.techniques", + "parameters.document", ] document_entity = models.Document.from_icat(icat_data, relations) From 0bbcc0d4f762c52b9fa4a4b558fff1a44fbd40b7 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 11:00:55 +0000 Subject: [PATCH 24/41] refactor: replace `fromisoformat` with `str_to_datetime_object` #265 --- datagateway_api/src/search_api/models.py | 5 +++-- test/search_api/test_models.py | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index 16f341f5..504478ab 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, ValidationError, validator from pydantic.error_wrappers import ErrorWrapper +from datagateway_api.src.common.date_handler import DateHandler from datagateway_api.src.search_api.panosc_mappings import mappings @@ -174,7 +175,7 @@ def set_is_public(cls, value): # noqa: B902, N805 if not value: return value - creation_date = datetime.fromisoformat(value) + creation_date = DateHandler.str_to_datetime_object(value) current_datetime = datetime.now(timezone.utc) three_years_ago = current_datetime - relativedelta(years=3) return creation_date < three_years_ago @@ -213,7 +214,7 @@ def set_is_public(cls, value): # noqa: B902, N805 if not value: return value - creation_date = datetime.fromisoformat(value) + creation_date = DateHandler.str_to_datetime_object(value) current_datetime = datetime.now(timezone.utc) three_years_ago = current_datetime - relativedelta(years=3) return creation_date < three_years_ago diff --git a/test/search_api/test_models.py b/test/search_api/test_models.py index 3c4e04a0..66eb1485 100644 --- a/test/search_api/test_models.py +++ b/test/search_api/test_models.py @@ -3,6 +3,7 @@ from pydantic import ValidationError import pytest +from datagateway_api.src.common.date_handler import DateHandler import datagateway_api.src.search_api.models as models @@ -199,7 +200,9 @@ DATASET_PANOSC_DATA = { "pid": DATASET_ICAT_DATA["doi"], "title": DATASET_ICAT_DATA["name"], - "creationDate": datetime.fromisoformat(DATASET_ICAT_DATA["createTime"]), + "creationDate": DateHandler.str_to_datetime_object( + DATASET_ICAT_DATA["createTime"], + ).replace(tzinfo=timezone.utc), "isPublic": True, "size": None, "documents": [], @@ -217,9 +220,15 @@ "title": INVESTIGATION_ICAT_DATA["name"], "summary": INVESTIGATION_ICAT_DATA["summary"], "doi": INVESTIGATION_ICAT_DATA["doi"], - "startDate": datetime.fromisoformat(INVESTIGATION_ICAT_DATA["startDate"]), - "endDate": datetime.fromisoformat(INVESTIGATION_ICAT_DATA["endDate"]), - "releaseDate": datetime.fromisoformat(INVESTIGATION_ICAT_DATA["releaseDate"]), + "startDate": DateHandler.str_to_datetime_object( + INVESTIGATION_ICAT_DATA["startDate"], + ).replace(tzinfo=timezone.utc), + "endDate": DateHandler.str_to_datetime_object( + INVESTIGATION_ICAT_DATA["endDate"], + ).replace(tzinfo=timezone.utc), + "releaseDate": DateHandler.str_to_datetime_object( + INVESTIGATION_ICAT_DATA["releaseDate"], + ).replace(tzinfo=timezone.utc), "license": None, "keywords": [KEYWORD_ICAT_DATA["name"]], "datasets": [], From 299cbcdd46672eb4424066d44891e03fd7c41115 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Fri, 28 Jan 2022 12:07:20 +0000 Subject: [PATCH 25/41] Apply suggestions from code review #265 Co-authored-by: Matthew Richards <32678030+MRichards99@users.noreply.github.com> --- datagateway_api/src/search_api/models.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index 16f341f5..a2a2211d 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -1,5 +1,4 @@ -import abc -from abc import ABC +from abc import ABC, abstractmethod from datetime import datetime, timezone import sys from typing import ClassVar, List, Optional, Union @@ -33,7 +32,7 @@ def _get_icat_field_value(icat_field_name, icat_data): class PaNOSCAttribute(ABC, BaseModel): @classmethod - @abc.abstractmethod + @abstractmethod def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 model_fields = cls.__fields__ @@ -54,9 +53,8 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 field_value = None for field_name in icat_field_name: try: - value = _get_icat_field_value(field_name, icat_data) - if value: - field_value = value + field_value = _get_icat_field_value(field_name, icat_data) + if field_value: break except KeyError: continue From 3f1b1cffdd1e57ab4eb1227b13e0906424adefd0 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 12:58:47 +0000 Subject: [PATCH 26/41] docs: add new comments and fix existing #265 --- datagateway_api/src/search_api/models.py | 62 ++++++++++++++++-------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index e16b559b..dea517a2 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -39,9 +39,9 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 model_data = {} for field in model_fields: - # Some fields have aliases so we must use them when creating a model instance. - # If a field does not have an alias then the `alias` property holds the name - # of the field + # Some fields have aliases so we must use them when creating a model + # instance. If a field does not have an alias then the `alias` property + # holds the name of the field field_alias = cls.__fields__[field].alias panosc_entity_name, icat_field_name = mappings.get_icat_mapping( @@ -58,16 +58,26 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 if field_value: break except KeyError: + # If an icat value cannot be found for the ICAT field name in the + # provided ICAT data then ignore the error. The field name could + # simply be a mapping of an optional PaNOSC entity field so ICAT + # may not return data for it which is fine. It could also be a list + # of mappings which is the case with the `value` field of the + # PaNOSC entity. When this is the case, ICAT only returns data for + # one of the mappings from the list so we can ignore the error. + # This also ignores errors for mandatory fields but this is not a + # problem because pydantic is responsible for validating whether + # data for mandatory fields is missing. continue if not field_value: continue if panosc_entity_name != cls.__name__: - # If we are here, it means that the field references another model so we - # have to get hold of its class definition and call its `from_icat` method - # to create an instance of itself with the ICAT data provided. Doing this - # allows for recursion. + # If we are here, it means that the field references another model so + # we have to get hold of its class definition and call its `from_icat` + # method to create an instance of itself with the ICAT data provided. + # Doing this allows for recursion. data = field_value if not isinstance(data, list): data = [data] @@ -113,9 +123,10 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 and required_related_field not in model_data ): # If we are here, it means that a related entity, which has a minimum - # cardinality of one, has been specified to be included as part of the entity - # but the relevant ICAT data needed for its creation cannot be found in the - # provided ICAT response. Because of this, a ValidationError is raised. + # cardinality of one, has been specified to be included as part of the + # entity but the relevant ICAT data needed for its creation cannot be + # found in the provided ICAT response. Because of this, a + # `ValidationError` is raised. error_wrapper = ErrorWrapper( TypeError("field required"), loc=required_related_field, ) @@ -292,16 +303,27 @@ class Parameter(PaNOSCAttribute): dataset: Optional[Dataset] = None document: Optional[Document] = None - # @root_validator(skip_on_failure=True) - # def validate_dataset_and_document(cls, values): # noqa: B902, N805 - # if values["dataset"] is None and values["document"] is None: - # raise TypeError("must have a dataset or document") - - # if values["dataset"] is not None and values["document"] is not None: - # # TODO - Should an exception be raised here instead? - # values["Document"] = None - - # return values + """ + Validator commented as it was decided to be disabled for the time being. The Data + Model states that a Parameter must be related to a Dataset or Document, however + considering that there is not a Parameter endpoint, it means that a Parameter can + only be included via Dataset or Document. It's unclear why anyone would query for + a Dataset or Document that includes Parameters which in turn includes a Dataset or + Document that are the same as the top level ones. To avoid errors being raised + as a result of Parameters not containing ICAT data for a Dataset or Document, the + validator has been disabled. + + @root_validator(skip_on_failure=True) + def validate_dataset_and_document(cls, values): # noqa: B902, N805 + if values["dataset"] is None and values["document"] is None: + raise TypeError("must have a dataset or document") + + if values["dataset"] is not None and values["document"] is not None: + # TODO - Should an exception be raised here instead? + values["Document"] = None + + return values + """ @classmethod def from_icat(cls, icat_data, required_related_fields): From 1d730d1d07a523d2b636d905911a5073aa51d305 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 12:59:43 +0000 Subject: [PATCH 27/41] refactor: refactor `_get_icat_field_value` logic #265 --- datagateway_api/src/search_api/models.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index dea517a2..81f8d568 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -18,14 +18,10 @@ def _get_icat_field_value(icat_field_name, icat_data): values = [] for data in icat_data: value = _get_icat_field_value(field_name, data) - if isinstance(value, list): - values.extend(value) - else: - values.append(value) - + value = [value] if not isinstance(value, list) else value + values.extend(value) icat_data = values - - if isinstance(icat_data, dict): + elif isinstance(icat_data, dict): icat_data = icat_data[field_name] return icat_data From 557172212c07efea7a7d7ea85793446cc0f80566 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 13:28:40 +0000 Subject: [PATCH 28/41] test: remove code for replacing datetime timezone #265 --- test/search_api/test_models.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/search_api/test_models.py b/test/search_api/test_models.py index 66eb1485..df6aa84b 100644 --- a/test/search_api/test_models.py +++ b/test/search_api/test_models.py @@ -202,7 +202,7 @@ "title": DATASET_ICAT_DATA["name"], "creationDate": DateHandler.str_to_datetime_object( DATASET_ICAT_DATA["createTime"], - ).replace(tzinfo=timezone.utc), + ), "isPublic": True, "size": None, "documents": [], @@ -222,13 +222,11 @@ "doi": INVESTIGATION_ICAT_DATA["doi"], "startDate": DateHandler.str_to_datetime_object( INVESTIGATION_ICAT_DATA["startDate"], - ).replace(tzinfo=timezone.utc), - "endDate": DateHandler.str_to_datetime_object( - INVESTIGATION_ICAT_DATA["endDate"], - ).replace(tzinfo=timezone.utc), + ), + "endDate": DateHandler.str_to_datetime_object(INVESTIGATION_ICAT_DATA["endDate"]), "releaseDate": DateHandler.str_to_datetime_object( INVESTIGATION_ICAT_DATA["releaseDate"], - ).replace(tzinfo=timezone.utc), + ), "license": None, "keywords": [KEYWORD_ICAT_DATA["name"]], "datasets": [], From 71765a590d996b9935299ebfb3de305660542ca1 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 13:41:40 +0000 Subject: [PATCH 29/41] refactor: refactor `from_icat` abstract method #265 --- datagateway_api/src/search_api/models.py | 44 +++++++++++------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index 81f8d568..f67e7f04 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -31,17 +31,17 @@ class PaNOSCAttribute(ABC, BaseModel): @classmethod @abstractmethod def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 - model_fields = cls.__fields__ + entity_fields = cls.__fields__ - model_data = {} - for field in model_fields: + entity_data = {} + for entity_field in entity_fields: # Some fields have aliases so we must use them when creating a model # instance. If a field does not have an alias then the `alias` property # holds the name of the field - field_alias = cls.__fields__[field].alias + entity_field_alias = cls.__fields__[entity_field].alias - panosc_entity_name, icat_field_name = mappings.get_icat_mapping( - cls.__name__, field_alias, + entity_name, icat_field_name = mappings.get_icat_mapping( + cls.__name__, entity_field_alias, ) if not isinstance(icat_field_name, list): @@ -69,54 +69,52 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 if not field_value: continue - if panosc_entity_name != cls.__name__: + if entity_name != cls.__name__: # If we are here, it means that the field references another model so # we have to get hold of its class definition and call its `from_icat` # method to create an instance of itself with the ICAT data provided. # Doing this allows for recursion. - data = field_value - if not isinstance(data, list): - data = [data] + data = ( + [field_value] if not isinstance(field_value, list) else field_value + ) required_related_fields_for_next_entity = [] for required_related_field in required_related_fields: required_related_field = required_related_field.split(".") if ( len(required_related_field) > 1 - and field_alias in required_related_field + and entity_field_alias in required_related_field ): required_related_fields_for_next_entity.extend( required_related_field[1:], ) - # Get the class of the referenced model - panosc_model_attr = getattr(sys.modules[__name__], panosc_entity_name) + # Get the class of the referenced entity + entity_attr = getattr(sys.modules[__name__], entity_name) field_value = [ - panosc_model_attr.from_icat( - d, required_related_fields_for_next_entity, - ) + entity_attr.from_icat(d, required_related_fields_for_next_entity) for d in data ] - field_outer_type = cls.__fields__[field].outer_type_ + entity_field_outer_type = cls.__fields__[entity_field].outer_type_ if ( - not hasattr(field_outer_type, "_name") - or field_outer_type._name != "List" + not hasattr(entity_field_outer_type, "_name") + or entity_field_outer_type._name != "List" ) and isinstance(field_value, list): # If the field does not hold list of values but `field_value` # is a list, then just get its first element field_value = field_value[0] - model_data[field_alias] = field_value + entity_data[entity_field_alias] = field_value for required_related_field in required_related_fields: required_related_field = required_related_field.split(".")[0] if ( - required_related_field in model_fields + required_related_field in entity_fields and required_related_field in cls._related_fields_with_min_cardinality_one - and required_related_field not in model_data + and required_related_field not in entity_data ): # If we are here, it means that a related entity, which has a minimum # cardinality of one, has been specified to be included as part of the @@ -128,7 +126,7 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 ) raise ValidationError(errors=[error_wrapper], model=cls) - return cls(**model_data) + return cls(**entity_data) class Affiliation(PaNOSCAttribute): From 8e472cff813cd2b5953824c763909dd904276321 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Fri, 28 Jan 2022 15:54:52 +0000 Subject: [PATCH 30/41] build: upgrade `python-icat` to 0.21.0 #305 --- poetry.lock | 67 +++++++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index 12bedca2..a7f7eafc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,12 +18,12 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock", "flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)", "tox"] +tests = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock"] docs = ["marshmallow (>=2.19.2)", "pyyaml (==5.3)", "sphinx (==2.4.1)", "sphinx-issues (==1.2.0)", "sphinx-rtd-theme (==0.4.3)"] lint = ["flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)"] -tests = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock"] -validation = ["prance[osv] (>=0.11)"] +dev = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock", "flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)", "tox"] yaml = ["PyYAML (>=3.10)"] +validation = ["prance[osv] (>=0.11)"] [[package]] name = "appdirs" @@ -50,10 +50,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] [[package]] name = "bandit" @@ -198,11 +198,11 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] @@ -409,8 +409,8 @@ Jinja2 = ">=3.0" Werkzeug = ">=2.0" [package.extras] -async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] +async = ["asgiref (>=3.2)"] [[package]] name = "flask-cors" @@ -528,8 +528,8 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -perf = ["ipython"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +perf = ["ipython"] [[package]] name = "iniconfig" @@ -666,8 +666,8 @@ click = ">=7" six = "*" [package.extras] -coverage = ["pytest-cov"] testing = ["mock", "pytest", "pytest-rerunfailures"] +coverage = ["pytest-cov"] [[package]] name = "pkginfo" @@ -692,8 +692,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +dev = ["pre-commit", "tox"] [[package]] name = "pprintpp" @@ -862,12 +862,12 @@ requests = ">=2.25.0" requests-toolbelt = ">=0.9.1" [package.extras] -autocompletion = ["argcomplete (>=1.10.0,<2)"] yaml = ["PyYaml (>=5.2)"] +autocompletion = ["argcomplete (>=1.10.0,<2)"] [[package]] name = "python-icat" -version = "0.20.0" +version = "0.21.0" description = "Python interface to ICAT and IDS" category = "main" optional = false @@ -894,10 +894,10 @@ tomlkit = "0.7.0" twine = ">=3,<4" [package.extras] -dev = ["tox", "isort", "black"] +test = ["coverage (>=5,<6)", "pytest (>=5,<6)", "pytest-xdist (>=1,<2)", "pytest-mock (>=2,<3)", "responses (==0.13.3)", "mock (==1.3.0)"] docs = ["Sphinx (==1.3.6)"] mypy = ["mypy", "types-requests"] -test = ["coverage (>=5,<6)", "pytest (>=5,<6)", "pytest-xdist (>=1,<2)", "pytest-mock (>=2,<3)", "responses (==0.13.3)", "mock (==1.3.0)"] +dev = ["tox", "isort", "black"] [[package]] name = "pytz" @@ -962,8 +962,8 @@ idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "requests-toolbelt" @@ -1065,25 +1065,25 @@ greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platfo importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] -aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] -asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3)"] -mariadb_connector = ["mariadb (>=1.0.1)"] mssql = ["pyodbc"] -mssql_pymssql = ["pymssql"] -mssql_pyodbc = ["pyodbc"] mypy = ["sqlalchemy2-stubs", "mypy (>=0.910)"] -mysql = ["mysqlclient (>=1.4.0,<2)", "mysqlclient (>=1.4.0)"] -mysql_connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] postgresql = ["psycopg2 (>=2.7)"] postgresql_asyncpg = ["greenlet (!=0.4.17)", "asyncpg"] -postgresql_pg8000 = ["pg8000 (>=1.16.6)"] -postgresql_psycopg2binary = ["psycopg2-binary"] postgresql_psycopg2cffi = ["psycopg2cffi"] +mysql_connector = ["mysql-connector-python"] pymysql = ["pymysql (<1)", "pymysql"] +mssql_pymssql = ["pymssql"] +oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] +postgresql_pg8000 = ["pg8000 (>=1.16.6)"] +aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] +asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3)"] +postgresql_psycopg2binary = ["psycopg2-binary"] +mysql = ["mysqlclient (>=1.4.0,<2)", "mysqlclient (>=1.4.0)"] sqlcipher = ["sqlcipher3-binary"] +aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] +mariadb_connector = ["mariadb (>=1.0.1)"] +mssql_pyodbc = ["pyodbc"] +asyncio = ["greenlet (!=0.4.17)"] [[package]] name = "stevedore" @@ -1149,9 +1149,9 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] telegram = ["requests"] +notebook = ["ipywidgets (>=6)"] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] [[package]] name = "twine" @@ -1238,7 +1238,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<4.0" -content-hash = "821a222ad69359ec266eff8d281ba0b3fe78c5249ff0d703f9359afb4c89efba" +content-hash = "489cc7708ea5d15d5c47bc7f4c21298761b4250ea93abdfcc7f1b7e85c7785d3" [metadata.files] aniso8601 = [ @@ -1525,6 +1525,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58"}, + {file = "greenlet-1.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965"}, {file = "greenlet-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708"}, {file = "greenlet-1.1.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23"}, {file = "greenlet-1.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee"}, @@ -1537,6 +1538,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168"}, + {file = "greenlet-1.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f"}, {file = "greenlet-1.1.2-cp36-cp36m-win32.whl", hash = "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa"}, {file = "greenlet-1.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d"}, {file = "greenlet-1.1.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4"}, @@ -1545,6 +1547,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5"}, + {file = "greenlet-1.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe"}, {file = "greenlet-1.1.2-cp37-cp37m-win32.whl", hash = "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc"}, {file = "greenlet-1.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06"}, {file = "greenlet-1.1.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0"}, @@ -1553,6 +1556,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b"}, + {file = "greenlet-1.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2"}, {file = "greenlet-1.1.2-cp38-cp38-win32.whl", hash = "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd"}, {file = "greenlet-1.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3"}, {file = "greenlet-1.1.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67"}, @@ -1561,6 +1565,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3"}, + {file = "greenlet-1.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3"}, {file = "greenlet-1.1.2-cp39-cp39-win32.whl", hash = "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf"}, {file = "greenlet-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd"}, {file = "greenlet-1.1.2.tar.gz", hash = "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a"}, @@ -1784,7 +1789,7 @@ python-gitlab = [ {file = "python_gitlab-2.10.1-py3-none-any.whl", hash = "sha256:581a219759515513ea9399e936ed7137437cfb681f52d2641626685c492c999d"}, ] python-icat = [ - {file = "python-icat-0.20.0.tar.gz", hash = "sha256:d09e85f0269ac5f8f6a43f2365eb269b532e611a5eaa3ab23ad7f742ee2bfa25"}, + {file = "python-icat-0.21.0.tar.gz", hash = "sha256:fd4d1515b8075677ee3672274a55d6ca287ce1cfa4e17b6b25371904764be999"}, ] python-semantic-release = [ {file = "python-semantic-release-7.19.2.tar.gz", hash = "sha256:8ca0e5f72d31e7b0603b95caad6fb2d5315483ac1fadd86648771966d9ec6f2c"}, diff --git a/pyproject.toml b/pyproject.toml index 62c96806..8f996076 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" PyYAML = "5.4" -python-icat = "0.20.0" +python-icat = "0.21.0" suds-community = "^0.8.4" py-object-pool = "^1.1" cachetools = "^4.2.1" From 691a59ea3f850475572c3a877fb739e5216c6fe7 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 31 Jan 2022 13:39:21 +0000 Subject: [PATCH 31/41] fix: fix list type field checking in Python 3.6 #265 --- datagateway_api/src/search_api/models.py | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index f67e7f04..2781dec1 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -11,6 +11,25 @@ from datagateway_api.src.search_api.panosc_mappings import mappings +def _is_panosc_entity_field_of_type_list(entity_field): + entity_field_outer_type = entity_field.outer_type_ + if ( + hasattr(entity_field_outer_type, "_name") + and entity_field_outer_type._name == "List" + ): + is_list = True + # The `_name` `outer_type_` attribute was introduced in Python 3.7 so to check + # whether the field is of type list in Python 3.6, we are checking the type of its + # defualt value. We must ensure that any new list fields that get added in future + # are assigned a list by default. + elif isinstance(entity_field.default, list): + is_list = True + else: + is_list = False + + return is_list + + def _get_icat_field_value(icat_field_name, icat_data): icat_field_name = icat_field_name.split(".") for field_name in icat_field_name: @@ -96,10 +115,8 @@ def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 for d in data ] - entity_field_outer_type = cls.__fields__[entity_field].outer_type_ - if ( - not hasattr(entity_field_outer_type, "_name") - or entity_field_outer_type._name != "List" + if not _is_panosc_entity_field_of_type_list( + cls.__fields__[entity_field], ) and isinstance(field_value, list): # If the field does not hold list of values but `field_value` # is a list, then just get its first element From a2534223af50b68c4195f843e8bac13d8d988bef Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 31 Jan 2022 13:40:57 +0000 Subject: [PATCH 32/41] style: address linting warnings #265 --- datagateway_api/src/search_api/filters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datagateway_api/src/search_api/filters.py b/datagateway_api/src/search_api/filters.py index 1ebbf579..cd840204 100644 --- a/datagateway_api/src/search_api/filters.py +++ b/datagateway_api/src/search_api/filters.py @@ -3,7 +3,6 @@ import logging from datagateway_api.src.common.date_handler import DateHandler -from datagateway_api.src.common.exceptions import FilterError from datagateway_api.src.datagateway_api.icat.filters import ( PythonICATIncludeFilter, PythonICATLimitFilter, From 31fa4ca3ece57fc19993bcd9dee0547b2b600e19 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 31 Jan 2022 15:30:28 +0000 Subject: [PATCH 33/41] 3.2.0 Automatically generated by python-semantic-release --- CHANGELOG.md | 5 +++++ pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec3063b..808aaddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ +## v3.2.0 (2022-01-31) +### Feature +* Add class to represent nested conditions #259 ([`583cbf2`](https://github.com/ral-facilities/datagateway-api/commit/583cbf29744b72c020429b61ae7442b19acef231)) +* Add first pass of query param implementation #259 ([`ee668e3`](https://github.com/ral-facilities/datagateway-api/commit/ee668e38cd43354851163616a93924ad84e14b90)) + ## v3.1.1 (2021-12-15) ### Fix * Correct reference to class name #264 ([`fc4c180`](https://github.com/ral-facilities/datagateway-api/commit/fc4c18085ab496d838e8d1e9e3f8c77f07826e9d)) diff --git a/pyproject.toml b/pyproject.toml index 8f996076..173742a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datagateway-api" -version = "3.1.1" +version = "3.2.0" description = "ICAT API to interface with the DataGateway" license = "Apache-2.0" readme = "README.md" From 2f0afa05101b411510bbb5f15c91c78fdc9db243 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 31 Jan 2022 16:09:24 +0000 Subject: [PATCH 34/41] test: tweak query params tests to avoid Parameter entity #259 --- .../test_search_api_query_filter_factory.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/search_api/test_search_api_query_filter_factory.py b/test/search_api/test_search_api_query_filter_factory.py index 90708c7c..5f836681 100644 --- a/test/search_api/test_search_api_query_filter_factory.py +++ b/test/search_api/test_search_api_query_filter_factory.py @@ -142,10 +142,10 @@ def test_valid_where_filter_text_operator( id="Multiple conditions (three), property values with no operator", ), pytest.param( - {"filter": {"where": {"and": [{"value": {"lt": 50}}]}}}, - "Parameter", + {"filter": {"where": {"and": [{"size": {"lt": 50}}]}}}, + "File", [], - [SearchAPIWhereFilter("value", 50, "lt")], + [SearchAPIWhereFilter("size", 50, "lt")], "and", id="Single condition, property value with operator", ), @@ -154,15 +154,15 @@ def test_valid_where_filter_text_operator( "filter": { "where": { "and": [ - {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, + {"role": {"like": "Test role"}}, + {"size": {"gte": 275}}, ], }, }, }, - "Parameter", - [SearchAPIWhereFilter("name", "Test name", "like")], - [SearchAPIWhereFilter("value", 275, "gte")], + "Member", + [SearchAPIWhereFilter("role", "Test role", "like")], + [SearchAPIWhereFilter("size", 275, "gte")], "and", id="Multiple conditions (two), property values with operator", ), @@ -172,18 +172,18 @@ def test_valid_where_filter_text_operator( "where": { "and": [ {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, - {"unit": {"nlike": "Test unit"}}, + {"size": {"gte": 275}}, + {"path": {"nlike": "Test path"}}, ], }, }, }, - "Parameter", + "File", [ SearchAPIWhereFilter("name", "Test name", "like"), - SearchAPIWhereFilter("value", 275, "gte"), + SearchAPIWhereFilter("size", 275, "gte"), ], - [SearchAPIWhereFilter("unit", "Test unit", "nlike")], + [SearchAPIWhereFilter("path", "Test path", "nlike")], "and", id="Multiple conditions (three), property values with operator", ), @@ -371,10 +371,10 @@ def test_valid_where_filter_with_and_boolean_operator( id="Multiple conditions (three), property values with no operator", ), pytest.param( - {"filter": {"where": {"or": [{"value": {"lt": 50}}]}}}, - "Parameter", + {"filter": {"where": {"or": [{"size": {"lt": 50}}]}}}, + "File", [], - [SearchAPIWhereFilter("value", 50, "lt")], + [SearchAPIWhereFilter("size", 50, "lt")], "or", id="Single condition, property value with operator", ), @@ -384,14 +384,14 @@ def test_valid_where_filter_with_and_boolean_operator( "where": { "or": [ {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, + {"size": {"gte": 275}}, ], }, }, }, - "Parameter", + "File", [SearchAPIWhereFilter("name", "Test name", "like")], - [SearchAPIWhereFilter("value", 275, "gte")], + [SearchAPIWhereFilter("size", 275, "gte")], "or", id="Multiple conditions (two), property values with operator", ), @@ -401,18 +401,18 @@ def test_valid_where_filter_with_and_boolean_operator( "where": { "or": [ {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, - {"unit": {"nlike": "Test unit"}}, + {"size": {"gte": 275}}, + {"path": {"nlike": "Test path"}}, ], }, }, }, - "Parameter", + "File", [ SearchAPIWhereFilter("name", "Test name", "like"), - SearchAPIWhereFilter("value", 275, "gte"), + SearchAPIWhereFilter("size", 275, "gte"), ], - [SearchAPIWhereFilter("unit", "Test unit", "nlike")], + [SearchAPIWhereFilter("path", "Test path", "nlike")], "or", id="Multiple conditions (three), property values with operator", ), From f94d361d3050dcfa62a2a600917cb59721715e3f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 31 Jan 2022 16:25:52 +0000 Subject: [PATCH 35/41] build: update example mappings --- datagateway_api/search_api_mapping.json.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index f93c2a04..643570bf 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -65,10 +65,10 @@ "Parameter": { "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", - "name": "type.name", + "name": "name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", - "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, + "dataset": {"Dataset": "investigation.investigationInstruments.instrument.datasetInstruments.dataset"}, "document": {"Document": "investigation"} }, "Person": { From 3f1b2d6a1d216d9596e5cda2c261e0911324f541 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 31 Jan 2022 16:31:45 +0000 Subject: [PATCH 36/41] style: fix typo #265 --- datagateway_api/src/search_api/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index 2781dec1..f9c8ae9a 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -20,7 +20,7 @@ def _is_panosc_entity_field_of_type_list(entity_field): is_list = True # The `_name` `outer_type_` attribute was introduced in Python 3.7 so to check # whether the field is of type list in Python 3.6, we are checking the type of its - # defualt value. We must ensure that any new list fields that get added in future + # default value. We must ensure that any new list fields that get added in future # are assigned a list by default. elif isinstance(entity_field.default, list): is_list = True From 63d7c21e87f1a793b309f9c031d2a921d8f3ede8 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 31 Jan 2022 16:49:52 +0000 Subject: [PATCH 37/41] test: update mappings on tests #260 --- test/search_api/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/search_api/conftest.py b/test/search_api/conftest.py index e5bfb508..e507ec19 100644 --- a/test/search_api/conftest.py +++ b/test/search_api/conftest.py @@ -89,7 +89,7 @@ def test_search_api_mappings_data(): "affiliation": {"Affiliation": "user.dataPublicationUsers.affiliations"}, }, "Parameter": { - "base_icat_entity": "InvestigationParameter", + "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", "name": "name", "value": ["numericValue", "stringValue", "dateTimeValue"], From 6c9780a3337823c95979b8bdff3ccf6b9a3a9fb6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 31 Jan 2022 17:30:04 +0000 Subject: [PATCH 38/41] 3.3.0 Automatically generated by python-semantic-release --- CHANGELOG.md | 12 ++++++++++++ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 808aaddf..97cd63f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ +## v3.3.0 (2022-01-31) +### Feature +* Add function to get PaNOSC to ICAT mapping for where filter #260 ([`34b1d81`](https://github.com/ral-facilities/datagateway-api/commit/34b1d819482aa3efdb4f8da321125d3e40d76617)) +* Convert PaNOSC to ICAT for where filter fields #260 ([`ff9595d`](https://github.com/ral-facilities/datagateway-api/commit/ff9595d2f571211db79dea02f702d4148b8879f3)) + +### Fix +* Fix example mapping file ([`3802cc9`](https://github.com/ral-facilities/datagateway-api/commit/3802cc9ee71a355d0ad87529f65112e8c3f8b881)) +* Update `__str__()` for WHERE filter to cope with applying filter #260 ([`8d259d7`](https://github.com/ral-facilities/datagateway-api/commit/8d259d75a28414f26cc569293720ef4e306e6844)) + +### Documentation +* Add docstring to static function #260 ([`618f6b9`](https://github.com/ral-facilities/datagateway-api/commit/618f6b9fead88f61a346b90cb2b85a90877b0410)) + ## v3.2.0 (2022-01-31) ### Feature * Add class to represent nested conditions #259 ([`583cbf2`](https://github.com/ral-facilities/datagateway-api/commit/583cbf29744b72c020429b61ae7442b19acef231)) diff --git a/pyproject.toml b/pyproject.toml index 173742a5..6fbb6040 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datagateway-api" -version = "3.2.0" +version = "3.3.0" description = "ICAT API to interface with the DataGateway" license = "Apache-2.0" readme = "README.md" From 358ac22504b6869686c4caf6b47282fda916cb60 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Mon, 31 Jan 2022 17:53:36 +0000 Subject: [PATCH 39/41] refactor: update PaNOSC Parameter name mapping in example file #265 --- datagateway_api/search_api_mapping.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index 02b4ffc8..19a81cfa 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -65,7 +65,7 @@ "Parameter": { "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", - "name": "name", + "name": "type.name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", "dataset": {"Dataset": "dataset"}, From 63f4f172e26e6284dce5abaea9e04aac53e3e560 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 31 Jan 2022 17:57:06 +0000 Subject: [PATCH 40/41] 3.4.0 Automatically generated by python-semantic-release --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97cd63f0..8deb20a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ +## v3.4.0 (2022-01-31) +### Feature +* Implement `regexp` operator #297 ([`bf3fe0e`](https://github.com/ral-facilities/datagateway-api/commit/bf3fe0ef2ac582d55dbd881edf6a81a93625ce91)) +* Implement `neq` operator #297 ([`9094bbb`](https://github.com/ral-facilities/datagateway-api/commit/9094bbb894ead20a53fadfd0e24b264af29548b9)) +* Implement `nin` operator #297 ([`00dbba5`](https://github.com/ral-facilities/datagateway-api/commit/00dbba525d5cd86cb5577f3b1621a7042cdd2fa0)) +* Implement `inq` operator #297 ([`fc1cf19`](https://github.com/ral-facilities/datagateway-api/commit/fc1cf194454a4da60652b1f68df278c4624ddc11)) +* Implement `between` operator #297 ([`4736888`](https://github.com/ral-facilities/datagateway-api/commit/4736888bf76cda0dbc00f997443ed565f0f5e760)) + ## v3.3.0 (2022-01-31) ### Feature * Add function to get PaNOSC to ICAT mapping for where filter #260 ([`34b1d81`](https://github.com/ral-facilities/datagateway-api/commit/34b1d819482aa3efdb4f8da321125d3e40d76617)) diff --git a/pyproject.toml b/pyproject.toml index 6fbb6040..a8663bbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datagateway-api" -version = "3.3.0" +version = "3.4.0" description = "ICAT API to interface with the DataGateway" license = "Apache-2.0" readme = "README.md" From 58e5f208a9602a45040d608edeebf62d4f60cba7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 31 Jan 2022 19:05:06 +0000 Subject: [PATCH 41/41] 3.5.0 Automatically generated by python-semantic-release --- CHANGELOG.md | 10 ++++++++++ pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8deb20a2..549a5776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ +## v3.5.0 (2022-01-31) +### Feature +* Implement basic version of `SearchAPIIncludeFilter` #261 ([`f2f53c9`](https://github.com/ral-facilities/datagateway-api/commit/f2f53c92229d052ae697787eb80a35dcd2ea3b45)) + +### Fix +* Fix list type field checking in Python 3.6 #265 ([`691a59e`](https://github.com/ral-facilities/datagateway-api/commit/691a59ea3f850475572c3a877fb739e5216c6fe7)) + +### Documentation +* Add new comments and fix existing #265 ([`3f1b1cf`](https://github.com/ral-facilities/datagateway-api/commit/3f1b1cffdd1e57ab4eb1227b13e0906424adefd0)) + ## v3.4.0 (2022-01-31) ### Feature * Implement `regexp` operator #297 ([`bf3fe0e`](https://github.com/ral-facilities/datagateway-api/commit/bf3fe0ef2ac582d55dbd881edf6a81a93625ce91)) diff --git a/pyproject.toml b/pyproject.toml index a8663bbd..51ea7c1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datagateway-api" -version = "3.4.0" +version = "3.5.0" description = "ICAT API to interface with the DataGateway" license = "Apache-2.0" readme = "README.md"