From 4736888bf76cda0dbc00f997443ed565f0f5e760 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:18:39 +0000 Subject: [PATCH 01/17] feat: implement `between` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 9877938a..a20a2e5a 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -88,6 +88,10 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) + elif self.operation == "between": + where_filter = self.create_condition( + self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", + ) else: raise FilterError(f"Bad operation given to where filter: {self.operation}") @@ -116,8 +120,7 @@ def create_condition(attribute_name, operator, value): # distinct filter is used in a request jpql_value = ( f"{value}" - if operator == "in" - or operator == "!=" + if operator in ("in", "not in", "!=", "between") or str(value).startswith("UPPER") or "o." in str(value) else f"'{value}'" From d254642c4cf28168c47c8eacdc8fcbf90e1c1943 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:21:45 +0000 Subject: [PATCH 02/17] validate values supplied with `between` operator #297 --- datagateway_api/src/common/filters.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index c99e26eb..623ec393 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -34,6 +34,18 @@ def __init__(self, field, value, operation): " be in a list format e.g. [1, 2, 3]", ) + if self.operation == "between": + if not isinstance(self.value, list): + raise BadRequestError( + "When using the 'between' operation for a WHERE filter, the values" + " must be in a list format e.g. [1, 2]", + ) + if isinstance(self.value, list) and len(self.value) != 2: + raise BadRequestError( + "When using the 'between' operation for a WHERE filter, the list" + "must contain two values e.g. [1, 2]", + ) + class DistinctFieldFilter(QueryFilter): precedence = 0 From f5525c9230240671744534f34919d9511a118374 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:22:39 +0000 Subject: [PATCH 03/17] test: test `between` operator #297 --- .../icat/filters/test_where_filter.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index bc41a5fa..4cc5ce50 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -23,6 +23,7 @@ class TestICATWhereFilter: pytest.param("gte", 5, ["%s >= '5'"], id="greater than or equal"), pytest.param("in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list"), pytest.param("in", [], ["%s in (NULL)"], id="empty list"), + pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), ], ) def test_valid_operations( @@ -33,9 +34,23 @@ def test_valid_operations( assert icat_query.conditions == {"id": expected_condition_value} - def test_invalid_in_operation(self, icat_query): + @pytest.mark.parametrize( + "operation, value", + [ + pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), + pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), + pytest.param("between", [], id="between - empty list"), + pytest.param( + "between", [1], id="between - list with less than two elements", + ), + pytest.param( + "between", [1, 2, 3], id="between - list with more than two elements", + ), + ], + ) + def test_invalid_operations_raise_bad_request_error(self, operation, value): with pytest.raises(BadRequestError): - PythonICATWhereFilter("id", "1, 2, 3, 4, 5", "in") + PythonICATWhereFilter("id", value, operation) def test_invalid_operation(self, icat_query): test_filter = PythonICATWhereFilter("id", 10, "non") From fc1cf194454a4da60652b1f68df278c4624ddc11 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:25:29 +0000 Subject: [PATCH 04/17] feat: implement `inq` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index a20a2e5a..dd42e815 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -88,6 +88,8 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) + elif self.operation == "inq": + self.operation = "in" elif self.operation == "between": where_filter = self.create_condition( self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", From b0134b8882c2c75f666b9c5516c49e1f06c4d72d Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:27:45 +0000 Subject: [PATCH 05/17] validate values supplied with `inq` operator #297 --- datagateway_api/src/common/filters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index 623ec393..716000c9 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,11 +27,11 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation == "in": + if self.operation in ["in", "inq"]: if not isinstance(self.value, list): raise BadRequestError( - "When using the 'in' operation for a WHERE filter, the values must" - " be in a list format e.g. [1, 2, 3]", + f"When using the {self.operation} operation for a WHERE filter, the" + f" values must be in a list format e.g. [1, 2, 3]", ) if self.operation == "between": From 1b142143fb27750e4745d73a5eeb12ce6267849b Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:33:58 +0000 Subject: [PATCH 06/17] test: test `inq` operator #297 --- .../datagateway_api/icat/filters/test_where_filter.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 4cc5ce50..703857aa 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -21,8 +21,14 @@ class TestICATWhereFilter: pytest.param("lte", 5, ["%s <= '5'"], id="less than or equal"), pytest.param("gt", 5, ["%s > '5'"], id="greater than"), pytest.param("gte", 5, ["%s >= '5'"], id="greater than or equal"), - pytest.param("in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list"), - pytest.param("in", [], ["%s in (NULL)"], id="empty list"), + pytest.param( + "in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (in)", + ), + pytest.param("in", [], ["%s in (NULL)"], id="in empty list (in)"), + pytest.param( + "inq", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (inq)", + ), + pytest.param("inq", [], ["%s in (NULL)"], id="in empty list (inq)"), pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), ], ) @@ -38,6 +44,7 @@ def test_valid_operations( "operation, value", [ pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), + pytest.param("inq", "1, 2, 3, 4, 5", id="in a list (inq)"), pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), pytest.param("between", [], id="between - empty list"), pytest.param( From 00dbba525d5cd86cb5577f3b1621a7042cdd2fa0 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:36:00 +0000 Subject: [PATCH 07/17] feat: implement `nin` operator #297 --- .../src/datagateway_api/icat/filters.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index dd42e815..84d9d333 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -90,6 +90,20 @@ def create_filter(self): where_filter = self.create_condition(self.field, "in", self.value) elif self.operation == "inq": self.operation = "in" + where_filter = self.create_filter() + elif self.operation == "nin": + # Convert self.value into a string with brackets equivalent to tuple format. + # Cannot convert straight to tuple as single element tuples contain a + # trailing comma which Python ICAT/JPQL doesn't accept + self.value = str(self.value).replace("[", "(").replace("]", ")") + + # DataGateway Search can send requests with blank lists. Adding NULL to the + # filter prevents the API from returning a 500. An empty list will be + # returned instead, equivalent to the DB backend + if self.value == "()": + self.value = "(NULL)" + + where_filter = self.create_condition(self.field, "not in", self.value) elif self.operation == "between": where_filter = self.create_condition( self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", From 92b5f36389d16f5c79f0199d7dd1408eda688d7a Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:37:02 +0000 Subject: [PATCH 08/17] validate values supplied with `nin` operator #297 --- datagateway_api/src/common/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index 716000c9..a6936d62 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,7 +27,7 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation in ["in", "inq"]: + if self.operation in ["in", "nin", "inq"]: if not isinstance(self.value, list): raise BadRequestError( f"When using the {self.operation} operation for a WHERE filter, the" From 8d572564fbb48714df5276543259aadeb26cb2f4 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:39:05 +0000 Subject: [PATCH 09/17] test: test `nin` operator #297 --- test/datagateway_api/icat/filters/test_where_filter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 703857aa..464c51a2 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -29,6 +29,10 @@ class TestICATWhereFilter: "inq", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (inq)", ), pytest.param("inq", [], ["%s in (NULL)"], id="in empty list (inq)"), + pytest.param( + "nin", [1, 2, 3, 4], ["%s not in (1, 2, 3, 4)"], id="not in a list", + ), + pytest.param("nin", [], ["%s not in (NULL)"], id="not in empty list"), pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), ], ) @@ -45,6 +49,7 @@ def test_valid_operations( [ pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), pytest.param("inq", "1, 2, 3, 4, 5", id="in a list (inq)"), + pytest.param("nin", "1, 2, 3, 4, 5", id="nin"), pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), pytest.param("between", [], id="between - empty list"), pytest.param( From 9094bbb894ead20a53fadfd0e24b264af29548b9 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:40:09 +0000 Subject: [PATCH 10/17] feat: implement `neq` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 84d9d333..ede6f4b9 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -51,6 +51,9 @@ def create_filter(self): where_filter = self.create_condition(self.field, "=", self.value) elif self.operation == "ne": where_filter = self.create_condition(self.field, "!=", self.value) + elif self.operation == "neq": + self.operation = "ne" + where_filter = self.create_filter() elif self.operation == "like": where_filter = self.create_condition(self.field, "like", f"%{self.value}%") elif self.operation == "ilike": From 5817491ea90dc4dca5868b0e35475e534ad95878 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:40:58 +0000 Subject: [PATCH 11/17] test: test `neq` operator #297 --- test/datagateway_api/icat/filters/test_where_filter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 464c51a2..9a200c77 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -10,7 +10,8 @@ class TestICATWhereFilter: "operation, value, expected_condition_value", [ pytest.param("eq", 5, ["%s = '5'"], id="equal"), - pytest.param("ne", 5, ["%s != 5"], id="not equal"), + pytest.param("ne", 5, ["%s != 5"], id="not equal (ne)"), + pytest.param("neq", 5, ["%s != 5"], id="not equal (neq)"), pytest.param("like", 5, ["%s like '%%5%%'"], id="like"), pytest.param("ilike", 5, ["UPPER(%s) like UPPER('%%5%%')"], id="ilike"), pytest.param("nlike", 5, ["%s not like '%%5%%'"], id="not like"), From bf3fe0ef2ac582d55dbd881edf6a81a93625ce91 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:41:51 +0000 Subject: [PATCH 12/17] feat: implement `regexp` operator #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index ede6f4b9..31452a9e 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -111,6 +111,8 @@ def create_filter(self): where_filter = self.create_condition( self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", ) + elif self.operation == "regexp": + where_filter = self.create_condition(self.field, "regexp", self.value) else: raise FilterError(f"Bad operation given to where filter: {self.operation}") From f25333f25791d927ab4601e942fa5e63bac3df6c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Wed, 19 Jan 2022 09:42:36 +0000 Subject: [PATCH 13/17] test: test `regexp` operator #297 --- test/datagateway_api/icat/filters/test_where_filter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 9a200c77..6abaaf00 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -35,6 +35,7 @@ class TestICATWhereFilter: ), pytest.param("nin", [], ["%s not in (NULL)"], id="not in empty list"), pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), + pytest.param("regexp", "^Test", ["%s regexp '^Test'"], id="regexp"), ], ) def test_valid_operations( From bd15023dc7ed56126540eab49fc34b454544b52c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 25 Jan 2022 15:59:41 +0000 Subject: [PATCH 14/17] refactor: remove duplicated code for `between` operator validation #297 --- datagateway_api/src/common/filters.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index a6936d62..bd0e636c 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,20 +27,13 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation in ["in", "nin", "inq"]: + if self.operation in ["in", "nin", "inq", "between"]: if not isinstance(self.value, list): raise BadRequestError( f"When using the {self.operation} operation for a WHERE filter, the" - f" values must be in a list format e.g. [1, 2, 3]", + f" values must be in a list format e.g. [1, 2]", ) - - if self.operation == "between": - if not isinstance(self.value, list): - raise BadRequestError( - "When using the 'between' operation for a WHERE filter, the values" - " must be in a list format e.g. [1, 2]", - ) - if isinstance(self.value, list) and len(self.value) != 2: + if self.operation == "between" and len(self.value) != 2: raise BadRequestError( "When using the 'between' operation for a WHERE filter, the list" "must contain two values e.g. [1, 2]", From 8b344ef9d7bf04cbe55467c2b6eabcff325a8a33 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> Date: Tue, 25 Jan 2022 16:13:15 +0000 Subject: [PATCH 15/17] refactor: merge logic for `ne` and `neq` operators #297 Co-authored-by: Matthew Richards <32678030+MRichards99@users.noreply.github.com> --- datagateway_api/src/datagateway_api/icat/filters.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 31452a9e..2a70a106 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -49,11 +49,8 @@ def create_filter(self): log.info("Creating condition for ICAT where filter") if self.operation == "eq": where_filter = self.create_condition(self.field, "=", self.value) - elif self.operation == "ne": + elif self.operation in ["ne", "neq"]: where_filter = self.create_condition(self.field, "!=", self.value) - elif self.operation == "neq": - self.operation = "ne" - where_filter = self.create_filter() elif self.operation == "like": where_filter = self.create_condition(self.field, "like", f"%{self.value}%") elif self.operation == "ilike": From 3211dfb277e4466895f89af70b770f262b2581c7 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 25 Jan 2022 16:20:10 +0000 Subject: [PATCH 16/17] refactor: merge logic for `in` and `inq` operators #297 --- datagateway_api/src/datagateway_api/icat/filters.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 2a70a106..03d79efd 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -75,7 +75,7 @@ def create_filter(self): where_filter = self.create_condition(self.field, ">", self.value) elif self.operation == "gte": where_filter = self.create_condition(self.field, ">=", self.value) - elif self.operation == "in": + elif self.operation in ["in", "inq"]: # Convert self.value into a string with brackets equivalent to tuple format. # Cannot convert straight to tuple as single element tuples contain a # trailing comma which Python ICAT/JPQL doesn't accept @@ -88,9 +88,6 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) - elif self.operation == "inq": - self.operation = "in" - where_filter = self.create_filter() elif self.operation == "nin": # Convert self.value into a string with brackets equivalent to tuple format. # Cannot convert straight to tuple as single element tuples contain a From 63f4f172e26e6284dce5abaea9e04aac53e3e560 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 31 Jan 2022 17:57:06 +0000 Subject: [PATCH 17/17] 3.4.0 Automatically generated by python-semantic-release --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97cd63f0..8deb20a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ +## v3.4.0 (2022-01-31) +### Feature +* Implement `regexp` operator #297 ([`bf3fe0e`](https://github.com/ral-facilities/datagateway-api/commit/bf3fe0ef2ac582d55dbd881edf6a81a93625ce91)) +* Implement `neq` operator #297 ([`9094bbb`](https://github.com/ral-facilities/datagateway-api/commit/9094bbb894ead20a53fadfd0e24b264af29548b9)) +* Implement `nin` operator #297 ([`00dbba5`](https://github.com/ral-facilities/datagateway-api/commit/00dbba525d5cd86cb5577f3b1621a7042cdd2fa0)) +* Implement `inq` operator #297 ([`fc1cf19`](https://github.com/ral-facilities/datagateway-api/commit/fc1cf194454a4da60652b1f68df278c4624ddc11)) +* Implement `between` operator #297 ([`4736888`](https://github.com/ral-facilities/datagateway-api/commit/4736888bf76cda0dbc00f997443ed565f0f5e760)) + ## v3.3.0 (2022-01-31) ### Feature * Add function to get PaNOSC to ICAT mapping for where filter #260 ([`34b1d81`](https://github.com/ral-facilities/datagateway-api/commit/34b1d819482aa3efdb4f8da321125d3e40d76617)) diff --git a/pyproject.toml b/pyproject.toml index 6fbb6040..a8663bbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datagateway-api" -version = "3.3.0" +version = "3.4.0" description = "ICAT API to interface with the DataGateway" license = "Apache-2.0" readme = "README.md"