From af4f30e6635b1a140b0327e0dbc19fa6778d2838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 27 Feb 2024 17:35:20 +0100 Subject: [PATCH 01/10] fix application of unique index on timed_beliefs, also add index on the same fields so searching is sped up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 69484cc9..93595efd 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -30,9 +30,9 @@ ) from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property -from sqlalchemy.orm import Session, backref, has_inherited_table, relationship +from sqlalchemy.orm import Session, backref, relationship, declarative_mixin from sqlalchemy.orm.util import AliasedClass -from sqlalchemy.schema import UniqueConstraint +from sqlalchemy.schema import Index from sqlalchemy.sql.elements import BinaryExpression from sqlalchemy.sql.expression import Selectable @@ -174,6 +174,7 @@ def source_id(self): return None +@declarative_mixin class TimedBeliefDBMixin(TimedBelief): """ Mixin class for a table with beliefs. @@ -182,17 +183,16 @@ class TimedBeliefDBMixin(TimedBelief): @declared_attr def __table_args__(cls): - if has_inherited_table(cls): - return ( - UniqueConstraint( - "event_start", - "belief_horizon", - "sensor_id", - "source_id", - name="_one_belief_by_one_source_uc", - ), - ) - return None + return ( + Index( + f"{cls.__tablename__}_quad_unique_and_search_idx", + "event_start", + "belief_horizon", + "sensor_id", + "source_id", + unique=True + ), + ) event_start = Column(DateTime(timezone=True), primary_key=True, index=True) belief_horizon = Column(Interval(), nullable=False, primary_key=True) From fee7f4f11c2055222aaeb51ecb4dc8879c3e35d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 27 Feb 2024 17:52:11 +0100 Subject: [PATCH 02/10] flake8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 93595efd..d6811503 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -30,7 +30,7 @@ ) from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property -from sqlalchemy.orm import Session, backref, relationship, declarative_mixin +from sqlalchemy.orm import Session, backref, declarative_mixin, relationship from sqlalchemy.orm.util import AliasedClass from sqlalchemy.schema import Index from sqlalchemy.sql.elements import BinaryExpression @@ -190,7 +190,7 @@ def __table_args__(cls): "belief_horizon", "sensor_id", "source_id", - unique=True + unique=True, ), ) From fc1587c925afb501c6a6f37b4c6dc894491d5242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 28 Feb 2024 13:18:06 +0100 Subject: [PATCH 03/10] the order of the fields matters, has to match the query's order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index d6811503..94550365 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -187,9 +187,9 @@ def __table_args__(cls): Index( f"{cls.__tablename__}_quad_unique_and_search_idx", "event_start", - "belief_horizon", - "sensor_id", "source_id", + "sensor_id", + "belief_horizon", unique=True, ), ) From a90c932b40c6c880c71c192a7803bfe400378b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 28 Feb 2024 14:04:25 +0100 Subject: [PATCH 04/10] add unique constraints separately, as it needs the probability fields as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 94550365..5b080509 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -32,7 +32,7 @@ from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property from sqlalchemy.orm import Session, backref, declarative_mixin, relationship from sqlalchemy.orm.util import AliasedClass -from sqlalchemy.schema import Index +from sqlalchemy.schema import Index, UniqueConstraint from sqlalchemy.sql.elements import BinaryExpression from sqlalchemy.sql.expression import Selectable @@ -184,13 +184,20 @@ class TimedBeliefDBMixin(TimedBelief): @declared_attr def __table_args__(cls): return ( + UniqueConstraint( + "event_start", + "belief_horizon", + "sensor_id", + "source_id", + "cumulative_probability", + name=f"{cls.__tablename__}_one_belief_by_one_source_uc", + ), Index( - f"{cls.__tablename__}_quad_unique_and_search_idx", + f"{cls.__tablename__}_quad_search_idx", "event_start", "source_id", "sensor_id", "belief_horizon", - unique=True, ), ) From 0f0be0e502543fac6206a84ccb21af00e9c722bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 28 Feb 2024 14:06:58 +0100 Subject: [PATCH 05/10] flake8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 5b080509..a14c1dea 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -185,13 +185,13 @@ class TimedBeliefDBMixin(TimedBelief): def __table_args__(cls): return ( UniqueConstraint( - "event_start", - "belief_horizon", - "sensor_id", - "source_id", - "cumulative_probability", - name=f"{cls.__tablename__}_one_belief_by_one_source_uc", - ), + "event_start", + "belief_horizon", + "sensor_id", + "source_id", + "cumulative_probability", + name=f"{cls.__tablename__}_one_belief_by_one_source_uc", + ), Index( f"{cls.__tablename__}_quad_search_idx", "event_start", From d0c2c14cc71b7cf84beaefe66b0dd51fce19c663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 28 Feb 2024 14:09:42 +0100 Subject: [PATCH 06/10] no UNIQUE index - without probability it makes no sense, with is covered by primary key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index a14c1dea..00217800 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -184,14 +184,6 @@ class TimedBeliefDBMixin(TimedBelief): @declared_attr def __table_args__(cls): return ( - UniqueConstraint( - "event_start", - "belief_horizon", - "sensor_id", - "source_id", - "cumulative_probability", - name=f"{cls.__tablename__}_one_belief_by_one_source_uc", - ), Index( f"{cls.__tablename__}_quad_search_idx", "event_start", From 93fd14320d93ce3067819b50138336eba3ab8655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 28 Feb 2024 14:22:16 +0100 Subject: [PATCH 07/10] remove unused import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 00217800..9a05238b 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -32,7 +32,7 @@ from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property from sqlalchemy.orm import Session, backref, declarative_mixin, relationship from sqlalchemy.orm.util import AliasedClass -from sqlalchemy.schema import Index, UniqueConstraint +from sqlalchemy.schema import Index from sqlalchemy.sql.elements import BinaryExpression from sqlalchemy.sql.expression import Selectable From b89a696f8dc67b5dc274118aa01767d171832c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 1 Mar 2024 10:00:50 +0100 Subject: [PATCH 08/10] do not include belief_horizon in index, just add as column (we are using min() on it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 9a05238b..725c5361 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -185,11 +185,13 @@ class TimedBeliefDBMixin(TimedBelief): def __table_args__(cls): return ( Index( - f"{cls.__tablename__}_quad_search_idx", + f"{cls.__tablename__}_search_session_idx", "event_start", - "source_id", "sensor_id", - "belief_horizon", + "source_id", + postgresql_include=[ + "belief_horizon", # we use min() on this + ] ), ) From f0a2b2ae0e1b027df2fa9f854fa10355c1d2e1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 1 Mar 2024 10:03:37 +0100 Subject: [PATCH 09/10] black MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 725c5361..2027104d 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -191,7 +191,7 @@ def __table_args__(cls): "source_id", postgresql_include=[ "belief_horizon", # we use min() on this - ] + ], ), ) @@ -1162,16 +1162,12 @@ def _for_each_belief( index_names.extend( ["event_start"] if "event_start" in df.index.names - else ["event_end"] - if "event_end" in df.index.names - else [] + else ["event_end"] if "event_end" in df.index.names else [] ) index_names.extend( ["belief_time"] if "belief_time" in df.index.names - else ["belief_horizon"] - if "belief_horizon" in df.index.names - else [] + else ["belief_horizon"] if "belief_horizon" in df.index.names else [] ) if collective_beliefs is False: index_names.append("source") @@ -1490,9 +1486,9 @@ def resample_events( column_functions = { "event_value": "mean", "source": "first", # keep the only source - belief_timing_col: "max" - if belief_timing_col == "belief_time" - else "min", # keep only most recent belief + belief_timing_col: ( + "max" if belief_timing_col == "belief_time" else "min" + ), # keep only most recent belief "cumulative_probability": "mean", # we just have one point on each CDF } df = downsample_beliefs_data_frame( @@ -1549,8 +1545,7 @@ def form_beliefs( belief_time: datetime, source: BeliefSource, event_start: datetime = None, - event_time_window: tuple[datetime, datetime] - | None = ( + event_time_window: tuple[datetime, datetime] | None = ( None, None, ), @@ -2111,9 +2106,7 @@ def assign_sensor_and_event_resolution(df, sensor, event_resolution): df.event_resolution = ( event_resolution if event_resolution - else sensor.event_resolution - if sensor - else None + else sensor.event_resolution if sensor else None ) From e104c010aec5aab37022d8a7ab8269841110d997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 1 Mar 2024 10:41:09 +0100 Subject: [PATCH 10/10] black with version we use in pre-commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- timely_beliefs/beliefs/classes.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index fc7b1462..ee2bd864 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -1173,12 +1173,16 @@ def _for_each_belief( index_names.extend( ["event_start"] if "event_start" in df.index.names - else ["event_end"] if "event_end" in df.index.names else [] + else ["event_end"] + if "event_end" in df.index.names + else [] ) index_names.extend( ["belief_time"] if "belief_time" in df.index.names - else ["belief_horizon"] if "belief_horizon" in df.index.names else [] + else ["belief_horizon"] + if "belief_horizon" in df.index.names + else [] ) if collective_beliefs is False: index_names.append("source") @@ -1556,7 +1560,8 @@ def form_beliefs( belief_time: datetime, source: BeliefSource, event_start: datetime = None, - event_time_window: tuple[datetime, datetime] | None = ( + event_time_window: tuple[datetime, datetime] + | None = ( None, None, ), @@ -2117,7 +2122,9 @@ def assign_sensor_and_event_resolution(df, sensor, event_resolution): df.event_resolution = ( event_resolution if event_resolution - else sensor.event_resolution if sensor else None + else sensor.event_resolution + if sensor + else None )