Merge branch 'master' into eli-bump-github-actions

CHANGES.rst

@@ -3,12 +3,29 @@ Changelog of threedi-schema
 
 
 
-0.227.4 (unreleased)
+0.228.2 (unreleased)
 --------------------
 
 - Nothing changed yet.
 
 
+0.228.1 (2024-11-26)
+--------------------
+
+- Add `progress_func` argument to schema.upgrade
+
+
+0.228.0 (2024-11-25)
+--------------------
+
+- Implement changes for schema version 300 concerning 1D
+- Remove v2 prefix from table names v2_channel, v2_windshielding, v2_cross_section_location, v2_pipe, v2_culvert` v2_orifice and v2_weir
+- Move data from v2_cross_section_definition to linked tables (cross_section_location, pipe, culvert, orifice and weir)
+- Move data from v2_manhole to connection_nodes and remove v2_manhole table
+- Rename v2_pumpstation to pump and add table pump_map that maps the end nodes to pumps
+- Remove tables v2_floodfill and v2_cross_section_definition
+
+
 0.227.3 (2024-11-04)
 --------------------
 

pytest.ini

@@ -7,4 +7,5 @@ markers =
 	migration_224: migration to schema 224
 	migration_225: migration to schema 225
 	migration_226: migration to schema 226
+	migration_228: migration to schema 228
 

threedi_schema/__init__.py

@@ -2,6 +2,6 @@
 from .domain import constants, custom_types, models  # NOQA
 
 # fmt: off
-__version__ = '0.227.4.dev0'
+__version__ = '0.228.2.dev0'
 
 # fmt: on

threedi_schema/application/schema.py

@@ -13,11 +13,11 @@
 from sqlalchemy import Column, Integer, MetaData, Table, text
 from sqlalchemy.exc import IntegrityError
 
-from ..domain import constants, models, views
+from ..domain import constants, models
 from ..infrastructure.spatial_index import ensure_spatial_indexes
 from ..infrastructure.spatialite_versions import copy_models, get_spatialite_version
-from ..infrastructure.views import recreate_views
 from .errors import MigrationMissingError, UpgradeFailedError
+from .upgrade_utils import setup_logging
 
 __all__ = ["ModelSchema"]
 
@@ -40,11 +40,12 @@ def get_schema_version():
         return int(env.get_head_revision())
 
 
-def _upgrade_database(db, revision="head", unsafe=True):
+def _upgrade_database(db, revision="head", unsafe=True, progress_func=None):
     """Upgrade ThreediDatabase instance"""
     engine = db.engine
-
     config = get_alembic_config(engine, unsafe=unsafe)
+    if progress_func is not None:
+        setup_logging(db.schema, revision, config, progress_func)
     alembic_command.upgrade(config, revision)
 
 
@@ -87,9 +88,9 @@ def upgrade(
         self,
         revision="head",
         backup=True,
-        set_views=True,
         upgrade_spatialite_version=False,
         convert_to_geopackage=False,
+        progress_func=None,
     ):
         """Upgrade the database to the latest version.
 
@@ -103,15 +104,14 @@ def upgrade(
         If the database is temporary already (or if it is PostGIS), disable
         it.
 
-        Specify 'set_views=True' to also (re)create views after the upgrade.
-        This is not compatible when upgrading to a different version than the
-        latest version.
-
         Specify 'upgrade_spatialite_version=True' to also upgrade the
         spatialite file version after the upgrade.
 
         Specify 'convert_to_geopackage=True' to also convert from spatialite
         to geopackage file version after the upgrade.
+
+        Specify a 'progress_func' to handle progress updates. `progress_func` should
+        expect a single argument representing the fraction of progress
         """
         try:
             rev_nr = get_schema_version() if revision == "head" else int(revision)
@@ -124,33 +124,26 @@ def upgrade(
                 f"Cannot convert to geopackage for {revision=} because geopackage support is "
                 "enabled from revision 300",
             )
-        if upgrade_spatialite_version and not set_views:
-            set_views = True
-            warnings.warn(
-                "Setting set_views to True because the spatialite version cannot be upgraded without setting the views",
-                UserWarning,
-            )
         v = self.get_version()
         if v is not None and v < constants.LATEST_SOUTH_MIGRATION_ID:
             raise MigrationMissingError(
                 f"This tool cannot update versions below "
                 f"{constants.LATEST_SOUTH_MIGRATION_ID}. Please consult the "
                 f"3Di documentation on how to update legacy databases."
             )
-        if set_views and revision not in ("head", get_schema_version()):
-            raise ValueError(f"Cannot set views when upgrading to version '{revision}'")
         if backup:
             with self.db.file_transaction() as work_db:
-                _upgrade_database(work_db, revision=revision, unsafe=True)
+                _upgrade_database(
+                    work_db, revision=revision, unsafe=True, progress_func=progress_func
+                )
         else:
-            _upgrade_database(self.db, revision=revision, unsafe=False)
+            _upgrade_database(
+                self.db, revision=revision, unsafe=False, progress_func=progress_func
+            )
         if upgrade_spatialite_version:
             self.upgrade_spatialite_version()
         elif convert_to_geopackage:
             self.convert_to_geopackage()
-            set_views = True
-        if set_views:
-            self.set_views()
 
     def validate_schema(self):
         """Very basic validation of 3Di schema.
@@ -178,20 +171,6 @@ def validate_schema(self):
             )
         return True
 
-    def set_views(self):
-        """(Re)create views in the spatialite according to the latest definitions."""
-        version = self.get_version()
-        schema_version = get_schema_version()
-        if version != schema_version:
-            raise MigrationMissingError(
-                f"Setting views requires schema version "
-                f"{schema_version}. Current version: {version}."
-            )
-
-        _, file_version = get_spatialite_version(self.db)
-
-        recreate_views(self.db, file_version, views.ALL_VIEWS, views.VIEWS_TO_DELETE)
-
     def set_spatial_indexes(self):
         """(Re)create spatial indexes in the spatialite according to the latest definitions."""
         version = self.get_version()

threedi_schema/application/upgrade_utils.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Callable, TYPE_CHECKING
+
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+
+if TYPE_CHECKING:
+    from .schema import ModelSchema
+else:
+    ModelSchema = None
+
+
+class ProgressHandler(logging.Handler):
+    def __init__(self, progress_func, total_steps):
+        super().__init__()
+        self.progress_func = progress_func
+        self.total_steps = total_steps
+        self.current_step = 0
+
+    def emit(self, record):
+        msg = record.getMessage()
+        if msg.startswith("Running upgrade"):
+            self.progress_func(100 * self.current_step / self.total_steps)
+            self.current_step += 1
+
+
+def get_upgrade_steps_count(
+    config: Config, current_revision: int, target_revision: str = "head"
+) -> int:
+    """
+    Count number of upgrade steps for a schematisation upgrade.
+
+    Args:
+        config: Config parameter containing the configuration information
+        current_revision: current revision as integer
+        target_revision: target revision as zero-padded 4 digit string or "head"
+    """
+    if target_revision != "head":
+        try:
+            int(target_revision)
+        except TypeError:
+            # this should lead to issues in the upgrade pipeline, lets not take over that error handling here
+            return 0
+    # walk_revisions also includes the revision from current_revision to previous
+    # reduce the number of steps with 1
+    offset = -1
+    # The first defined revision is 200; revision numbers < 200 will cause walk_revisions to fail
+    if current_revision < 200:
+        current_revision = 200
+        # set offset to 0 because previous to current is not included in walk_revisions
+        offset = 0
+    if target_revision != "head" and int(target_revision) < current_revision:
+        # assume that this will be correctly handled by alembic
+        return 0
+    current_revision_str = f"{current_revision:04d}"
+    script = ScriptDirectory.from_config(config)
+    # Determine upgrade steps
+    revisions = script.walk_revisions(current_revision_str, target_revision)
+    return len(list(revisions)) + offset
+
+
+def setup_logging(
+    schema: ModelSchema,
+    target_revision: str,
+    config: Config,
+    progress_func: Callable[[float], None],
+):
+    """
+    Set up logging for schematisation upgrade
+
+    Args:
+        schema: ModelSchema object representing the current schema of the application
+        target_revision: A str specifying the target revision for migration
+        config: Config object containing configuration settings
+        progress_func: A Callable with a single argument of type float, used to track progress during migration
+    """
+    n_steps = get_upgrade_steps_count(config, schema.get_version(), target_revision)
+    logger = logging.getLogger("alembic.runtime.migration")
+    logger.setLevel(logging.INFO)
+    handler = ProgressHandler(progress_func, total_steps=n_steps)
+    logger.addHandler(handler)

threedi_schema/domain/constants.py

@@ -62,12 +62,19 @@ class CalculationTypeCulvert(Enum):
     DOUBLE_CONNECTED = 105
 
 
+# TODO: rename enum (?)
 class CalculationTypeNode(Enum):
     EMBEDDED = 0
     ISOLATED = 1
     CONNECTED = 2
 
 
+class AmbiguousClosedError(Exception):
+    def __init__(self, shape):
+        self.shape = shape
+        super().__init__(f"Closed state is ambiguous for shape: {self.shape}")
+
+
 class CrossSectionShape(Enum):
     CLOSED_RECTANGLE = 0
     RECTANGLE = 1
@@ -78,6 +85,22 @@ class CrossSectionShape(Enum):
     TABULATED_YZ = 7
     INVERTED_EGG = 8
 
+    @property
+    def is_tabulated(self):
+        return self in {
+            CrossSectionShape.TABULATED_RECTANGLE,
+            CrossSectionShape.TABULATED_TRAPEZIUM,
+            CrossSectionShape.TABULATED_YZ,
+        }
+
+    @property
+    def is_closed(self):
+        if self.is_tabulated:
+            raise AmbiguousClosedError(self)
+        if self == CrossSectionShape.RECTANGLE:
+            return False
+        return True
+
 
 class FrictionType(Enum):
     CHEZY = 1
@@ -159,17 +182,6 @@ class InfiltrationSurfaceOption(Enum):
     WET_SURFACE = 2
 
 
-class ZoomCategories(Enum):
-    # Visibility in live-site: 0 is lowest for smallest level (i.e. ditch)
-    # and 5 for highest (rivers).
-    LOWEST_VISIBILITY = 0
-    LOW_VISIBILITY = 1
-    MEDIUM_LOW_VISIBILITY = 2
-    MEDIUM_VISIBILITY = 3
-    HIGH_VISIBILITY = 4
-    HIGHEST_VISIBILITY = 5
-
-
 class InflowType(Enum):
     NO_INFLOW = 0
     IMPERVIOUS_SURFACE = 1
@@ -205,12 +217,21 @@ class ControlType(Enum):
 
 
 class StructureControlTypes(Enum):
-    pumpstation = "v2_pumpstation"
-    pipe = "v2_pipe"
-    orifice = "v2_orifice"
-    culvert = "v2_culvert"
-    weir = "v2_weir"
-    channel = "v2_channel"
+    pumpstation = "pump"
+    pipe = "pipe"
+    orifice = "orifice"
+    culvert = "culvert"
+    weir = "weir"
+    channel = "channel"
+
+    def get_legacy_value(self) -> str:
+        """
+        Get value of structure control as used in schema 2.x
+        """
+        if self == StructureControlTypes.pumpstation:
+            return "v2_pumpstation"
+        else:
+            return f"v2_{self.value}"
 
 
 class ControlTableActionTypes(Enum):
@@ -240,3 +261,8 @@ class AdvectionTypes1D(Enum):
     MOMENTUM_CONSERVATIVE = 1
     ENERGY_CONSERVATIVE = 2
     COMBINED_MOMENTUM_AND_ENERGY_CONSERVATIVE = 3
+
+
+class NodeOpenWaterDetection(Enum):
+    HAS_CHANNEL = 0
+    HAS_STORAGE = 1