Merge branch 'master' into eli-rewrite-geopackage-conversion

CHANGES.rst

@@ -2,8 +2,7 @@ Changelog of threedi-schema
 ===================================================
 
 
-
-0.229.1 (unreleased)
+0.229.2 (unreleased)
 --------------------
 
 - Rename sqlite table "tags" to "tag"
@@ -16,6 +15,13 @@ Changelog of threedi-schema
 - Remove v2 related views from sqlite
 
 
+0.229.1 (2025-01-15)
+--------------------
+
+- Fix setting of geometry columns for revision 223 and 228
+- Fix incorrect creation of geometry for dry weather flow and surface during migration
+
+
 0.229.0 (2025-01-08)
 --------------------
 
@@ -29,6 +35,13 @@ Changelog of threedi-schema
 - Remove v2 related views from sqlite
 
 
+0.228.4 (2025-01-10)
+--------------------
+
+- Fix incorrectly setting of geometry for pipe, weir and orifice in migration
+- Fix issue where invalid geometries broke migration 228 for culverts
+
+
 0.228.3 (2024-12-10)
 --------------------
 

threedi_schema/__init__.py

@@ -2,6 +2,6 @@
 from .domain import constants, custom_types, models  # NOQA
 
 # fmt: off
-__version__ = '0.229.1.dev0'
+__version__ = '0.229.2.dev0'
 
 # fmt: on

threedi_schema/migrations/versions/0223_upgrade_db_inflow.py

@@ -6,6 +6,7 @@
 
 """
 import json
+import uuid
 import warnings
 from pathlib import Path
 from typing import Dict, List, Tuple
@@ -302,12 +303,13 @@ def copy_polygons(src_table: str, tmp_geom: str):
 
 def create_buffer_polygons(src_table: str, tmp_geom: str):
     # create circular polygon of area 1 around the connection node
+    srid = get_global_srid()
     surf_id = f"{src_table.strip('v2_')}_id"
     query = f"""
         WITH connection_data AS (
             SELECT
                 {src_table}_map.{surf_id} AS item_id,
-                ST_Buffer(v2_connection_nodes.the_geom, 1) AS buffer_geom
+                ST_Transform(ST_Buffer(ST_Transform(v2_connection_nodes.the_geom, {srid}), 1), 4326) AS buffer_geom
             FROM
                 v2_connection_nodes
             JOIN
@@ -482,6 +484,29 @@ def populate_dry_weather_flow_distribution():
     op.execute(sa.text(sql_query))
 
 
+def make_geom_col_notnull(table_name):
+    # Make control_measure_map.geom not nullable by creating a new table with
+    # not nullable geometry column, copying the data from control_measure_map
+    # to the new table, dropping the original and renaming the new one to control_measure_map
+    # For some reason, changing this via batch_op.alter_column does not seem to work
+
+    # Retrieve column names and types from table
+    # Note that it is expected that the geometry column is the last column!
+    connection = op.get_bind()
+    columns = connection.execute(sa.text(f"PRAGMA table_info('{table_name}')")).fetchall()
+    col_names = [col[1] for col in columns]
+    col_types = [col[2] for col in columns]
+    cols = (['id INTEGER PRIMARY KEY'] +
+            [f'{cname} {ctype}' for cname, ctype in zip(col_names[:-1], col_types[:-1]) if cname != 'id'] +
+            [f'geom {columns[-1][2]} NOT NULL'])
+    # Create new table, insert data, drop original and rename to table_name
+    temp_name = f'_temp_223_{uuid.uuid4().hex}'
+    op.execute(sa.text(f"CREATE TABLE {temp_name} ({','.join(cols)});"))
+    op.execute(sa.text(f"INSERT INTO {temp_name} ({','.join(col_names)}) SELECT {','.join(col_names)} FROM {table_name}"))
+    drop_geo_table(op, table_name)
+    op.execute(sa.text(f"ALTER TABLE {temp_name} RENAME TO {table_name};"))
+
+
 def fix_geometry_columns():
     GEO_COL_INFO = [
         ('dry_weather_flow', 'geom', 'POLYGON'),
@@ -490,8 +515,7 @@ def fix_geometry_columns():
         ('surface_map', 'geom', 'LINESTRING'),
     ]
     for table, column, geotype in GEO_COL_INFO:
-        with op.batch_alter_table(table) as batch_op:
-            batch_op.alter_column(column_name=column, nullable=False)
+        make_geom_col_notnull(table)
         migration_query = f"SELECT RecoverGeometryColumn('{table}', '{column}', {4326}, '{geotype}', 'XY')"
         op.execute(sa.text(migration_query))
 

threedi_schema/migrations/versions/0228_upgrade_db_1D.py

@@ -100,6 +100,17 @@
 
 RETYPE_COLUMNS = {}
 
+GEOM_TYPES = {'channel': 'LINESTRING',
+              'orifice': 'LINESTRING',
+              'weir': 'LINESTRING',
+              'culvert': 'LINESTRING',
+              'pipe': 'LINESTRING',
+              'pump': 'POINT',
+              'pump_map': 'LINESTRING',
+              'connection_node': 'POINT',
+              'cross_section_location': 'POINT',
+              'windshielding_1d': 'POINT',
+              }
 
 def add_columns_to_tables(table_columns: List[Tuple[str, Column]]):
     # no checks for existence are done, this will fail if any column already exists
@@ -135,14 +146,6 @@ def remove_tables(tables: List[str]):
     for table in tables:
         drop_geo_table(op, table)
 
-
-def get_geom_type(table_name, geo_col_name):
-    connection = op.get_bind()
-    columns = connection.execute(sa.text(f"PRAGMA table_info('{table_name}')")).fetchall()
-    for col in columns:
-        if col[1] == geo_col_name:
-            return col[2]
-
 def modify_table(old_table_name, new_table_name):
     # Create a new table named `new_table_name` by copying the
     # data from `old_table_name`.
@@ -157,7 +160,7 @@ def modify_table(old_table_name, new_table_name):
     col_names = [col[1] for col in columns]
     col_types = [col[2] for col in columns]
     # get type of the geometry column
-    geom_type = get_geom_type(old_table_name, 'the_geom')
+    geom_type = GEOM_TYPES[new_table_name]
     # create list of new columns and types for creating the new table
     # create list of old columns to copy to new table
     skip_cols = ['id', 'the_geom']
@@ -195,7 +198,7 @@ def fix_geometry_columns():
     tables = ['channel', 'connection_node', 'cross_section_location', 'culvert',
               'orifice', 'pipe', 'pump', 'pump_map', 'weir', 'windshielding_1d']
     for table in tables:
-        geom_type = get_geom_type(table, geo_col_name='geom')
+        geom_type = GEOM_TYPES[table]
         op.execute(sa.text(f"SELECT RecoverGeometryColumn('{table}', "
                            f"'geom', {4326}, '{geom_type}', 'XY')"))
         op.execute(sa.text(f"SELECT CreateSpatialIndex('{table}', 'geom')"))
@@ -336,21 +339,46 @@ def migrate_cross_section_definition_to_object(table_name: str):
 
 
 def set_geom_for_object(table_name: str, col_name: str = 'the_geom'):
-    # line from connection_node_start_id to connection_node_end_id
-    # SELECT load_extension('mod_spatialite');
     op.execute(sa.text(f"SELECT AddGeometryColumn('{table_name}', '{col_name}', 4326, 'LINESTRING', 'XY', 0);"))
     q = f"""
         UPDATE
-            {table_name}
-        SET the_geom = (
-            SELECT MakeLine(start_node.the_geom, end_node.the_geom) FROM {table_name} AS object 
-            JOIN v2_connection_nodes AS start_node ON object.connection_node_start_id = start_node.id
-            JOIN v2_connection_nodes AS end_node ON object.connection_node_end_id = end_node.id
-        )         
+            {table_name} AS object
+        SET 
+            {col_name} = (
+                SELECT 
+                    MakeLine(start_node.the_geom, end_node.the_geom) 
+                FROM 
+                    v2_connection_nodes AS start_node,
+                    v2_connection_nodes AS end_node 
+                WHERE 
+                    object.connection_node_start_id = start_node.id
+                    AND 
+                    object.connection_node_end_id = end_node.id
+            )         
     """
     op.execute(sa.text(q))
 
 
+def fix_geom_for_culvert():
+    new_table_name = f'_temp_228_fix_culvert_{uuid.uuid4().hex}'
+    old_table_name = 'v2_culvert'
+    connection = op.get_bind()
+    columns = connection.execute(sa.text(f"PRAGMA table_info('{old_table_name}')")).fetchall()
+    # get all column names and types
+    col_names = [col[1] for col in columns if col[1] not in ['id', 'the_geom']]
+    col_types = [col[2] for col in columns if col[1] in col_names]
+    # Create new table (temp), insert data, drop original and rename temp to table_name
+    new_col_str = ','.join(['id INTEGER PRIMARY KEY NOT NULL'] + [f'{cname} {ctype}' for cname, ctype in
+                                                                  zip(col_names, col_types)])
+    op.execute(sa.text(f"CREATE TABLE {new_table_name} ({new_col_str});"))
+    op.execute(sa.text(f"SELECT AddGeometryColumn('{new_table_name}', 'the_geom', 4326, 'LINESTRING', 'XY', 0);"))
+    # Copy data
+    op.execute(sa.text(f"INSERT INTO {new_table_name} (id, {','.join(col_names)}, the_geom) "
+                       f"SELECT id, {','.join(col_names)}, the_geom FROM {old_table_name}"))
+    op.execute(sa.text("DROP TABLE v2_culvert"))
+    op.execute(sa.text(f"ALTER TABLE {new_table_name} RENAME TO v2_culvert"))
+
+
 def set_geom_for_v2_pumpstation():
     op.execute(sa.text(f"SELECT AddGeometryColumn('v2_pumpstation', 'the_geom', 4326, 'POINT', 'XY', 0);"))
     q = f"""
@@ -529,6 +557,8 @@ def upgrade():
         # Set geometry for tables without one
         if table_name != 'v2_culvert':
             set_geom_for_object(table_name)
+        else:
+            fix_geom_for_culvert()
     set_geom_for_v2_pumpstation()
     for old_table_name, new_table_name in RENAME_TABLES:
         modify_table(old_table_name, new_table_name)