Merge pull request #304 from DHI-GRAS/migration

Add bare-bones migration script

terracotta/__init__.py

@@ -13,7 +13,6 @@
         "`python setup.py develop` in the Terracotta package folder."
     ) from None
 
-
 # initialize settings, define settings API
 from typing import Mapping, Any, Set
 from terracotta.config import parse_config, TerracottaSettings

terracotta/drivers/relational_meta_store.py

@@ -258,7 +258,7 @@ def _initialize_database(
                 "key_name", self.SQLA_STRING(self.SQL_KEY_SIZE), primary_key=True
             ),
             sqla.Column("description", self.SQLA_STRING(8000)),
-            sqla.Column("index", sqla.types.Integer, unique=True),
+            sqla.Column("idx", sqla.types.Integer, unique=True),
         )
         _ = sqla.Table(
             "datasets",
@@ -290,9 +290,7 @@ def _initialize_database(
             conn.execute(
                 key_names_table.insert(),
                 [
-                    dict(
-                        key_name=key, description=key_descriptions.get(key, ""), index=i
-                    )
+                    dict(key_name=key, description=key_descriptions.get(key, ""), idx=i)
                     for i, key in enumerate(keys)
                 ],
             )
@@ -307,7 +305,7 @@ def get_keys(self) -> OrderedDict:
             result = conn.execute(
                 sqla.select(
                     keys_table.c["key_name"], keys_table.c["description"]
-                ).order_by(keys_table.c["index"])
+                ).order_by(keys_table.c["idx"])
             )
         return OrderedDict((row.key_name, row.description) for row in result.all())
 

terracotta/migrations/__init__.py

@@ -0,0 +1,22 @@
+"""migrations/__init__.py
+
+Define available migrations.
+"""
+
+import os
+import glob
+import importlib
+
+
+MIGRATIONS = {}
+
+glob_pattern = os.path.join(os.path.dirname(__file__), "v*_*.py")
+
+for modpath in glob.glob(glob_pattern):
+    modname = os.path.basename(modpath)[: -len(".py")]
+    mod = importlib.import_module(f"{__name__}.{modname}")
+    assert all(
+        hasattr(mod, attr) for attr in ("up_version", "down_version", "upgrade_sql")
+    )
+    assert mod.down_version not in MIGRATIONS
+    MIGRATIONS[mod.down_version] = mod

terracotta/migrations/v0_8.py

@@ -0,0 +1,8 @@
+up_version = (0, 8)
+down_version = (0, 7)
+
+upgrade_sql = [
+    "CREATE TABLE key_names (key_name TEXT PRIMARY KEY, description TEXT, idx INTEGER UNIQUE)",
+    "INSERT INTO key_names (key_name, description, idx) SELECT key, description, row_number() over (order by (select NULL)) FROM keys",
+    "UPDATE terracotta SET version='0.8.0'",
+]

terracotta/scripts/cli.py

@@ -85,6 +85,9 @@ def entrypoint() -> None:
 
 cli.add_command(serve)
 
+from terracotta.scripts.migrate import migrate
+
+cli.add_command(migrate)
 
 if __name__ == "__main__":
     entrypoint()

terracotta/scripts/migrate.py

@@ -0,0 +1,91 @@
+"""scripts/migrate.py
+
+Migrate databases between Terracotta versions.
+"""
+
+from typing import Tuple
+
+import click
+import sqlalchemy as sqla
+
+from terracotta import get_driver, __version__
+from terracotta.migrations import MIGRATIONS
+from terracotta.drivers.relational_meta_store import RelationalMetaStore
+
+
+def parse_version(verstr: str) -> Tuple[int, ...]:
+    """Convert 'v<major>.<minor>.<patch>' to (major, minor, patch)"""
+    components = verstr.split(".")
+    components[0] = components[0].lstrip("v")
+    return tuple(int(c) for c in components[:3])
+
+
+def join_version(vertuple: Tuple[int, ...]) -> str:
+    return "v" + ".".join(map(str, vertuple))
+
+
+@click.argument("DATABASE", required=True)
+@click.option("--from", "from_version", required=False, default=None)
+@click.option("--to", "to_version", required=False, default=__version__)
+@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation.")
+@click.command("migrate")
+def migrate(database: str, to_version: str, from_version: str, yes: bool) -> None:
+    from_version_tuple, to_version_tuple, tc_version_tuple = (
+        parse_version(v)[:2] if v is not None else None
+        for v in (from_version, to_version, __version__)
+    )
+
+    driver = get_driver(database)
+    meta_store = driver.meta_store
+    assert isinstance(meta_store, RelationalMetaStore)
+
+    if to_version_tuple > tc_version_tuple:
+        raise ValueError(
+            f"Unknown target version {join_version(to_version_tuple)} (this is {join_version(tc_version_tuple)}). Try upgrading terracotta."
+        )
+
+    if from_version_tuple is None:
+        try:  # type: ignore
+            with meta_store.connect(verify=False):
+                from_version_tuple = parse_version(driver.db_version)[:2]
+        except Exception as e:
+            raise RuntimeError("Cannot determine database version.") from e
+
+    if from_version_tuple == to_version_tuple:
+        click.echo("Already at target version, nothing to do.")
+        return
+
+    migration_chain = []
+    current_version = from_version_tuple
+
+    while current_version != to_version_tuple:
+        if current_version not in MIGRATIONS:
+            raise RuntimeError("Unexpected error")
+
+        migration = MIGRATIONS[current_version]
+        migration_chain.append(migration)
+        current_version = migration.up_version
+
+    click.echo("Upgrade path found\n")
+
+    for migration in migration_chain:
+        click.echo(
+            f"{join_version(migration.down_version)} -> {join_version(migration.up_version)}"
+        )
+
+        for cmd in migration.upgrade_sql:
+            click.echo(f"    {cmd}")
+
+        click.echo("")
+
+    click.echo(
+        f"This will upgrade the database from {join_version(from_version_tuple)} -> {join_version(to_version_tuple)} and execute the above SQL commands."
+    )
+
+    if not yes:
+        click.confirm("Continue?", abort=True)
+
+    with meta_store.connect(verify=False) as conn:
+        for migration in migration_chain:
+            for cmd in migration.upgrade_sql:
+                conn.execute(sqla.text(cmd))

terracotta/scripts/serve.py

@@ -3,7 +3,7 @@
 Use Flask development server to serve up raster files or database locally.
 """
 
-from typing import Optional, Any, Tuple, Sequence
+from typing import Optional, Any, Tuple, Sequence, cast
 import os
 import tempfile
 import logging
@@ -119,13 +119,20 @@ def push_to_last(seq: Sequence[Any], index: int) -> Tuple[Any, ...]:
 
         database = dbfile.name
 
+    database = cast(str, database)
+
     update_settings(
         DRIVER_PATH=database,
         DRIVER_PROVIDER=database_provider,
         DEBUG=debug,
         FLASK_PROFILE=profile,
     )
 
+    # ensure database can be connected to
+    driver = get_driver(database, provider=database_provider)
+    with driver.connect():
+        pass
+
     # find suitable port
     port_range = [port] if port is not None else range(5000, 5100)
     port = find_open_port(port_range)

tests/conftest.py

@@ -359,6 +359,19 @@ def testdb(raster_file, tmpdir_factory):
     return dbpath
 
 
+@pytest.fixture(scope="session")
+def v07_db(tmpdir_factory):
+    """A read-only, pre-populated test database"""
+    import shutil
+
+    dbpath = tmpdir_factory.mktemp("db").join("db-outdated.sqlite")
+    shutil.copyfile(
+        os.path.join(os.path.dirname(__file__), "data", "db-v075.sqlite"), dbpath
+    )
+
+    return dbpath
+
+
 @pytest.fixture()
 def use_testdb(testdb, monkeypatch):
     import terracotta
@@ -487,3 +500,22 @@ def random_string(length):
 
     else:
         return NotImplementedError(f"unknown provider {provider}")
+
+
+@pytest.fixture()
+def force_reload():
+    """Force a reload of the Terracotta module"""
+    import sys
+
+    def purge_module(modname):
+        for mod in list(sys.modules.values()):
+            if mod is None:
+                continue
+            if mod.__name__ == modname or mod.__name__.startswith(f"{modname}."):
+                del sys.modules[mod.__name__]
+
+    purge_module("terracotta")
+    try:
+        yield
+    finally:
+        purge_module("terracotta")

tests/scripts/test_migrate.py

@@ -0,0 +1,39 @@
+from click.testing import CliRunner
+
+
+def parse_version(verstr):
+    """Convert 'v<major>.<minor>.<patch>' to (major, minor, patch)"""
+    components = verstr.split(".")
+    components[0] = components[0].lstrip("v")
+    return tuple(int(c) for c in components[:3])
+
+
+def test_migrate(v07_db, monkeypatch, force_reload):
+    """Test database migration to next major version if one is available."""
+    with monkeypatch.context() as m:
+        # pretend we are at the next major version
+        import terracotta
+
+        current_version = parse_version(terracotta.__version__)
+        next_major_version = (current_version[0], current_version[1] + 1, 0)
+        m.setattr(terracotta, "__version__", ".".join(map(str, next_major_version)))
+
+        # run migration
+        from terracotta import get_driver
+        from terracotta.scripts import cli
+        from terracotta.migrations import MIGRATIONS
+
+        runner = CliRunner()
+        result = runner.invoke(
+            cli.cli, ["migrate", str(v07_db), "--from", "v0.7", "--yes"]
+        )
+        assert result.exit_code == 0
+
+        if next_major_version[:2] not in [m.up_version for m in MIGRATIONS.values()]:
+            assert "Unknown target version" in result.output
+            return
+
+        assert "Upgrade path found" in result.output
+
+        driver_updated = get_driver(str(v07_db), provider="sqlite")
+        assert driver_updated.key_names == ("key1", "akey", "key2")