Start with (file) config system for openeo_driver

related to #168, Open-EO/openeo-geopyspark-integrationtests#6

openeo_driver/_version.py

@@ -1 +1 @@
-__version__ = "0.41.1a1"
+__version__ = "0.42.0a1"

openeo_driver/backend.py

@@ -25,6 +25,7 @@
 from openeo.capabilities import ComparableVersion
 from openeo.internal.process_graph_visitor import ProcessGraphVisitor
 from openeo.util import rfc3339, dict_no_none
+from openeo_driver.config import OpenEoBackendConfig, get_backend_config
 from openeo_driver.datacube import DriverDataCube, DriverMlModel, DriverVectorCube
 from openeo_driver.datastructs import SarBackscatterArgs
 from openeo_driver.dry_run import SourceConstraint
@@ -634,13 +635,16 @@ class OpenEoBackendImplementation:
     vector_cube_cls = DriverVectorCube
 
     def __init__(
-            self,
-            secondary_services: Optional[SecondaryServices] = None,
-            catalog: Optional[AbstractCollectionCatalog] = None,
-            batch_jobs: Optional[BatchJobs] = None,
-            user_defined_processes: Optional[UserDefinedProcesses] = None,
-            processing: Optional[Processing] = None,
+        self,
+        *,
+        secondary_services: Optional[SecondaryServices] = None,
+        catalog: Optional[AbstractCollectionCatalog] = None,
+        batch_jobs: Optional[BatchJobs] = None,
+        user_defined_processes: Optional[UserDefinedProcesses] = None,
+        processing: Optional[Processing] = None,
+        config: Optional[OpenEoBackendConfig] = None,
     ):
+        self.config: OpenEoBackendConfig = config or get_backend_config()
         self.secondary_services = secondary_services
         self.catalog = catalog
         self.batch_jobs = batch_jobs
@@ -658,7 +662,7 @@ def health_check(self, options: Optional[dict] = None) -> Union[str, dict, flask
         return "OK"
 
     def oidc_providers(self) -> List[OidcProvider]:
-        return []
+        return self.config.oidc_providers
 
     def file_formats(self) -> dict:
         """

openeo_driver/config/__init__.py

@@ -0,0 +1,2 @@
+from openeo_driver.config.config import OpenEoBackendConfig, ConfigException
+from openeo_driver.config.load import get_backend_config

openeo_driver/config/config.py

@@ -0,0 +1,21 @@
+from typing import List, Optional
+
+import attrs
+
+from openeo_driver.users.oidc import OidcProvider
+
+
+class ConfigException(ValueError):
+    pass
+
+
+@attrs.frozen
+class OpenEoBackendConfig:
+    """
+    Configuration for openEO backend.
+    """
+
+    # identifier for this config
+    id: Optional[str] = None
+
+    oidc_providers: List[OidcProvider] = attrs.Factory(list)

openeo_driver/config/default.py

@@ -0,0 +1,8 @@
+"""
+Default OpenEoBackendConfig
+"""
+from openeo_driver.config import OpenEoBackendConfig
+
+config = OpenEoBackendConfig(
+    id="default",
+)

openeo_driver/config/load.py

@@ -0,0 +1,77 @@
+"""
+Utilities for (lazy) loading of config
+"""
+import importlib.resources
+import logging
+import os
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from openeo_driver.config.config import ConfigException
+
+_log = logging.getLogger(__name__)
+
+from openeo_driver.config import OpenEoBackendConfig
+
+
+def load_from_py_file(
+    path: Union[str, Path],
+    variable: str = "config",
+    expected_class: Optional[type] = OpenEoBackendConfig,
+) -> Any:
+    """Load a config value from a Python file."""
+    path = Path(path)
+    _log.info(
+        f"Loading configuration from Python file {path!r} (variable {variable!r})"
+    )
+
+    # Based on flask's Config.from_pyfile
+    with path.open(mode="rb") as f:
+        code = compile(f.read(), path, "exec")
+    globals = {"__file__": str(path)}
+    exec(code, globals)
+
+    try:
+        config = globals[variable]
+    except KeyError:
+        raise ConfigException(
+            f"No variable {variable!r} found in config file {path!r}"
+        ) from None
+
+    if expected_class:
+        if not isinstance(config, expected_class):
+            raise ConfigException(
+                f"Expected {expected_class.__name__} but got {type(config).__name__}"
+            )
+    return config
+
+
+class ConfigGetter:
+    """Config loader, with lazy-loading and flushing."""
+
+    def __init__(self):
+        self._config: Optional[OpenEoBackendConfig] = None
+
+    def __call__(self, force_reload: bool = False) -> OpenEoBackendConfig:
+        if self._config is None or force_reload:
+            self._config = self._load()
+        return self._config
+
+    def _load(self) -> OpenEoBackendConfig:
+        with importlib.resources.path(
+            "openeo_driver.config", "default.py"
+        ) as default_config:
+            config_path = os.environ.get(
+                "OPENEO_BACKEND_CONFIG",
+                default_config,
+            )
+            config = load_from_py_file(
+                path=config_path, variable="config", expected_class=OpenEoBackendConfig
+            )
+            return config
+
+    def flush(self):
+        self._config = None
+
+
+get_backend_config = ConfigGetter()

openeo_driver/dummy/dummy_backend.py

@@ -31,6 +31,7 @@
     Processing,
     BatchJobResultMetadata,
 )
+from openeo_driver.config import OpenEoBackendConfig
 from openeo_driver.datacube import DriverDataCube, DriverMlModel, DriverVectorCube
 from openeo_driver.datastructs import StacAsset
 from openeo_driver.delayed_vector import DelayedVector
@@ -788,53 +789,20 @@ class DummyBackendImplementation(OpenEoBackendImplementation):
 
     vector_cube_cls = DummyVectorCube
 
-    def __init__(self, processing: Optional[Processing] = None):
+    def __init__(
+        self,
+        processing: Optional[Processing] = None,
+        config: Optional[OpenEoBackendConfig] = None,
+    ):
         super(DummyBackendImplementation, self).__init__(
             secondary_services=DummySecondaryServices(),
             catalog=DummyCatalog(),
             batch_jobs=DummyBatchJobs(),
             user_defined_processes=DummyUserDefinedProcesses(),
             processing=processing or DummyProcessing(),
+            config=config,
         )
 
-    def oidc_providers(self) -> List[OidcProvider]:
-        return [
-            OidcProvider(id="testprovider", issuer="https://oidc.test", scopes=["openid"], title="Test"),
-            OidcProvider(
-                id="eoidc", issuer="https://eoidc.test", scopes=["openid"], title="e-OIDC",
-                default_clients=[{
-                    "id": "badcafef00d",
-                    "grant_types": ["urn:ietf:params:oauth:grant-type:device_code+pkce", "refresh_token"]
-                }],
-            ),
-            # Allow testing with Keycloak setup running in docker on localhost.
-            OidcProvider(
-                id="local", title="Local Keycloak",
-                issuer="http://localhost:9090/auth/realms/master", scopes=["openid"],
-            ),
-            # Allow testing the dummy backend with EGI
-            OidcProvider(
-                id="egi",
-                issuer="https://aai.egi.eu/auth/realms/egi/",
-                scopes=[
-                    "openid", "email",
-                    "eduperson_entitlement",
-                    "eduperson_scoped_affiliation",
-                ],
-                title="EGI Check-in",
-            ),
-            OidcProvider(
-                id="egi-dev",
-                issuer="https://aai-dev.egi.eu/auth/realms/egi",
-                scopes=[
-                    "openid", "email",
-                    "eduperson_entitlement",
-                    "eduperson_scoped_affiliation",
-                ],
-                title="EGI Check-in (dev)",
-            ),
-        ]
-
     def file_formats(self) -> dict:
         return {
             "input": {

openeo_driver/dummy/dummy_config.py

@@ -0,0 +1,62 @@
+from openeo_driver.config import OpenEoBackendConfig
+from openeo_driver.users.oidc import OidcProvider
+
+oidc_providers = [
+    OidcProvider(
+        id="testprovider",
+        issuer="https://oidc.test",
+        scopes=["openid"],
+        title="Test",
+    ),
+    OidcProvider(
+        id="eoidc",
+        issuer="https://eoidc.test",
+        scopes=["openid"],
+        title="e-OIDC",
+        default_clients=[
+            {
+                "id": "badcafef00d",
+                "grant_types": [
+                    "urn:ietf:params:oauth:grant-type:device_code+pkce",
+                    "refresh_token",
+                ],
+            }
+        ],
+    ),
+    # Allow testing with Keycloak setup running in docker on localhost.
+    OidcProvider(
+        id="local",
+        title="Local Keycloak",
+        issuer="http://localhost:9090/auth/realms/master",
+        scopes=["openid"],
+    ),
+    # Allow testing the dummy backend with EGI
+    OidcProvider(
+        id="egi",
+        issuer="https://aai.egi.eu/auth/realms/egi/",
+        scopes=[
+            "openid",
+            "email",
+            "eduperson_entitlement",
+            "eduperson_scoped_affiliation",
+        ],
+        title="EGI Check-in",
+    ),
+    OidcProvider(
+        id="egi-dev",
+        issuer="https://aai-dev.egi.eu/auth/realms/egi",
+        scopes=[
+            "openid",
+            "email",
+            "eduperson_entitlement",
+            "eduperson_scoped_affiliation",
+        ],
+        title="EGI Check-in (dev)",
+    ),
+]
+
+
+config = OpenEoBackendConfig(
+    id="dummy",
+    oidc_providers=oidc_providers,
+)

tests/conftest.py

@@ -9,7 +9,9 @@
 import pytest
 import pythonjsonlogger.jsonlogger
 
+import openeo_driver.dummy.dummy_config
 from openeo_driver.backend import UserDefinedProcesses
+from openeo_driver.config import OpenEoBackendConfig
 from openeo_driver.dummy.dummy_backend import DummyBackendImplementation
 from openeo_driver.server import build_backend_deploy_metadata
 from openeo_driver.testing import UrllibMocker
@@ -25,9 +27,15 @@ def pytest_configure(config):
     os.environ["TZ"] = "UTC"
     time.tzset()
 
+
+@pytest.fixture(scope="session")
+def backend_config() -> OpenEoBackendConfig:
+    return openeo_driver.dummy.dummy_config.config
+
+
 @pytest.fixture(scope="module")
-def backend_implementation() -> DummyBackendImplementation:
-    return DummyBackendImplementation()
+def backend_implementation(backend_config) -> DummyBackendImplementation:
+    return DummyBackendImplementation(config=backend_config)
 
 
 @pytest.fixture

tests/test_config.py

@@ -0,0 +1,83 @@
+import random
+import textwrap
+
+import attrs.exceptions
+import pytest
+
+from openeo_driver.config import (
+    OpenEoBackendConfig,
+    get_backend_config,
+    ConfigException,
+)
+from openeo_driver.config.load import load_from_py_file
+
+
+def test_config_immutable():
+    conf = OpenEoBackendConfig(id="dontchangeme!")
+    assert conf.id == "dontchangeme!"
+    with pytest.raises(attrs.exceptions.FrozenInstanceError):
+        conf.id = "let's try?"
+    assert conf.id == "dontchangeme!"
+
+
+def test_load_from_py_file_default(tmp_path):
+    path = tmp_path / "myconfig.py"
+    cid = f"config-{random.randint(1, 100000)}"
+
+    content = f"""
+        from openeo_driver.config import OpenEoBackendConfig
+        cid = {cid!r}
+        config = OpenEoBackendConfig(
+            id=cid
+        )
+    """
+    content = textwrap.dedent(content)
+    path.write_text(content)
+
+    config = load_from_py_file(path)
+    assert isinstance(config, OpenEoBackendConfig)
+    assert config.id == cid
+
+
+def test_load_from_py_file_custom(tmp_path):
+    path = tmp_path / "myconfig.py"
+    cid = f"config-{random.randint(1, 100000)}"
+    path.write_text(f'konff = ("hello world", {cid!r}, list(range(3)))')
+    config = load_from_py_file(path, variable="konff", expected_class=tuple)
+    assert isinstance(config, tuple)
+    assert config == ("hello world", cid, [0, 1, 2])
+
+
+def test_load_from_py_file_wrong_type(tmp_path):
+    path = tmp_path / "myconfig.py"
+    path.write_text(f"config = [3, 5, 8]")
+    with pytest.raises(
+        ConfigException, match="Expected OpenEoBackendConfig but got list"
+    ):
+        _ = load_from_py_file(path)
+
+
+def test_get_backend_config(monkeypatch, tmp_path):
+    path = tmp_path / "myconfig.py"
+    content = """
+        import random
+        from openeo_driver.config import OpenEoBackendConfig
+        config = OpenEoBackendConfig(
+            id=f"config-{random.randint(1, 100000)}"
+        )
+    """
+    content = textwrap.dedent(content)
+    path.write_text(content)
+
+    monkeypatch.setenv("OPENEO_BACKEND_CONFIG", str(path))
+
+    get_backend_config.flush()
+    config1 = get_backend_config()
+    assert isinstance(config1, OpenEoBackendConfig)
+
+    config2 = get_backend_config()
+    assert config2 is config1
+
+    get_backend_config.flush()
+    config3 = get_backend_config()
+    assert not (config3 is config1)