diff --git a/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py b/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py index 9b2797974..a9d222121 100644 --- a/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py +++ b/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py @@ -18,8 +18,7 @@ class CondaLock(lock.LockPlugin): def _conda_command(self, conda_store) -> str: - with conda_store.session_factory() as db: - settings = conda_store.get_settings(db=db) + settings = conda_store.get_settings() return settings.conda_command def _conda_flags(self, conda_store) -> str: diff --git a/conda-store-server/conda_store_server/_internal/server/app.py b/conda-store-server/conda_store_server/_internal/server/app.py index 0995f5199..0c149f533 100644 --- a/conda-store-server/conda_store_server/_internal/server/app.py +++ b/conda-store-server/conda_store_server/_internal/server/app.py @@ -388,7 +388,6 @@ def start(self): dbutil.upgrade(self.conda_store.config.database_url) with self.conda_store.session_factory() as db: - self.conda_store.ensure_settings(db) self.conda_store.ensure_namespace(db) self.conda_store.ensure_conda_channels(db) diff --git a/conda-store-server/conda_store_server/_internal/server/views/api.py b/conda-store-server/conda_store_server/_internal/server/views/api.py index 7ff6770c7..791ab4020 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/api.py +++ b/conda-store-server/conda_store_server/_internal/server/views/api.py @@ -1429,28 +1429,25 @@ async def api_get_settings( namespace: str = None, environment_name: str = None, ): - with conda_store.get_db() as db: - if namespace is None: - arn = "" - elif environment_name is None: - arn = namespace - else: - arn = f"{namespace}/{environment_name}" - - auth.authorize_request( - request, - arn, - {Permissions.SETTING_READ}, - require=True, - ) + if namespace is None: + arn = "" + elif environment_name is None: + arn = namespace + else: + arn = f"{namespace}/{environment_name}" + + auth.authorize_request( + request, + arn, + {Permissions.SETTING_READ}, + require=True, + ) - return { - "status": "ok", - "data": conda_store.get_settings( - db, namespace, environment_name - ).model_dump(), - "message": None, - } + return { + "status": "ok", + "data": conda_store.get_settings(namespace, environment_name).model_dump(), + "message": None, + } @router_api.put( @@ -1473,28 +1470,27 @@ async def api_put_settings( namespace: str = None, environment_name: str = None, ): - with conda_store.get_db() as db: - if namespace is None: - arn = "" - elif environment_name is None: - arn = namespace - else: - arn = f"{namespace}/{environment_name}" + if namespace is None: + arn = "" + elif environment_name is None: + arn = namespace + else: + arn = f"{namespace}/{environment_name}" + + auth.authorize_request( + request, + arn, + {Permissions.SETTING_UPDATE}, + require=True, + ) - auth.authorize_request( - request, - arn, - {Permissions.SETTING_UPDATE}, - require=True, - ) + try: + conda_store.set_settings(namespace, environment_name, data) + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) - try: - conda_store.set_settings(db, namespace, environment_name, data) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e.args[0])) - - return { - "status": "ok", - "data": None, - "message": f"global setting keys {list(data.keys())} updated", - } + return { + "status": "ok", + "data": None, + "message": f"global setting keys {list(data.keys())} updated", + } diff --git a/conda-store-server/conda_store_server/_internal/server/views/ui.py b/conda-store-server/conda_store_server/_internal/server/views/ui.py index 1d7643f02..63399d2f7 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/ui.py +++ b/conda-store-server/conda_store_server/_internal/server/views/ui.py @@ -293,34 +293,33 @@ async def ui_get_setting( namespace: str = None, environment_name: str = None, ): - with conda_store.get_db() as db: - if namespace is None: - arn = "" - elif environment_name is None: - arn = namespace - else: - arn = f"{namespace}/{environment_name}" - - auth.authorize_request( - request, - arn, - {Permissions.SETTING_READ}, - require=True, - ) - - api_setting_url = str(request.url_for("api_put_settings")) - if namespace is not None: - api_setting_url += f"{namespace}/" - if environment_name is not None: - api_setting_url += f"{environment_name}/" - - context = { - "request": request, - "namespace": namespace, - "environment_name": environment_name, - "api_settings_url": api_setting_url, - "settings": conda_store.get_settings( - db, namespace=namespace, environment_name=environment_name - ), - } + if namespace is None: + arn = "" + elif environment_name is None: + arn = namespace + else: + arn = f"{namespace}/{environment_name}" + + auth.authorize_request( + request, + arn, + {Permissions.SETTING_READ}, + require=True, + ) + + api_setting_url = str(request.url_for("api_put_settings")) + if namespace is not None: + api_setting_url += f"{namespace}/" + if environment_name is not None: + api_setting_url += f"{environment_name}/" + + context = { + "request": request, + "namespace": namespace, + "environment_name": environment_name, + "api_settings_url": api_setting_url, + "settings": conda_store.get_settings( + namespace=namespace, environment_name=environment_name + ), + } return templates.TemplateResponse(request, "setting.html", context) diff --git a/conda-store-server/conda_store_server/_internal/settings.py b/conda-store-server/conda_store_server/_internal/settings.py new file mode 100644 index 000000000..d76472b42 --- /dev/null +++ b/conda-store-server/conda_store_server/_internal/settings.py @@ -0,0 +1,178 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import functools +from typing import Any, Callable, Dict + +import pydantic +from sqlalchemy.orm import Session + +from conda_store_server import api +from conda_store_server._internal import schema + + +class Settings: + def __init__(self, db: Session, deployment_default: schema.Settings): + self.db = db + self.deployment_default = deployment_default.model_dump() + + def _ensure_closed_session(func: Callable): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + result = func(self, *args, **kwargs) + self.db.close() + return result + + return wrapper + + @_ensure_closed_session + def set_settings( + self, + namespace: str | None = None, + environment_name: str | None = None, + data: Dict[str, Any] | None = None, + ): + """Persist settings to the database + + Parameters + ---------- + namespace : str, optional + namespace to use to retrieve settings + environment_name : str, optional + environment to use to retrieve settings + data : dict + settings to upsert + """ + if data is None: + return + + # if an environment is specified without a namespace, it is not possible to + # determine which environment the setting should be applied to. Raise an error. + if namespace is None and environment_name is not None: + raise ValueError( + f"Environment {environment_name} is set without a namespace. Please specify a namespace." + ) + + setting_keys = schema.Settings.model_fields.keys() + if not data.keys() <= setting_keys: + invalid_keys = data.keys() - setting_keys + raise ValueError(f"Invalid setting keys {invalid_keys}") + + for key, value in data.items(): + field = schema.Settings.model_fields[key] + global_setting = field.json_schema_extra["metadata"]["global"] + if global_setting and ( + namespace is not None or environment_name is not None + ): + raise ValueError( + f"Setting {key} is a global setting and cannot be set within a namespace or environment" + ) + + validator = pydantic.TypeAdapter(field.annotation) + validator.validate_python(value) + + if namespace is not None and environment_name is not None: + prefix = f"setting/{namespace}/{environment_name}" + elif namespace is not None: + prefix = f"setting/{namespace}" + else: + prefix = "setting" + + api.set_kvstore_key_values(self.db, prefix, data) + + @_ensure_closed_session + def get_settings( + self, namespace: str | None = None, environment_name: str | None = None + ) -> schema.Settings: + """Get full schema.settings object for a given level of specificity. + If no namespace or environment is given, then the default settings + are returned. Settings merged follow the merge rules for updating + dict's. So, lists and dict fields are overwritten opposed to merged. + + Parameters + ---------- + namespace : str, optional + namespace to use to retrieve settings + environment_name : str, optional + environment to use to retrieve settings + + Returns + ------- + schema.Settings + merged settings object + """ + # build default/global settings object + settings = self.deployment_default + settings.update(api.get_kvstore_key_values(self.db, "setting")) + + # bulid list of prefixes to check from least precidence to highest precedence + prefixes = [] + if namespace is not None: + prefixes.append(f"setting/{namespace}") + if namespace is not None and environment_name is not None: + prefixes.append(f"setting/{namespace}/{environment_name}") + + if len(prefixes) > 0: + # get the fields that scoped globally. These are the keys that will NOT be + # merged on for namespace and environment prefixes. + global_fields = [ + k + for k, v in schema.Settings.model_fields.items() + if v.json_schema_extra["metadata"]["global"] + ] + # start building settings with the least specific defaults + for prefix in prefixes: + new_settings = api.get_kvstore_key_values(self.db, prefix) + # remove any global fields + new_settings = { + k: v for k, v in new_settings.items() if k not in global_fields + } + settings.update(new_settings) + + return schema.Settings(**settings) + + @_ensure_closed_session + def get_setting( + self, + key: str, + namespace: str | None = None, + environment_name: str | None = None, + ) -> Any: # noqa: ANN401 + """Get a given setting at the given level of specificity. Will short + cut and look up global setting directly even if a namespace/environment + is specified + + Parameters + ---------- + key : str + name of the setting to return + namespace : str, optional + namespace to use to retrieve settings + environment_name : str, optional + environment to use to retrieve settings + + Returns + ------- + Any + setting value, merged for the given level of specificity + """ + field = schema.Settings.model_fields.get(key) + if field is None: + return None + + prefixes = ["setting"] + if field.json_schema_extra["metadata"]["global"] is False: + if namespace is not None: + prefixes.append(f"setting/{namespace}") + if environment_name is not None: + prefixes.append(f"setting/{namespace}/{environment_name}") + + # start building settings with the least specific defaults + result = self.deployment_default.get(key) + for prefix in prefixes: + value = api.get_kvstore_key(self.db, prefix, key) + if value is not None: + result = value + + return result diff --git a/conda-store-server/conda_store_server/_internal/worker/build.py b/conda-store-server/conda_store_server/_internal/worker/build.py index 0e6489742..04b25d3a8 100644 --- a/conda-store-server/conda_store_server/_internal/worker/build.py +++ b/conda-store-server/conda_store_server/_internal/worker/build.py @@ -198,7 +198,6 @@ def build_conda_environment(db: Session, conda_store, build): ) settings = conda_store.get_settings( - db=db, namespace=build.environment.namespace.name, environment_name=build.environment.name, ) @@ -361,7 +360,6 @@ def solve_conda_environment(db: Session, conda_store, solve: orm.Solve): def build_conda_env_export(db: Session, conda_store, build: orm.Build): conda_prefix = build.build_path(conda_store) settings = conda_store.get_settings( - db=db, namespace=build.environment.namespace.name, environment_name=build.environment.name, ) @@ -431,7 +429,6 @@ def build_constructor_installer(db: Session, conda_store, build: orm.Build): conda_prefix = build.build_path(conda_store) settings = conda_store.get_settings( - db=db, namespace=build.environment.namespace.name, environment_name=build.environment.name, ) diff --git a/conda-store-server/conda_store_server/_internal/worker/tasks.py b/conda-store-server/conda_store_server/_internal/worker/tasks.py index b0cddd1bc..bd6b6af84 100644 --- a/conda-store-server/conda_store_server/_internal/worker/tasks.py +++ b/conda-store-server/conda_store_server/_internal/worker/tasks.py @@ -84,7 +84,7 @@ def task_watch_paths(self): conda_store = self.worker.conda_store with conda_store.session_factory() as db: - settings = conda_store.get_settings(db) + settings = conda_store.get_settings() conda_store.configuration(db).update_storage_metrics( db, conda_store.config.store_directory @@ -154,7 +154,7 @@ def task_update_conda_channels(self): def task_update_conda_channel(self, channel_name): conda_store = self.worker.conda_store with conda_store.session_factory() as db: - settings = conda_store.get_settings(db) + settings = conda_store.get_settings() # sanitize the channel name as it's an URL, and it's used for the lock. sanitizing = { @@ -283,7 +283,7 @@ def delete_build_artifact(db: Session, conda_store, build_artifact): def task_delete_build(self, build_id): conda_store = self.worker.conda_store with conda_store.session_factory() as db: - settings = conda_store.get_settings(db) + settings = conda_store.get_settings() build = api.get_build(db, build_id) diff --git a/conda-store-server/conda_store_server/api.py b/conda-store-server/conda_store_server/api.py index aaf01c26d..0cc37456e 100644 --- a/conda-store-server/conda_store_server/api.py +++ b/conda-store-server/conda_store_server/api.py @@ -798,6 +798,19 @@ def get_kvstore_key_values(db, prefix: str): } +def get_kvstore_key(db, prefix: str, key: str): + """Get value for a particular prefix and key""" + row = ( + db.query(orm.KeyValueStore) + .filter(orm.KeyValueStore.prefix == prefix) + .filter(orm.KeyValueStore.key == key) + .first() + ) + if row is None: + return None + return row.value + + def set_kvstore_key_values(db, prefix: str, d: Dict[str, Any], update: bool = True): """Set key, values for a particular prefix""" for key, value in d.items(): diff --git a/conda-store-server/conda_store_server/conda_store.py b/conda-store-server/conda_store_server/conda_store.py index cccb9a16c..7f753d96e 100644 --- a/conda-store-server/conda_store_server/conda_store.py +++ b/conda-store-server/conda_store_server/conda_store.py @@ -8,13 +8,12 @@ from contextlib import contextmanager from typing import Any, Dict -import pydantic from celery import Celery, group from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool from conda_store_server import CONDA_STORE_DIR, api, conda_store_config, storage -from conda_store_server._internal import conda_utils, orm, schema, utils +from conda_store_server._internal import conda_utils, orm, schema, settings, utils from conda_store_server.exception import CondaStoreError from conda_store_server.plugins import hookspec, plugin_manager from conda_store_server.plugins.types import lock @@ -62,6 +61,21 @@ def get_db(self): finally: db.close() + @property + def settings(self): + if hasattr(self, "_settings"): + return self._settings + + with self.get_db() as db: + # setup the setting object with a session. Once this block finishes + # excuting, the db session will close. By default in sqlalchemy, this + # will release the connection, however, the connection may be restablished. + # ref: https://docs.sqlalchemy.org/en/20/orm/session_basics.html#closing + self._settings = settings.Settings( + db=db, deployment_default=schema.Settings(**self.config.trait_values()) + ) + return self._settings + @property def redis(self): import redis @@ -144,34 +158,6 @@ def lock_plugin(self) -> tuple[str, lock.LockPlugin]: locker = lock_plugin.backend() return lock_plugin.name, locker - def ensure_settings(self, db: Session): - """Ensure that conda-store traitlets settings are applied""" - settings = schema.Settings( - default_namespace=self.config.default_namespace, - filesystem_namespace=self.config.filesystem_namespace, - default_uid=self.config.default_uid, - default_gid=self.config.default_gid, - default_permissions=self.config.default_permissions, - storage_threshold=self.config.storage_threshold, - conda_command=self.config.conda_command, - conda_platforms=self.config.conda_platforms, - conda_max_solve_time=self.config.conda_max_solve_time, - conda_indexed_channels=self.config.conda_indexed_channels, - build_artifacts_kept_on_deletion=self.config.build_artifacts_kept_on_deletion, - conda_solve_platforms=self.config.conda_solve_platforms, - conda_channel_alias=self.config.conda_channel_alias, - conda_default_channels=self.config.conda_default_channels, - conda_allowed_channels=self.config.conda_allowed_channels, - conda_default_packages=self.config.conda_default_packages, - conda_required_packages=self.config.conda_required_packages, - conda_included_packages=self.config.conda_included_packages, - pypi_default_packages=self.config.pypi_default_packages, - pypi_required_packages=self.config.pypi_required_packages, - pypi_included_packages=self.config.pypi_included_packages, - build_artifacts=self.config.build_artifacts, - ) - api.set_kvstore_key_values(db, "setting", settings.model_dump(), update=False) - def ensure_namespace(self, db: Session): """Ensure that conda-store default namespaces exists""" api.ensure_namespace(db, self.config.default_namespace) @@ -184,7 +170,7 @@ def ensure_conda_channels(self, db: Session): """Ensure that conda-store indexed channels and packages are in database""" self.log.info("updating conda store channels") - settings = self.get_settings(db) + settings = self.get_settings() for channel in settings.conda_indexed_channels: normalized_channel = conda_utils.normalize_channel_name( @@ -194,66 +180,31 @@ def ensure_conda_channels(self, db: Session): def set_settings( self, - db: Session, namespace: str = None, environment_name: str = None, data: Dict[str, Any] = {}, ): - setting_keys = schema.Settings.model_fields.keys() - if not data.keys() <= setting_keys: - invalid_keys = data.keys() - setting_keys - raise ValueError(f"Invalid setting keys {invalid_keys}") - - for key, value in data.items(): - field = schema.Settings.model_fields[key] - global_setting = field.json_schema_extra["metadata"]["global"] - if global_setting and ( - namespace is not None or environment_name is not None - ): - raise ValueError( - f"Setting {key} is a global setting cannot be set within namespace or environment" - ) - - try: - validator = pydantic.TypeAdapter(field.annotation) - validator.validate_python(value) - except Exception as e: - raise ValueError( - f"Invalid parsing of setting {key} expected type {field.annotation} ran into error {e}" - ) - - if namespace is not None and environment_name is not None: - prefix = f"setting/{namespace}/{environment_name}" - elif namespace is not None: - prefix = f"setting/{namespace}" - else: - prefix = "setting" - - api.set_kvstore_key_values(db, prefix, data) + return self.settings.set_settings( + namespace=namespace, environment_name=environment_name, data=data + ) def get_settings( - self, db: Session, namespace: str = None, environment_name: str = None + self, namespace: str = None, environment_name: str = None ) -> schema.Settings: - # setting logic is intentionally done in python code - # rather than using the database for merges and ordering - # becuase in the future we may likely want to do some - # more complex logic around settings - - prefixes = ["setting"] - if namespace is not None: - prefixes.append(f"setting/{namespace}") - if namespace is not None and environment_name is not None: - prefixes.append(f"setting/{namespace}/{environment_name}") - - settings = {} - for prefix in prefixes: - settings.update(api.get_kvstore_key_values(db, prefix)) + return self.settings.get_settings( + namespace=namespace, environment_name=environment_name + ) - return schema.Settings(**settings) + def get_setting( + self, key: str, namespace: str = None, environment_name: str = None + ) -> schema.Settings: + return self.settings.get_setting( + key=key, namespace=namespace, environment_name=environment_name + ) def register_solve(self, db: Session, specification: schema.CondaSpecification): """Registers a solve for a given specification""" - settings = self.get_settings(db) + settings = self.get_settings() self.config.validate_action( db=db, @@ -295,7 +246,7 @@ def register_environment( is_lockfile: bool = False, ): """Register a given specification to conda store with given namespace/name.""" - settings = self.get_settings(db) + settings = self.get_settings() namespace = namespace or settings.default_namespace namespace = api.ensure_namespace(db, name=namespace) @@ -364,7 +315,7 @@ def create_build(self, db: Session, environment_id: int, specification_sha256: s ) settings = self.get_settings( - db, namespace=environment.namespace.name, environment_name=environment.name + namespace=environment.namespace.name, environment_name=environment.name ) specification = api.get_specification(db, specification_sha256) diff --git a/conda-store-server/conda_store_server/conda_store_config.py b/conda-store-server/conda_store_server/conda_store_config.py index c95dae81e..b98b870c2 100644 --- a/conda-store-server/conda_store_server/conda_store_config.py +++ b/conda-store-server/conda_store_server/conda_store_config.py @@ -31,7 +31,7 @@ def conda_store_validate_specification( specification: schema.CondaSpecification, ) -> schema.CondaSpecification: settings = conda_store.get_settings( - db, namespace=namespace, environment_name=specification.name + namespace=namespace, environment_name=specification.name ) specification = environment.validate_environment_channels(specification, settings) @@ -51,7 +51,7 @@ def conda_store_validate_action( namespace: str, action: auth_schema.Permissions, ) -> None: - settings = conda_store.get_settings(db) + settings = conda_store.get_settings() system_metrics = api.get_system_metrics(db) if action in ( diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 8db52c10f..59119dc14 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -190,6 +190,7 @@ ignore = [ "RET504", # unnecessary-assign "RET505", # superfluous-else-return "RET506", # superfluous-else-raise + "SIM102", # collapsible-if "SIM105", # suppressible-exception "SIM115", # open-file-with-context-handler "SIM117", # multiple-with-statements diff --git a/conda-store-server/tests/_internal/server/views/test_api.py b/conda-store-server/tests/_internal/server/views/test_api.py index 8ab794403..d11205e54 100644 --- a/conda-store-server/tests/_internal/server/views/test_api.py +++ b/conda-store-server/tests/_internal/server/views/test_api.py @@ -1008,7 +1008,7 @@ def test_put_environment_settings_auth_invliad_type(testclient, authenticate, ro r = schema.APIPutSetting.model_validate(response.json()) assert r.status == schema.APIStatus.ERROR - assert "Invalid parsing" in r.message + assert "validation error" in r.message @pytest.mark.parametrize( diff --git a/conda-store-server/tests/_internal/test_settings.py b/conda-store-server/tests/_internal/test_settings.py new file mode 100644 index 000000000..cb6e6fe5f --- /dev/null +++ b/conda-store-server/tests/_internal/test_settings.py @@ -0,0 +1,260 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from unittest import mock + +import pydantic +import pytest + +from conda_store_server import api +from conda_store_server._internal import schema +from conda_store_server._internal.settings import Settings + + +@pytest.fixture +def settings(db) -> Settings: + default_settings = schema.Settings( + default_uid=999, default_gid=999, conda_channel_alias="defaultchannelalias" + ) + + # setup test global settings + global_settings = { + "default_uid": 888, + "conda_channel_alias": "globalchannelalias", + "conda_command": "myglobalcondacommand", + } + api.set_kvstore_key_values(db, "setting", global_settings) + + # setup test namespace settings + namespace_settings = { + "conda_channel_alias": "namespacechannelalias", + "conda_command": "mynamespacecondacommand", + "conda_default_packages": ["ipykernel"], + } + api.set_kvstore_key_values(db, "setting/test_namespace", namespace_settings) + + # setup test namespace (two) settings + namespace_two_settings = { + "conda_channel_alias": "namespacechannelalias", + } + api.set_kvstore_key_values(db, "setting/test_namespace_two", namespace_two_settings) + + # setup test environment settings + environment_settings = { + "conda_channel_alias": "envchannelalias", + "conda_default_packages": ["numpy"], + } + api.set_kvstore_key_values( + db, "setting/test_namespace/test_env", environment_settings + ) + + return Settings(db=db, deployment_default=default_settings) + + +def test_ensure_session_is_closed(settings: Settings): + # run a query against the db to start a transaction + settings.get_settings() + # ensure that the settings object cleans up it's transaction + assert not settings.db.in_transaction() + + +@mock.patch("conda_store_server.api.get_kvstore_key_values") +def test_ensure_session_is_closed_on_error( + mock_get_kvstore_key_values, settings: Settings +): + mock_get_kvstore_key_values.side_effect = Exception + + # run a query that will raise an exception + with pytest.raises(Exception): + settings.get_settings() + + # ensure that the settings object cleans up it's transaction + assert not settings.db.in_transaction() + + +def test_get_settings_default(settings: Settings): + test_settings = settings.get_settings() + + # ensure that we get the deployment default values + assert test_settings.default_gid == 999 + + # ensure we get the settings overridden by the global settings + assert test_settings.default_uid == 888 + assert test_settings.conda_channel_alias == "globalchannelalias" + assert test_settings.conda_command == "myglobalcondacommand" + + # ensure that we get the default value for unset settings + assert test_settings.default_namespace == "default" + + +def test_get_settings_namespace(settings: Settings): + test_settings = settings.get_settings(namespace="test_namespace") + + # ensure that we get the deployment default values + assert test_settings.default_gid == 999 + + # ensure that we get the global defaults + assert test_settings.default_uid == 888 + + # ensure we get the settings overridden by the namespace settings + assert test_settings.conda_channel_alias == "namespacechannelalias" + # the "global" metadata setting should be respected. Since `conda_command` + # is scoped globally, even though we are looking at the "namespace" level, + # the "global" value should be returned + assert test_settings.conda_command == "myglobalcondacommand" + assert test_settings.conda_default_packages == ["ipykernel"] + + # ensure that we get the default value for unset settings + assert test_settings.default_namespace == "default" + + +def test_get_settings_namespace_two(settings: Settings): + test_settings = settings.get_settings(namespace="test_namespace_two") + + # ensure that we get the deployment default values + assert test_settings.default_gid == 999 + + # ensure that we get the global defaults + assert test_settings.default_uid == 888 + + # ensure we get the settings overridden by the namespace settings + assert test_settings.conda_channel_alias == "namespacechannelalias" + assert test_settings.conda_command == "myglobalcondacommand" + assert test_settings.conda_default_packages == [] + + # ensure that we get the default value for unset settings + assert test_settings.default_namespace == "default" + + +def test_get_settings_environment(settings: Settings): + test_settings = settings.get_settings( + namespace="test_namespace", environment_name="test_env" + ) + + # ensure that we get the deployment default values + assert test_settings.default_gid == 999 + + # ensure that we get the global defaults + assert test_settings.default_uid == 888 + + # ensure we get the settings overridden by the environment settings + assert test_settings.conda_channel_alias == "envchannelalias" + assert test_settings.conda_command == "myglobalcondacommand" + assert test_settings.conda_default_packages == ["numpy"] + + # ensure that we get the default value for unset settings + assert test_settings.default_namespace == "default" + + +def test_get_settings_namespace_dne(settings: Settings): + # get settings for namespace that does not exist - we should + # still get the default settings + test_settings = settings.get_settings(namespace="idontexist") + + # ensure that we get the deployment default values + assert test_settings.default_gid == 999 + + # ensure we get the settings overridden by the global settings + assert test_settings.default_uid == 888 + assert test_settings.conda_command == "myglobalcondacommand" + + +def test_set_settings_global_default(settings: Settings): + # set test settings + settings.set_settings(data={"default_uid": 0}) + + # ensure the setting is persisted to the right level + check_settings = settings.get_settings() + assert check_settings.default_uid == 0 + check_settings = settings.get_settings(namespace="test_namespace") + assert check_settings.default_uid == 0 + + +def test_set_settings_global_overriden_by_default(settings: Settings): + # set test settings + settings.set_settings(data={"conda_channel_alias": "newchanelalias"}) + + # ensure the setting is persisted to the right level + check_settings = settings.get_settings() + assert check_settings.conda_channel_alias == "newchanelalias" + check_settings = settings.get_settings(namespace="test_namespace") + assert check_settings.conda_channel_alias == "namespacechannelalias" + + +def test_set_settings_invalid_setting_field(settings: Settings): + with pytest.raises(ValueError, match=r"Invalid setting keys"): + settings.set_settings( + data={"idontexist": "sure", "conda_channel_alias": "mynewalias"} + ) + + +def test_set_settings_invalid_setting_type(settings: Settings): + with pytest.raises(pydantic.ValidationError): + settings.set_settings(data={"conda_channel_alias": [1, 2, 3]}) + + +def test_set_settings_invalid_level(settings: Settings): + with pytest.raises( + ValueError, + match="Setting default_uid is a global setting and cannot be set within a namespace or environment", + ): + settings.set_settings(namespace="mynamespace", data={"default_uid": 777}) + + +def test_set_settings_environment_without_namespace(settings: Settings): + with pytest.raises(ValueError, match=r"Please specify a namespace"): + settings.set_settings( + environment_name="test_env", data={"conda_channel_alias": "mynewalias"} + ) + + +def test_get_global_setting(settings: Settings): + test_setting = settings.get_setting("default_namespace") + assert test_setting == "default" + + +def test_get_setting_invalid(settings: Settings): + test_setting = settings.get_setting("notarealfield") + assert test_setting is None + + test_setting = settings.get_setting( + "notarealfield", namespace="test_namespace", environment_name="test_env" + ) + assert test_setting is None + + +def test_get_setting_overriden(settings: Settings): + # conda_channel_alias is not a global command, it may be overriden + test_setting = settings.get_setting("conda_channel_alias") + assert test_setting == "globalchannelalias" + # default_uid is a global setting. It should never be overriden + test_setting = settings.get_setting("default_uid") + assert test_setting == 888 + # conda_command is also a global setting. Even if somehow the value gets + # injected into the db (as in this test setup), get_setting should honour + # just the global setting + test_setting = settings.get_setting("conda_command") + assert test_setting == "myglobalcondacommand" + + test_setting = settings.get_setting( + "conda_channel_alias", namespace="test_namespace" + ) + assert test_setting == "namespacechannelalias" + test_setting = settings.get_setting("default_uid", namespace="test_namespace") + assert test_setting == 888 + test_setting = settings.get_setting("conda_command", namespace="test_namespace") + assert test_setting == "myglobalcondacommand" + + test_setting = settings.get_setting( + "conda_channel_alias", namespace="test_namespace", environment_name="test_env" + ) + assert test_setting == "envchannelalias" + test_setting = settings.get_setting( + "default_uid", namespace="test_namespace", environment_name="test_env" + ) + assert test_setting == 888 + test_setting = settings.get_setting( + "conda_command", namespace="test_namespace", environment_name="test_env" + ) + assert test_setting == "myglobalcondacommand" diff --git a/conda-store-server/tests/conftest.py b/conda-store-server/tests/conftest.py index 4a9c0c981..86cf972a7 100644 --- a/conda-store-server/tests/conftest.py +++ b/conda-store-server/tests/conftest.py @@ -129,7 +129,6 @@ def conda_store_api_server(conda_store_api_config): dbutil.upgrade(_conda_store.config.database_url) with _conda_store.session_factory() as db: - _conda_store.ensure_settings(db) _conda_store.configuration(db).update_storage_metrics( db, _conda_store.config.store_directory ) @@ -159,7 +158,6 @@ def conda_store_server(conda_store_config): dbutil.upgrade(_conda_store.config.database_url) with _conda_store.session_factory() as db: - _conda_store.ensure_settings(db) _conda_store.configuration(db).update_storage_metrics( db, _conda_store.config.store_directory ) @@ -252,7 +250,6 @@ def conda_store(conda_store_config): dbutil.upgrade(_conda_store.config.database_url) with _conda_store.session_factory() as db: - _conda_store.ensure_settings(db) _conda_store.configuration(db).update_storage_metrics( db, _conda_store.config.store_directory ) diff --git a/conda-store-server/tests/test_api.py b/conda-store-server/tests/test_api.py index 652eaee9e..608958a7b 100644 --- a/conda-store-server/tests/test_api.py +++ b/conda-store-server/tests/test_api.py @@ -339,17 +339,30 @@ def test_get_set_keyvaluestore(db): api.set_kvstore_key_values(db, "pytest/1", setting_2) api.set_kvstore_key_values(db, "pytest/1/2", setting_3) + # check get_kvstore_key_values assert setting_1 == api.get_kvstore_key_values(db, "pytest") assert setting_2 == api.get_kvstore_key_values(db, "pytest/1") assert setting_3 == api.get_kvstore_key_values(db, "pytest/1/2") + # check get_kvstore_value + assert api.get_kvstore_key(db, "pytest", "b") == 2 + assert api.get_kvstore_key(db, "pytest/1", "d") == 2 + assert api.get_kvstore_key(db, "pytest/1/2", "f") == 2 + # test updating a prefix api.set_kvstore_key_values(db, "pytest", setting_2) assert api.get_kvstore_key_values(db, "pytest") == {**setting_1, **setting_2} + assert api.get_kvstore_key(db, "pytest", "d") == 2 # test updating a prefix api.set_kvstore_key_values(db, "pytest", {"c": 999, "d": 999}, update=False) assert api.get_kvstore_key_values(db, "pytest") == {**setting_1, **setting_2} + assert api.get_kvstore_key(db, "pytest", "d") == 2 + + +def test_get_kvstore_key_dne(db): + # db starts empty, try to get a value that does not exist + assert api.get_kvstore_key(db, "pytest", "c") is None def test_build_path_too_long(db, conda_store, simple_specification): diff --git a/conda-store-server/tests/test_conda_store_config.py b/conda-store-server/tests/test_conda_store_config.py index edbb7dc79..f3c25e617 100644 --- a/conda-store-server/tests/test_conda_store_config.py +++ b/conda-store-server/tests/test_conda_store_config.py @@ -137,7 +137,6 @@ def a_route(a: str, b: str): def test_conda_store_settings_conda_channels_packages_validate_valid(db, conda_store): conda_store.set_settings( - db, data={ "conda_allowed_channels": ["conda-forge"], "conda_included_packages": ["ipykernel"], @@ -168,7 +167,6 @@ def test_conda_store_settings_conda_channels_packages_validate_valid(db, conda_s ] conda_store.set_settings( - db, namespace="default", data={ "conda_allowed_channels": ["conda-forge"], @@ -201,7 +199,6 @@ def test_conda_store_settings_conda_channels_packages_validate_valid(db, conda_s ] conda_store.set_settings( - db, namespace="default", environment_name="test", data={ diff --git a/docusaurus-docs/conda-store/references/configuring-conda-store.md b/docusaurus-docs/conda-store/references/configuring-conda-store.md new file mode 100644 index 000000000..c9f1674c1 --- /dev/null +++ b/docusaurus-docs/conda-store/references/configuring-conda-store.md @@ -0,0 +1,44 @@ +--- +description: configuring conda-store +--- + +# Configuring conda-store + +The conda-store server has two types of config. + +* server configuration config, eg. url for database endpoints, redis +endpoints, port to run on, etc. +* application settings config, eg. the default namespace, default conda +command, default packages to install in environments, etc. + +The server configuration is always specified in the [conda-store configuration file](./configuration-options.md). + +There are two options to set the application settings config. These +are through: + +* the conda-store configuration file. See the available options in the + [configuration options doc](./configuration-options.md) +* the conda-store admin console. Available at `/admin` + +There are multiple levels which settings config can be applied to conda store. + +* deployment defaults (controlled by the config file) +* global defaults (controlled by the admin console) +* namespace defaults (controlled by the admin console) +* environment default (controlled by the admin console) + +The most specific settings config will take precedence over the more +general setting. For example, if the `conda_command` is specified in +all four settings config levels, the most specific, environment settings +config will be applied. + +:::note +Since the deployment defaults are of lowest precedent when settings are +being generated, it is likely that they will be overridden by another +by another level, for example the global defaults. + +So, it is recommended that users change global defaults from the admin console. +::: + +The full list of application settings config is described in the +[settings pydantic model](https://github.com/conda-incubator/conda-store/blob/main/conda-store-server/conda_store_server/_internal/schema.py#L203).