From be64faf9c2bb641d207a2b8a22a03ba68217682c Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Thu, 4 Nov 2021 17:44:44 +0000 Subject: [PATCH] add configuration option for datagateway and panosc mode #256 BREAKING CHANGE: extend configuration to allow for different API modes --- datagateway_api/common/config.py | 128 ++++++----- .../common/datagateway_api/icat/filters.py | 4 +- .../datagateway_api/icat/icat_client_pool.py | 7 +- .../common/datagateway_api/icat/lru_cache.py | 2 +- .../datagateway_api/query_filter_factory.py | 2 +- datagateway_api/config.json.example | 42 ++-- datagateway_api/src/api_start_utils.py | 198 ++++++++++-------- util/icat_db_generator.py | 2 +- 8 files changed, 227 insertions(+), 158 deletions(-) diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index b55f3f0b..73a92b1f 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -17,32 +17,12 @@ log = logging.getLogger() -class TestUserCredentials(BaseModel): - username: StrictStr - password: StrictStr - - -class APIConfig(BaseModel): +class DatagatewayAPI(BaseModel): """ Configuration model class that implements pydantic's BaseModel class to allow for - validation of the API config data using Python type annotations. It ensures that - all config options exist before getting too far into the setup of the API. It takes - the backend into account, meaning only the config options for the backend used is + validation of the DatagatewayAPI config data using Python type annotations. It takes + the backend into account, meaning only the config options for the backend used are required. - - If a mandatory config option is missing or misspelled, or has a wrong value type, - Pydantic raises a validation error with a breakdown of what was wrong and the - application is exited. - - Config options used for testing are not checked here as they should only be used - during tests, not in the typical running of the API. - - Some options used when running the API (host, debug_mode etc.) aren't mandatory - when running the API in production (these options aren't used in the `wsgi.py` - entrypoint). As a result, they're not present in `config_keys`. However, they - are required when using `main.py` as an entrypoint. In any case of these - specific missing config options when using that entrypoint, they are checked at - API startup so any missing options will be caught quickly. """ backend: StrictStr @@ -50,34 +30,9 @@ class APIConfig(BaseModel): client_pool_init_size: Optional[StrictInt] client_pool_max_size: Optional[StrictInt] db_url: Optional[StrictStr] - debug_mode: StrictBool - flask_reloader: StrictBool - generate_swagger: StrictBool - host: StrictStr + extension: StrictStr icat_check_cert: Optional[StrictBool] icat_url: Optional[StrictStr] - log_level: StrictStr - log_location: StrictStr - port: StrictStr - test_mechanism: StrictStr - test_user_credentials: TestUserCredentials - - @classmethod - def load(cls, path=Path(__file__).parent.parent / "config.json"): - """ - Loads the config data from the JSON file and returns it as a APIConfig pydantic - model. Exits the application if it fails to locate the JSON config file or - the APIConfig model validation fails. - - :param path: path to the configuration file - :return: APIConfig model object that contains the config data - """ - try: - with open(path, encoding="utf-8") as target: - data = json.load(target) - return cls(**data) - except (IOError, ValidationError) as error: - sys.exit(f"An error occurred while trying to load the config data: {error}") @validator("db_url", always=True) def require_db_config_value(cls, value, values): # noqa: B902, N805 @@ -136,7 +91,82 @@ def set_backend_type(self, backend_type): self.backend = backend_type class Config: + """ + The behaviour of the BaseModel class can be controlled via this class. + """ + + # Enables assignment validation on the BaseModel fields. Useful for when the + # backend type is changed using the set_backend_type function. validate_assignment = True +class SearchAPI(BaseModel): + """ + Configuration model class that implements pydantic's BaseModel class to allow for + validation of the SearchAPI config data using Python type annotations. + """ + + client_pool_init_size: StrictInt + client_pool_max_size: StrictInt + extension: StrictStr + icat_check_cert: StrictBool + icat_url: StrictStr + + +class TestUserCredentials(BaseModel): + username: StrictStr + password: StrictStr + + +class APIConfig(BaseModel): + """ + Configuration model class that implements pydantic's BaseModel class to allow for + validation of the API config data using Python type annotations. It ensures that + all required config options exist before getting too far into the setup of the API. + + If a mandatory config option is missing or misspelled, or has a wrong value type, + Pydantic raises a validation error with a breakdown of what was wrong and the + application is exited. + + Config options used for testing are not checked here as they should only be used + during tests, not in the typical running of the API. + + Some options used when running the API (host, debug_mode etc.) aren't mandatory + when running the API in production (these options aren't used in the `wsgi.py` + entrypoint). As a result, they're not present in `config_keys`. However, they + are required when using `main.py` as an entrypoint. In any case of these + specific missing config options when using that entrypoint, they are checked at + API startup so any missing options will be caught quickly. + """ + + datagateway_api: Optional[DatagatewayAPI] + debug_mode: StrictBool + flask_reloader: StrictBool + generate_swagger: StrictBool + host: StrictStr + log_level: StrictStr + log_location: StrictStr + port: StrictStr + search_api: Optional[SearchAPI] + test_mechanism: StrictStr + test_user_credentials: TestUserCredentials + + @classmethod + def load(cls, path=Path(__file__).parent.parent / "config.json"): + """ + Loads the config data from the JSON file and returns it as a APIConfig pydantic + model. Exits the application if it fails to locate the JSON config file or + the APIConfig model validation fails. + + :param path: path to the configuration file + :return: APIConfig model object that contains the config data + """ + try: + with open(path, encoding="utf-8") as target: + data = json.load(target) + return cls(**data) + except (IOError, ValidationError) as error: + sys.exit(f"An error occurred while trying to load the config data: {error}") + + config = APIConfig.load() diff --git a/datagateway_api/common/datagateway_api/icat/filters.py b/datagateway_api/common/datagateway_api/icat/filters.py index 57d5d8d3..b7692303 100644 --- a/datagateway_api/common/datagateway_api/icat/filters.py +++ b/datagateway_api/common/datagateway_api/icat/filters.py @@ -214,7 +214,9 @@ def __init__(self, skip_value): super().__init__(skip_value) def apply_filter(self, query): - icat_properties = get_icat_properties(config.icat_url, config.icat_check_cert) + icat_properties = get_icat_properties( + config.datagateway_api.icat_url, config.datagateway_api.icat_check_cert, + ) icat_set_limit(query, self.skip_value, icat_properties["maxEntities"]) diff --git a/datagateway_api/common/datagateway_api/icat/icat_client_pool.py b/datagateway_api/common/datagateway_api/icat/icat_client_pool.py index bb1eba28..7ed1a689 100644 --- a/datagateway_api/common/datagateway_api/icat/icat_client_pool.py +++ b/datagateway_api/common/datagateway_api/icat/icat_client_pool.py @@ -13,7 +13,8 @@ class ICATClient(Client): def __init__(self): super().__init__( - config.icat_url, checkCert=config.icat_check_cert, + config.datagateway_api.icat_url, + checkCert=config.datagateway_api.icat_check_cert, ) # When clients are cleaned up, sessions won't be logged out self.autoLogout = False @@ -35,8 +36,8 @@ def create_client_pool(): return ObjectPool( ICATClient, - min_init=config.client_pool_init_size, - max_capacity=config.client_pool_max_size, + min_init=config.datagateway_api.client_pool_init_size, + max_capacity=config.datagateway_api.client_pool_max_size, max_reusable=0, expires=0, ) diff --git a/datagateway_api/common/datagateway_api/icat/lru_cache.py b/datagateway_api/common/datagateway_api/icat/lru_cache.py index 010118e4..4f00e54a 100644 --- a/datagateway_api/common/datagateway_api/icat/lru_cache.py +++ b/datagateway_api/common/datagateway_api/icat/lru_cache.py @@ -19,7 +19,7 @@ class ExtendedLRUCache(LRUCache): """ def __init__(self): - super().__init__(maxsize=config.client_cache_size) + super().__init__(maxsize=config.datagateway_api.client_cache_size) def popitem(self): key, client = super().popitem() diff --git a/datagateway_api/common/datagateway_api/query_filter_factory.py b/datagateway_api/common/datagateway_api/query_filter_factory.py index 30c4a7de..fdba98be 100644 --- a/datagateway_api/common/datagateway_api/query_filter_factory.py +++ b/datagateway_api/common/datagateway_api/query_filter_factory.py @@ -27,7 +27,7 @@ def get_query_filter(request_filter): :raises FilterError: If the filter name is not recognised """ - backend_type = config.backend + backend_type = config.datagateway_api.backend if backend_type == "db": from datagateway_api.common.datagateway_api.database.filters import ( DatabaseDistinctFieldFilter as DistinctFieldFilter, diff --git a/datagateway_api/config.json.example b/datagateway_api/config.json.example index 68137ce9..a82f1c4c 100644 --- a/datagateway_api/config.json.example +++ b/datagateway_api/config.json.example @@ -1,18 +1,28 @@ { - "backend": "db", - "client_cache_size": 5, - "client_pool_init_size": 2, - "client_pool_max_size": 5, - "db_url": "mysql+pymysql://icatdbuser:icatdbuserpw@localhost:3306/icatdb", - "flask_reloader": false, - "icat_url": "https://localhost:8181", - "icat_check_cert": false, - "log_level": "WARN", - "log_location": "/home/runner/work/datagateway-api/datagateway-api/logs.log", - "debug_mode": false, - "generate_swagger": false, - "host": "127.0.0.1", - "port": "5000", - "test_user_credentials": {"username": "root", "password": "pw"}, - "test_mechanism": "simple" + "datagateway_api": { + "extension": "/datagateway-api", + "backend": "db", + "client_cache_size": 5, + "client_pool_init_size": 2, + "client_pool_max_size": 5, + "db_url": "mysql+pymysql://icatdbuser:icatdbuserpw@localhost:3306/icatdb", + "icat_url": "https://localhost:8181", + "icat_check_cert": false + }, + "search_api": { + "extension": "/search-api", + "icat_url": "https://localhost:8181", + "icat_check_cert": false, + "client_pool_init_size": 2, + "client_pool_max_size": 5 + }, + "flask_reloader": false, + "log_level": "WARN", + "log_location": "/home/runner/work/datagateway-api/datagateway-api/logs.log", + "debug_mode": false, + "generate_swagger": false, + "host": "127.0.0.1", + "port": "5000", + "test_user_credentials": {"username": "root", "password": "pw"}, + "test_mechanism": "simple" } diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index 370c1052..c5e2ba68 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -8,8 +8,13 @@ from flask_swagger_ui import get_swaggerui_blueprint from datagateway_api.common.config import config -from datagateway_api.common.datagateway_api.backends import create_backend -from datagateway_api.common.datagateway_api.database.helpers import db + +# Only attempt to create a DataGateway API backend if the datagateway_api object +# is present in the config. This ensures that the API does not error on startup +# due to an AttributeError exception being thrown if the object is missing. +if config.datagateway_api is not None: + from datagateway_api.common.datagateway_api.backends import create_backend +from datagateway_api.common.datagateway_api.database.helpers import db # noqa: I202 from datagateway_api.common.datagateway_api.icat.icat_client_pool import ( create_client_pool, ) @@ -67,16 +72,17 @@ def create_app_infrastructure(flask_app): flask_app.url_map.strict_slashes = False api = CustomErrorHandledApi(flask_app) - try: - backend_type = flask_app.config["TEST_BACKEND"] - config.set_backend_type(backend_type) - except KeyError: - backend_type = config.backend + if config.datagateway_api is not None: + try: + backend_type = flask_app.config["TEST_BACKEND"] + config.datagateway_api.set_backend_type(backend_type) + except KeyError: + backend_type = config.datagateway_api.backend - if backend_type == "db": - flask_app.config["SQLALCHEMY_DATABASE_URI"] = config.db_url - flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False - db.init_app(flask_app) + if backend_type == "db": + flask_app.config["SQLALCHEMY_DATABASE_URI"] = config.datagateway_api.db_url + flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False + db.init_app(flask_app) initialise_spec(spec) @@ -84,92 +90,112 @@ def create_app_infrastructure(flask_app): def create_api_endpoints(flask_app, api, spec): - try: - backend_type = flask_app.config["TEST_BACKEND"] - config.set_backend_type(backend_type) - except KeyError: - backend_type = config.backend - - backend = create_backend(backend_type) - - icat_client_pool = None - if backend_type == "python_icat": - # Create client pool - icat_client_pool = create_client_pool() - - for entity_name in endpoints: - get_endpoint_resource = get_endpoint( - entity_name, endpoints[entity_name], backend, client_pool=icat_client_pool, + if config.datagateway_api is not None: + try: + backend_type = flask_app.config["TEST_BACKEND"] + config.datagateway_api.set_backend_type(backend_type) + except KeyError: + backend_type = config.datagateway_api.backend + + backend = create_backend(backend_type) + + icat_client_pool = None + if backend_type == "python_icat": + # Create client pool + icat_client_pool = create_client_pool() + + for entity_name in endpoints: + get_endpoint_resource = get_endpoint( + entity_name, + endpoints[entity_name], + backend, + client_pool=icat_client_pool, + ) + api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") + spec.path(resource=get_endpoint_resource, api=api) + + get_id_endpoint_resource = get_id_endpoint( + entity_name, + endpoints[entity_name], + backend, + client_pool=icat_client_pool, + ) + api.add_resource( + get_id_endpoint_resource, f"/{entity_name.lower()}/", + ) + spec.path(resource=get_id_endpoint_resource, api=api) + + get_count_endpoint_resource = get_count_endpoint( + entity_name, + endpoints[entity_name], + backend, + client_pool=icat_client_pool, + ) + api.add_resource( + get_count_endpoint_resource, f"/{entity_name.lower()}/count", + ) + spec.path(resource=get_count_endpoint_resource, api=api) + + get_find_one_endpoint_resource = get_find_one_endpoint( + entity_name, + endpoints[entity_name], + backend, + client_pool=icat_client_pool, + ) + api.add_resource( + get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone", + ) + spec.path(resource=get_find_one_endpoint_resource, api=api) + + # Session endpoint + session_endpoint_resource = session_endpoints( + backend, client_pool=icat_client_pool, ) - api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") - spec.path(resource=get_endpoint_resource, api=api) + api.add_resource(session_endpoint_resource, "/sessions") + spec.path(resource=session_endpoint_resource, api=api) - get_id_endpoint_resource = get_id_endpoint( - entity_name, endpoints[entity_name], backend, client_pool=icat_client_pool, + # Table specific endpoints + instrument_facility_cycle_resource = instrument_facility_cycles_endpoint( + backend, client_pool=icat_client_pool, ) - api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") - spec.path(resource=get_id_endpoint_resource, api=api) - - get_count_endpoint_resource = get_count_endpoint( - entity_name, endpoints[entity_name], backend, client_pool=icat_client_pool, + api.add_resource( + instrument_facility_cycle_resource, "/instruments//facilitycycles", ) - api.add_resource(get_count_endpoint_resource, f"/{entity_name.lower()}/count") - spec.path(resource=get_count_endpoint_resource, api=api) + spec.path(resource=instrument_facility_cycle_resource, api=api) - get_find_one_endpoint_resource = get_find_one_endpoint( - entity_name, endpoints[entity_name], backend, client_pool=icat_client_pool, + count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( + backend, client_pool=icat_client_pool, ) api.add_resource( - get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone", + count_instrument_facility_cycle_res, + "/instruments//facilitycycles/count", ) - spec.path(resource=get_find_one_endpoint_resource, api=api) - - # Session endpoint - session_endpoint_resource = session_endpoints(backend, client_pool=icat_client_pool) - api.add_resource(session_endpoint_resource, "/sessions") - spec.path(resource=session_endpoint_resource, api=api) - - # Table specific endpoints - instrument_facility_cycle_resource = instrument_facility_cycles_endpoint( - backend, client_pool=icat_client_pool, - ) - api.add_resource( - instrument_facility_cycle_resource, "/instruments//facilitycycles", - ) - spec.path(resource=instrument_facility_cycle_resource, api=api) - - count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( - backend, client_pool=icat_client_pool, - ) - api.add_resource( - count_instrument_facility_cycle_res, - "/instruments//facilitycycles/count", - ) - spec.path(resource=count_instrument_facility_cycle_res, api=api) + spec.path(resource=count_instrument_facility_cycle_res, api=api) - instrument_investigation_resource = instrument_investigation_endpoint( - backend, client_pool=icat_client_pool, - ) - api.add_resource( - instrument_investigation_resource, - "/instruments//facilitycycles//investigations", - ) - spec.path(resource=instrument_investigation_resource, api=api) + instrument_investigation_resource = instrument_investigation_endpoint( + backend, client_pool=icat_client_pool, + ) + api.add_resource( + instrument_investigation_resource, + "/instruments//facilitycycles/" + "/investigations", + ) + spec.path(resource=instrument_investigation_resource, api=api) - count_instrument_investigation_resource = count_instrument_investigation_endpoint( - backend, client_pool=icat_client_pool, - ) - api.add_resource( - count_instrument_investigation_resource, - "/instruments//facilitycycles//investigations" - "/count", - ) - spec.path(resource=count_instrument_investigation_resource, api=api) + count_instrument_investigation_res = count_instrument_investigation_endpoint( + backend, client_pool=icat_client_pool, + ) + api.add_resource( + count_instrument_investigation_res, + "/instruments//facilitycycles/" + "/investigations/count", + ) + spec.path(resource=count_instrument_investigation_res, api=api) - # Ping endpoint - ping_resource = ping_endpoint(backend, client_pool=icat_client_pool) - api.add_resource(ping_resource, "/ping") - spec.path(resource=ping_resource, api=api) + # Ping endpoint + ping_resource = ping_endpoint(backend, client_pool=icat_client_pool) + api.add_resource(ping_resource, "/ping") + spec.path(resource=ping_resource, api=api) def openapi_config(spec): diff --git a/util/icat_db_generator.py b/util/icat_db_generator.py index 720a5648..94e4afaf 100644 --- a/util/icat_db_generator.py +++ b/util/icat_db_generator.py @@ -36,7 +36,7 @@ engine = create_engine( - config.db_url, poolclass=QueuePool, pool_size=100, max_overflow=0, + config.datagateway_api.db_url, poolclass=QueuePool, pool_size=100, max_overflow=0, ) session_factory = sessionmaker(engine) session = scoped_session(session_factory)()