From 72b805f8c054bfeabc6a9bf956a75ba07938cc4d Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 30 Aug 2024 14:16:53 +0200 Subject: [PATCH 1/2] Issue #117 create `openeo_aggregator.config` subpackage to allow including example configs in src tree --- README.md | 2 +- src/openeo_aggregator/config/__init__.py | 12 ++++++++++++ src/openeo_aggregator/{ => config}/config.py | 0 src/openeo_aggregator/testing.py | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 src/openeo_aggregator/config/__init__.py rename src/openeo_aggregator/{ => config}/config.py (100%) diff --git a/README.md b/README.md index 34f0abb..584cfc2 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ is grouped by an `AggregatorBackendConfig` container object (subclass of `OpenEoBackendConfig` as defined in the `openeo-python-driver` framework project). The most important config value is `aggregator_backends`, which defines the backends to "aggregate". -See `src/openeo_aggregator/config.py` for more details and other available configuration options. +See `src/openeo_aggregator/config/config.py` for more details and other available configuration options. The `conf` folder contains config files for the dev and production variant of this application config: diff --git a/src/openeo_aggregator/config/__init__.py b/src/openeo_aggregator/config/__init__.py new file mode 100644 index 0000000..fc3e5f5 --- /dev/null +++ b/src/openeo_aggregator/config/__init__.py @@ -0,0 +1,12 @@ +from openeo_aggregator.config.config import ( + CONNECTION_TIMEOUT_DEFAULT, + CONNECTION_TIMEOUT_INIT, + CONNECTION_TIMEOUT_JOB_LOGS, + CONNECTION_TIMEOUT_JOB_START, + CONNECTION_TIMEOUT_RESULT, + STREAM_CHUNK_SIZE_DEFAULT, + AggregatorBackendConfig, + ConfigException, + ProcessAllowed, + get_backend_config, +) diff --git a/src/openeo_aggregator/config.py b/src/openeo_aggregator/config/config.py similarity index 100% rename from src/openeo_aggregator/config.py rename to src/openeo_aggregator/config/config.py diff --git a/src/openeo_aggregator/testing.py b/src/openeo_aggregator/testing.py index 2d780c7..6cf60d8 100644 --- a/src/openeo_aggregator/testing.py +++ b/src/openeo_aggregator/testing.py @@ -321,6 +321,6 @@ def config_overrides(**kwargs): ... def test_stuff(): """ return openeo_driver.testing.config_overrides( - config_getter=openeo_aggregator.config._config_getter, + config_getter=openeo_aggregator.config.config._config_getter, **kwargs, ) From 187a125efe170368759265c54402fb3b1e0172fe Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 30 Aug 2024 17:29:41 +0200 Subject: [PATCH 2/2] Issue #117 move example configs inside package tree --- CHANGELOG.md | 4 ++++ Dockerfile | 1 + README.md | 12 +++++------- conf/gunicorn.dev.py | 1 + conf/gunicorn.prod.py | 1 + scripts/run-gunicorn.sh | 2 +- setup.py | 4 +++- src/openeo_aggregator/about.py | 2 +- .../config/examples}/aggregator.dummy.py | 0 .../config/examples/gunicorn-config.py | 9 +++++++++ 10 files changed, 26 insertions(+), 10 deletions(-) rename {conf => src/openeo_aggregator/config/examples}/aggregator.dummy.py (100%) create mode 100644 src/openeo_aggregator/config/examples/gunicorn-config.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 73167f7..489132d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is roughly based on [Keep a Changelog](https://keepachangelog.com/en/ +## 0.37.0 + +- move example configs inside package source tree ([#117](https://github.com/Open-EO/openeo-aggregator/issues/117)) + ## 0.36.0 - openeo.cloud configs: only consider "vito" for SENTINEL2_L2A collection ([#139](https://github.com/Open-EO/openeo-aggregator/issues/139)) diff --git a/Dockerfile b/Dockerfile index 1820f1b..d040b86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,4 +52,5 @@ RUN pip install --upgrade pip && \ pip install . +# TODO #117/#143 eliminate conf/gunicorn.prod.py reference CMD ["gunicorn", "--config=conf/gunicorn.prod.py", "openeo_aggregator.app:create_app()"] diff --git a/README.md b/README.md index 584cfc2..be2768f 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,10 @@ like command line options or env variables, as shown above. ### Gunicorn config -For gunicorn there are also configuration files in the `conf` folder. -The production docker based run for examples uses +For gunicorn there is an example config at `src/openeo_aggregator/config/examples/gunicorn-config.py`, +for example to be used like this: - gunicorn --config=conf/gunicorn.prod.py openeo_aggregator.app:create_app() + gunicorn --config=src/openeo_aggregator/config/examples/gunicorn-config.py openeo_aggregator.app:create_app() ### Application/Flask config @@ -78,11 +78,9 @@ variant of this application config: - `conf/aggregator.prod.py` Use the env var `OPENEO_BACKEND_CONFIG` to point to the desired config path. -By default, `conf/aggregator.dummy.py` is loaded. +For example, using the example dummy config from the repo: -For example, when using the Docker image, add something like this - - -e OPENEO_BACKEND_CONFIG=/home/openeo/aggregator/conf/aggregator.dev.py + export OPENEO_BACKEND_CONFIG=src/openeo_aggregator/config/examples/aggregator.dummy.py Also note that these concrete config files will be refactored out of the `openeo-aggregator` repo at some point in the future ([#117](https://github.com/Open-EO/openeo-aggregator/issues/117)) diff --git a/conf/gunicorn.dev.py b/conf/gunicorn.dev.py index dbbe8df..260f185 100644 --- a/conf/gunicorn.dev.py +++ b/conf/gunicorn.dev.py @@ -1,4 +1,5 @@ # See https://docs.gunicorn.org/en/stable/settings.html +# TODO #117 eliminate this file from this repository bind = ["0.0.0.0:8080"] diff --git a/conf/gunicorn.prod.py b/conf/gunicorn.prod.py index b851c6d..1554834 100644 --- a/conf/gunicorn.prod.py +++ b/conf/gunicorn.prod.py @@ -1,4 +1,5 @@ # See https://docs.gunicorn.org/en/stable/settings.html +# TODO #117 eliminate this file from this repository bind = ["0.0.0.0:8080"] diff --git a/scripts/run-gunicorn.sh b/scripts/run-gunicorn.sh index ac5ac91..9910464 100755 --- a/scripts/run-gunicorn.sh +++ b/scripts/run-gunicorn.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -gunicorn --config=conf/gunicorn.dev.py 'openeo_aggregator.app:create_app()' +gunicorn --config=src/openeo_aggregator/config/examples/gunicorn-config.py 'openeo_aggregator.app:create_app()' diff --git a/setup.py b/setup.py index 9f8ec35..1f6ee04 100644 --- a/setup.py +++ b/setup.py @@ -20,15 +20,17 @@ url="https://github.com/Open-EO/openeo-aggregator", packages=find_packages(where="src", include=["openeo_aggregator", "openeo_aggregator.*"]), package_dir={"": "src"}, + package_data={"openeo_aggregator": ["config/examples/*.py"]}, include_package_data=True, data_files=[ ( "openeo-aggregator-data", [ "CHANGELOG.md", + # TODO #117 eliminate gunicorn.dev.py reference "conf/gunicorn.dev.py", + # TODO #117 eliminate gunicorn.prod.py reference "conf/gunicorn.prod.py", - "conf/aggregator.dummy.py", ], ), ], diff --git a/src/openeo_aggregator/about.py b/src/openeo_aggregator/about.py index fa9a1e7..d7a9f53 100644 --- a/src/openeo_aggregator/about.py +++ b/src/openeo_aggregator/about.py @@ -2,7 +2,7 @@ import sys from typing import Optional -__version__ = "0.36.0a1" +__version__ = "0.37.0a1" def log_version_info(logger: Optional[logging.Logger] = None): diff --git a/conf/aggregator.dummy.py b/src/openeo_aggregator/config/examples/aggregator.dummy.py similarity index 100% rename from conf/aggregator.dummy.py rename to src/openeo_aggregator/config/examples/aggregator.dummy.py diff --git a/src/openeo_aggregator/config/examples/gunicorn-config.py b/src/openeo_aggregator/config/examples/gunicorn-config.py new file mode 100644 index 0000000..34d6d90 --- /dev/null +++ b/src/openeo_aggregator/config/examples/gunicorn-config.py @@ -0,0 +1,9 @@ +# See https://docs.gunicorn.org/en/stable/settings.html + +bind = ["0.0.0.0:8080"] + +workers = 1 +threads = 1 + +# Worker timeout +timeout = 60