Merge pull request #45 from alan-turing-institute/update-pydantic

Update dependencies

alembic/env.py

@@ -10,7 +10,7 @@
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-config.set_main_option("sqlalchemy.url", settings.postgres_dsn)
+config.set_main_option("sqlalchemy.url", str(settings.postgres_dsn))
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "RCTab"
-version = "1.3.1"
+version = "1.4.0"
 description = "The RCTab API. Manage Azure budgets and usage"
 authors = []
 
@@ -13,14 +13,16 @@ azure-identity = "^1.16.0"
 celery = "^5.3.5"
 databases = { version = "^0.7.0", extras = ["postgresql"] }
 fastapi = "^0.111.0"
-fastapimsal = { git = "https://github.com/alan-turing-institute/fastapimsal", tag = "0.5.0" }
+fastapimsal = { git = "https://github.com/alan-turing-institute/fastapimsal", tag = "0.6.0" }
 Jinja2 = "^3.1.3"
-opencensus-ext-azure = "^1.1.7"
+myst-parser = {version = "^3.0.1", optional = true}
 numpy = "<2.0.0"
+opencensus-ext-azure = "^1.1.7"
 pandas = "^1.1.3"
 plotly = "^5.14.1"
 psycopg2-binary = "^2.9.3"
-pydantic = { extras = ["email"], version = "^1.10.15" }
+pydantic = { extras = ["email"], version = "^2.7.1" }
+pydantic-settings = "^2.3.4"
 PyJWT = { extras = ["crypto"], version = "^2.4.0" }
 python-dotenv = "^1.0.1"
 redis = {extras = ["hiredis"], version = "^5.0.1"}
@@ -30,9 +32,8 @@ sqlalchemy = "~1.4.52"
 sphinx = {version = "^7.2.6", optional = true}
 sphinx-rtd-theme = {version = "^1.3.0", optional = true}
 sphinxcontrib-napoleon = {version = "^0.7", optional = true}
-myst-parser = {version = "^3.0.1", optional = true}
 secure = "^0.3.0"
-uvicorn = { version = "^0.17.6", extras = ["standard"] }
+uvicorn = { version = "^0.30.1", extras = ["standard"] }
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.3.0"

rctab/crud/schema.py

@@ -24,7 +24,7 @@ class UserRBAC(BaseModel):
     """The access that a user has to RCTab."""
 
     oid: UUID
-    username: Optional[str]
+    username: Optional[str] = None
     has_access: bool
     is_admin: bool
 
@@ -34,52 +34,52 @@ class Usage(HashBaseModel):
 
     # See https://docs.microsoft.com/en-us/rest/api/consumption/usage-details/list#legacyusagedetail
     id: str
-    name: Optional[str]
-    type: Optional[str]
-    tags: Optional[Dict[str, str]]
-    billing_account_id: Optional[str]
-    billing_account_name: Optional[str]
-    billing_period_start_date: Optional[datetime.date]
-    billing_period_end_date: Optional[datetime.date]
-    billing_profile_id: Optional[str]
-    billing_profile_name: Optional[str]
-    account_owner_id: Optional[str]
-    account_name: Optional[str]
+    name: Optional[str] = None
+    type: Optional[str] = None
+    tags: Optional[Dict[str, str]] = None
+    billing_account_id: Optional[str] = None
+    billing_account_name: Optional[str] = None
+    billing_period_start_date: Optional[datetime.date] = None
+    billing_period_end_date: Optional[datetime.date] = None
+    billing_profile_id: Optional[str] = None
+    billing_profile_name: Optional[str] = None
+    account_owner_id: Optional[str] = None
+    account_name: Optional[str] = None
     subscription_id: UUID
-    subscription_name: Optional[str]
+    subscription_name: Optional[str] = None
     date: datetime.date
-    product: Optional[str]
-    part_number: Optional[str]
-    meter_id: Optional[str]
-    quantity: Optional[float]
-    effective_price: Optional[float]
-    cost: Optional[float]
-    amortised_cost: Optional[float]
+    product: Optional[str] = None
+    part_number: Optional[str] = None
+    meter_id: Optional[str] = None
+    quantity: Optional[float] = None
+    effective_price: Optional[float] = None
+    cost: Optional[float] = None
+    amortised_cost: Optional[float] = None
     total_cost: float
-    unit_price: Optional[float]
-    billing_currency: Optional[str]
-    resource_location: Optional[str]
-    consumed_service: Optional[str]
-    resource_id: Optional[str]
-    resource_name: Optional[str]
-    service_info1: Optional[str]
-    service_info2: Optional[str]
-    additional_info: Optional[str]
-    invoice_section: Optional[str]
-    cost_center: Optional[str]
-    resource_group: Optional[str]
-    reservation_id: Optional[str]
-    reservation_name: Optional[str]
-    product_order_id: Optional[str]
-    offer_id: Optional[str]
-    is_azure_credit_eligible: Optional[bool]
-    term: Optional[str]
-    publisher_name: Optional[str]
-    publisher_type: Optional[str]
-    plan_name: Optional[str]
-    charge_type: Optional[str]
-    frequency: Optional[str]
-    monthly_upload: Optional[datetime.date]
+    unit_price: Optional[float] = None
+    billing_currency: Optional[str] = None
+    resource_location: Optional[str] = None
+    consumed_service: Optional[str] = None
+    resource_id: Optional[str] = None
+    resource_name: Optional[str] = None
+    service_info1: Optional[str] = None
+    service_info2: Optional[str] = None
+    additional_info: Optional[str] = None
+    invoice_section: Optional[str] = None
+    cost_center: Optional[str] = None
+    resource_group: Optional[str] = None
+    reservation_id: Optional[str] = None
+    reservation_name: Optional[str] = None
+    product_order_id: Optional[str] = None
+    offer_id: Optional[str] = None
+    is_azure_credit_eligible: Optional[bool] = None
+    term: Optional[str] = None
+    publisher_name: Optional[str] = None
+    publisher_type: Optional[str] = None
+    plan_name: Optional[str] = None
+    charge_type: Optional[str] = None
+    frequency: Optional[str] = None
+    monthly_upload: Optional[datetime.date] = None
 
 
 class AllUsage(BaseModel):
@@ -92,7 +92,7 @@ class CMUsage(HashBaseModel):
     """A usage object from the Azure cost management API."""
 
     subscription_id: UUID
-    name: Optional[str]
+    name: Optional[str] = None
     start_datetime: datetime.date
     end_datetime: datetime.date
     cost: confloat(ge=0.0)  # type: ignore
@@ -112,8 +112,8 @@ class RoleAssignment(HashBaseModel):
     role_name: str
     principal_id: str
     display_name: str
-    mail: Optional[str]
-    scope: Optional[str]
+    mail: Optional[str] = None
+    scope: Optional[str] = None
 
 
 class SubscriptionState(str, Enum):

rctab/routers/accounting/allocations.py

@@ -108,6 +108,7 @@ async def post_subscription_allocation(
 @router.get("/allocations", response_model=List[AllocationListItem])
 async def get_subscription_allocations(
     subscription: SubscriptionItem, _: UserRBAC = Depends(token_admin_verified)
-) -> Any:
+) -> List[AllocationListItem]:
     """Return a list of allocations for a subscription."""
-    return await get_allocations(subscription.sub_id)
+    rows = await get_allocations(subscription.sub_id)
+    return [AllocationListItem(**dict(x)) for x in rows]

rctab/routers/accounting/approvals.py

@@ -2,7 +2,7 @@
 
 import datetime
 from datetime import timedelta
-from typing import Any, List, Optional
+from typing import Any, List
 
 from fastapi import Depends, HTTPException
 from sqlalchemy import insert
@@ -143,9 +143,10 @@ async def check_approval(approval: Approval) -> None:
 @router.get("/approvals", response_model=List[ApprovalListItem])
 async def get_subscription_approvals(
     subscription: SubscriptionItem, _: UserRBAC = Depends(token_admin_verified)
-) -> Optional[List[ApprovalListItem]]:
+) -> List[ApprovalListItem]:
     """Returns a list approvals for a subscription."""
-    return await get_approvals(subscription.sub_id)
+    rows = await get_approvals(subscription.sub_id)
+    return [ApprovalListItem(**dict(x)) for x in rows]
 
 
 @router.post("/approve", status_code=200)

rctab/routers/accounting/cost_recovery.py

@@ -214,21 +214,22 @@ async def calc_cost_recovery_app(
 @router.post("/cli-cost-recovery", response_model=List[CostRecovery])
 async def post_calc_cost_recovery_cli(
     recovery_month: CostRecoveryMonth, user: UserRBAC = Depends(token_admin_verified)
-) -> Any:
+) -> List[CostRecovery]:
     """Route for the CLI to trigger cost recovery calculation."""
-    resp = await calc_cost_recovery(
+    rows = await calc_cost_recovery(
         recovery_month, commit_transaction=True, admin=user.oid
     )
 
-    return resp
+    return [CostRecovery(**dict(x)) for x in rows]
 
 
 @router.get("/cli-cost-recovery", response_model=List[CostRecovery])
 async def get_calc_cost_recovery_cli(
     recovery_month: CostRecoveryMonth, user: UserRBAC = Depends(token_admin_verified)
-) -> Any:
+) -> List[CostRecovery]:
     """Route for the CLI to do a dry-run of the cost recovery calculation."""
-    result = await calc_cost_recovery(
+    rows = await calc_cost_recovery(
         recovery_month, commit_transaction=False, admin=user.oid
     )
-    return result
+
+    return [CostRecovery(**dict(x)) for x in rows]

rctab/routers/accounting/subscription.py

@@ -34,10 +34,12 @@ async def create_subscription(subscription: SubscriptionItem, user: UserRBAC) ->
 async def get_subscription(
     sub_id: Optional[UUID] = None,
     _: UserRBAC = Depends(token_admin_verified),
-) -> SubscriptionDetails:
+) -> List[SubscriptionDetails]:
     """Whether a subscription with the specified uuid is registered."""
-    retval = await get_subscriptions_with_disable(sub_id)
-    return retval
+    rows = [dict(x) for x in await get_subscriptions_with_disable(sub_id)]
+    result = [SubscriptionDetails(**x) for x in rows]
+
+    return result
 
 
 @router.post("/subscription")

rctab/routers/accounting/usage.py

@@ -3,7 +3,7 @@
 import calendar
 import datetime
 import logging
-from typing import Any, Dict, List
+from typing import Dict, List
 from uuid import UUID
 
 import jwt
@@ -239,11 +239,13 @@ async def post_usage(
 
 
 @router.get("/all-usage", response_model=List[Usage])
-async def get_usage(_: UserRBAC = Depends(token_admin_verified)) -> Any:
+async def get_usage(_: UserRBAC = Depends(token_admin_verified)) -> List[Usage]:
     """Get all usage data."""
     usage_query = select([accounting_models.usage])
+    rows = [dict(x) for x in await database.fetch_all(usage_query)]
+    result = [Usage(**x) for x in rows]
 
-    return await database.fetch_all(usage_query)
+    return result
 
 
 @router.post("/all-cm-usage", response_model=TmpReturnStatus)
@@ -278,7 +280,9 @@ async def post_cm_usage(
 
 
 @router.get("/all-cm-usage", response_model=List[CMUsage])
-async def get_cm_usage(_: UserRBAC = Depends(token_admin_verified)) -> Any:
+async def get_cm_usage(_: UserRBAC = Depends(token_admin_verified)) -> List[CMUsage]:
     """Get all cost-management data."""
     cm_query = select([accounting_models.costmanagement])
-    return await database.fetch_all(cm_query)
+    rows = [dict(x) for x in await database.fetch_all(cm_query)]
+    result = [CMUsage(**x) for x in rows]
+    return result