From 570b63ad8dc72b0afaf3da6914b2480313368827 Mon Sep 17 00:00:00 2001 From: Sidney Richards Date: Thu, 19 Sep 2024 14:35:35 +0200 Subject: [PATCH] [#2764] Resolve cases in case list concurrently Resolving cases involves several sequential network requests, which creates a lot of latency when loading a lost of cases. This commit runs the resolving concurrently, and also centralizes the logic in the CaseListService. --- src/open_inwoner/cms/cases/views/cases.py | 161 +++++++++++++++++++++- src/open_inwoner/openzaak/cases.py | 127 ----------------- 2 files changed, 157 insertions(+), 131 deletions(-) delete mode 100644 src/open_inwoner/openzaak/cases.py diff --git a/src/open_inwoner/cms/cases/views/cases.py b/src/open_inwoner/cms/cases/views/cases.py index 4dce8399ac..6678a5853c 100644 --- a/src/open_inwoner/cms/cases/views/cases.py +++ b/src/open_inwoner/cms/cases/views/cases.py @@ -1,7 +1,10 @@ import concurrent.futures +import functools import logging from dataclasses import dataclass +from typing import Callable +from django.conf import settings from django.http import HttpRequest from django.urls import reverse from django.utils.functional import cached_property @@ -14,11 +17,16 @@ from open_inwoner.htmx.mixins import RequiresHtmxMixin from open_inwoner.openzaak.api_models import Zaak -from open_inwoner.openzaak.cases import preprocess_data +from open_inwoner.openzaak.clients import CatalogiClient, ZakenClient from open_inwoner.openzaak.formapi import fetch_open_submissions -from open_inwoner.openzaak.models import OpenZaakConfig, ZGWApiGroupConfig +from open_inwoner.openzaak.models import ( + OpenZaakConfig, + ZaakTypeConfig, + ZaakTypeStatusTypeConfig, + ZGWApiGroupConfig, +) from open_inwoner.openzaak.types import UniformCase -from open_inwoner.openzaak.utils import get_user_fetch_parameters +from open_inwoner.openzaak.utils import get_user_fetch_parameters, is_zaak_visible from open_inwoner.utils.mixins import PaginationMixin from open_inwoner.utils.views import CommonPageMixin @@ -52,7 +60,7 @@ def get_cases_for_api_group(self, group: ZGWApiGroupConfig): raw_cases = group.zaken_client.fetch_cases( **get_user_fetch_parameters(self.request) ) - preprocessed_cases = preprocess_data(raw_cases, group) + preprocessed_cases = self.resolve_cases(raw_cases, group) return preprocessed_cases def get_cases(self) -> list[ZaakWithApiGroup]: @@ -97,6 +105,151 @@ def get_case_status_frequencies(self): return {status: case_statuses.count(status) for status in case_statuses} + @staticmethod + def _resolve_zaak_type_and_configs( + case: Zaak, *, client: CatalogiClient + ) -> Callable[[Zaak], None] | None: + """ + Resolve `case.zaaktype` (`str`) to a `ZaakType(ZGWModel)` object + + Note: the result of `fetch_single_case_type` is cached, hence a request + is only made for new case type urls + """ + if not isinstance(case.zaaktype, str): + return + + case_type = client.fetch_single_case_type(case.zaaktype) + if not case_type: + logger.error("Unable to resolve zaaktype for url: %s", case.zaaktype) + return + + def setter(case_): + case_.zaaktype = case_type + + return setter + + @staticmethod + def _resolve_status_and_status_type( + case: Zaak, *, zaken_client: ZakenClient, catalogi_client: CatalogiClient + ) -> Callable[[Zaak], None] | None: + if not isinstance(case.status, str): + return + + status = zaken_client.fetch_single_status(case.status) + if not status: + logger.error("") + return None + + status_type = None + status_type = catalogi_client.fetch_single_status_type(status.statustype) + if not status_type: + logger.error("") + return None + + def setter(case_): + case_.status = status + case_.status.statustype = status_type + + return setter + + @staticmethod + def _resolve_resultaat_and_resultaat_type( + case: Zaak, *, zaken_client: ZakenClient, catalogi_client: CatalogiClient + ) -> Callable[[Zaak], None] | None: + if not isinstance(case.resultaat, str): + logger.debug("`case.resultaat` is not a str but %s", type(case.resultaat)) + return + + resultaat = zaken_client.fetch_single_result(case.resultaat) + if not resultaat: + logger.error("Unable to fetch resultaat for %s", case) + return + + resultaattype = catalogi_client.fetch_single_resultaat_type( + resultaat.resultaattype + ) + if not resultaattype: + logger.error( + "Unable to resolve resultaattype for %s", resultaat.resultaattype + ) + return + + def setter(case_: Zaak): + case_.resultaat = resultaat + case_.resultaat.resultaattype = resultaattype + + return setter + + def resolve_cases(self, cases: list[Zaak], group: ZGWApiGroupConfig) -> list[Zaak]: + with parallel(max_workers=settings.CASE_LIST_NUM_THREADS) as executor: + futures = [ + executor.submit(self.resolve_case, case, group) for case in cases + ] + concurrent.futures.wait(futures) + + cases = [case for case in cases if case.status and is_zaak_visible(case)] + cases.sort(key=lambda case: case.startdatum, reverse=True) + + return cases + + def resolve_case( + self, + case: Zaak, + group: ZGWApiGroupConfig, + ): + logger.debug("Resolving case %s with group %s", case, group) + + functions = [ + functools.partial( + CaseListService._resolve_resultaat_and_resultaat_type, + zaken_client=group.zaken_client, + catalogi_client=group.catalogi_client, + ), + functools.partial( + CaseListService._resolve_status_and_status_type, + zaken_client=group.zaken_client, + catalogi_client=group.catalogi_client, + ), + functools.partial( + CaseListService._resolve_zaak_type_and_configs, + client=group.catalogi_client, + ), + ] + + # use contextmanager to ensure the `requests.Session` is reused + with group.catalogi_client, group.zaken_client: + with parallel() as _executor: + futures = [_executor.submit(func, case) for func in functions] + + for task in concurrent.futures.as_completed(futures): + if exc := task.exception(): + logger.error( + "Error in resolving case: %s", exc, stack_info=True + ) + + update_case = task.result() + if hasattr(update_case, "__call__"): + update_case(case) + + try: + zaaktype_config = ZaakTypeConfig.objects.filter_case_type( + case.zaaktype + ).get() + case.zaaktype_config = zaaktype_config + + if zaaktype_config: + statustype_config = ZaakTypeStatusTypeConfig.objects.get( + zaaktype_config=zaaktype_config, + statustype_url=case.status.statustype.url, + ) + case.statustype_config = statustype_config + except ( + ZaakTypeConfig.DoesNotExist, + AttributeError, + ZaakTypeStatusTypeConfig.DoesNotExist, + ): + logger.exception("Unable to resolve zaaktype_config and statustype_config") + class OuterCaseListView( OuterCaseAccessMixin, CommonPageMixin, BaseBreadcrumbMixin, TemplateView diff --git a/src/open_inwoner/openzaak/cases.py b/src/open_inwoner/openzaak/cases.py deleted file mode 100644 index 25045f708b..0000000000 --- a/src/open_inwoner/openzaak/cases.py +++ /dev/null @@ -1,127 +0,0 @@ -import concurrent.futures -import logging - -from django.conf import settings - -from zgw_consumers.concurrent import parallel - -from .api_models import Zaak -from .clients import CatalogiClient, ZakenClient -from .models import ZaakTypeConfig, ZaakTypeStatusTypeConfig, ZGWApiGroupConfig -from .utils import is_zaak_visible - -logger = logging.getLogger(__name__) - - -def resolve_zaak_type(case: Zaak, client: CatalogiClient) -> None: - """ - Resolve `case.zaaktype` (`str`) to a `ZaakType(ZGWModel)` object - - Note: the result of `fetch_single_case_type` is cached, hence a request - is only made for new case type urls - """ - case_type_url = case.zaaktype - if client: - case_type = client.fetch_single_case_type(case_type_url) - case.zaaktype = case_type - - -def resolve_status(case: Zaak, client: ZakenClient | None = None) -> None: - """ - Resolve `case.status` (`str`) to a `Status(ZGWModel)` object - """ - if client: - case.status = client.fetch_single_status(case.status) - - -def resolve_status_type(case: Zaak, client: CatalogiClient | None = None) -> None: - """ - Resolve `case.status.statustype` (`str`) to a `StatusType(ZGWModel)` object - """ - statustype_url = case.status.statustype - if client: - case.status.statustype = client.fetch_single_status_type(statustype_url) - - -def resolve_resultaat(case: Zaak, client: ZakenClient | None = None) -> None: - """ - Resolve `case.resultaat` (`str`) to a `Resultaat(ZGWModel)` object - """ - if case.resultaat: - case.resultaat = client.fetch_single_result(case.resultaat) - - -def resolve_resultaat_type(case: Zaak, client: CatalogiClient | None = None) -> None: - """ - Resolve `case.resultaat.resultaattype` (`str`) to a `ResultaatType(ZGWModel)` object - """ - if client and case.resultaat: - case.resultaat.resultaattype = client.fetch_single_resultaat_type( - case.resultaat.resultaattype - ) - - -def add_zaak_type_config(case: Zaak) -> None: - """ - Add `ZaakTypeConfig` corresponding to the zaaktype type url of the case - - Note: must be called after `resolve_zaak_type` since we're using the `uuid` and - `identificatie` from `case.zaaktype` - """ - try: - case.zaaktype_config = ZaakTypeConfig.objects.filter_case_type( - case.zaaktype - ).get() - except ZaakTypeConfig.DoesNotExist: - pass - - -def add_status_type_config(case: Zaak) -> None: - """ - Add `ZaakTypeStatusTypeConfig` corresponding to the status type url of the case - - Note: must be called after `resolve_status_type` since we're getting the - status type url from `case.status.statustype` - """ - try: - case.statustype_config = ZaakTypeStatusTypeConfig.objects.get( - zaaktype_config=case.zaaktype_config, - statustype_url=case.status.statustype.url, - ) - except (AttributeError, ZaakTypeStatusTypeConfig.DoesNotExist): - pass - - -def preprocess_data(cases: list[Zaak], group: ZGWApiGroupConfig) -> list[Zaak]: - """ - Resolve zaaktype and statustype, add status type config, filter for visibility - - Note: we need to iterate twice over `cases` because the `zaak_type` must be - resolved to a `ZaakType` object before we can filter by visibility - """ - - def preprocess_case(case: Zaak) -> None: - resolve_status(case, client=group.zaken_client) - resolve_status_type(case, client=group.catalogi_client) - resolve_resultaat(case, client=group.zaken_client) - resolve_resultaat_type(case, client=group.catalogi_client) - add_zaak_type_config(case) - add_status_type_config(case) - - # use contextmanager to ensure the `requests.Session` is reused - with group.catalogi_client, group.zaken_client: - with parallel(max_workers=settings.CASE_LIST_NUM_THREADS) as executor: - futures = [ - executor.submit(resolve_zaak_type, case, client=group.catalogi_client) - for case in cases - ] - concurrent.futures.wait(futures) - - cases = [case for case in cases if case.status and is_zaak_visible(case)] - - futures = [executor.submit(preprocess_case, case) for case in cases] - concurrent.futures.wait(futures) - - cases.sort(key=lambda case: case.startdatum, reverse=True) - - return cases