feat (icd): added scraper for icd

care/facility/api/viewsets/icd.py

@@ -0,0 +1,24 @@
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+from rest_framework.viewsets import ViewSet
+
+
+def serailize_data(icd11_object):
+    result = []
+    for object in icd11_object:
+        if type(object) == tuple:
+            object = object[0]
+        result.append({"id": object.ID, "label": object.label})
+    return result
+
+
+class ICDViewSet(ViewSet):
+    permission_classes = (IsAuthenticated,)
+
+    def list(self, request):
+        from care.facility.static_data.icd11 import ICDDiseases
+
+        queryset = ICDDiseases
+        if request.GET.get("query", False):
+            queryset = queryset.search.label(request.GET["query"], limit=100)
+        return Response(serailize_data(queryset[0:100]))

care/facility/management/commands/scrape_icd_data.py

@@ -0,0 +1,15 @@
+from django.core.management.base import BaseCommand
+
+from care.facility.tasks.icd.scraper import ICDScraper
+
+
+class Command(BaseCommand):
+    """
+    Management command to scrape ICD11 Disease Data
+    """
+
+    help = "Dump ICD11 Data to Json"
+
+    def handle(self, *args, **options):
+        scraper = ICDScraper()
+        scraper.scrape()

care/facility/static_data/icd11.py

@@ -0,0 +1,41 @@
+import json
+
+from littletable import Table
+
+
+def fetch_data():
+    with open("data/icd11.json", "r") as json_file:
+        return json.load(json_file)
+
+
+def parse_int(str_val):
+    try:
+        return int(str_val)
+    except BaseException:
+        return -1
+
+
+ICDDiseases = Table("ICD11")
+icd11_objects = fetch_data()
+entity_id = ""
+IGNORE_FIELDS = [
+    "isLeaf",
+    "classKind",
+    "isAdoptedChild",
+    "averageDepth",
+    "breadthValue",
+    "Suggested",
+]
+
+for icd11_object in icd11_objects:
+    for field in IGNORE_FIELDS:
+        icd11_object.pop(field, "")
+    entity_id = icd11_object["ID"].split("/")[-1]
+    icd11_object["ID"] = parse_int(entity_id)
+    if icd11_object["ID"] == -1:
+        continue
+    if icd11_object["ID"]:
+        ICDDiseases.insert(icd11_object)
+
+ICDDiseases.create_search_index("label")
+ICDDiseases.create_index("ID", unique=True)

care/facility/tasks/icd/scraper.py

@@ -0,0 +1,70 @@
+import json
+import time
+
+import requests
+from django.conf import settings
+
+
+class ICDScraper:
+    def __init__(self):
+        self.root_concept_url = settings.ICD_SCRAPER_ROOT_CONCEPTS_URL
+        self.child_concept_url = settings.ICD_SCRAPER_CHILD_CONCEPTS_URL
+        self.scraped_concepts = []
+        self.scraped_concept_dict = {}
+
+    def add_query(self, url, query={}):
+        return url + "?" + "&".join(map(lambda k: str(k) + "=" + str(query[k]), query.keys()))
+
+    def get_child_concepts(self, p_concept, p_parent_id):
+        if p_concept['ID'] in self.scraped_concept_dict:
+            print(f"[-] Skipped duplicate, {p_concept['label']}")
+            return
+
+        self.scraped_concepts.append({**p_concept, 'parentId': p_parent_id})
+        self.scraped_concept_dict[p_concept['ID']] = True
+
+        print(f"[+] Added {p_concept['label']}")
+
+        if p_concept['isLeaf']:
+            return
+
+        concepts = []
+        try:
+            concepts = requests.get(self.add_query(self.child_concept_url, {
+                'useHtml': 'false',
+                'ConceptId': p_concept['ID'],
+            })).json()
+        except Exception as e:
+            print("[x] Error encountered: ", e)
+            with open('error.txt', 'a') as error_file:
+                error_file.write(f"{p_concept['label']}\n")
+
+            time.sleep(10)
+            concepts = requests.get(self.add_query(self.child_concept_url, {
+                'useHtml': 'false',
+                'ConceptId': p_concept['ID'],
+            })).json()
+
+        for concept in concepts:
+            self.get_child_concepts(concept, p_concept['ID'])
+
+    def scrape(self):
+        self.scraped_concepts = []
+        self.scraped_concept_dict = {}
+        root_concepts = requests.get(self.add_query(
+            self.root_concept_url, {'useHtml': 'false'})).json()
+
+        skip = [
+            "V Supplementary section for functioning assessment",
+            "X Extension Codes"
+        ]
+
+        for root_concept in root_concepts:
+            if root_concept['label'] in skip:
+                continue
+
+            self.get_child_concepts(root_concept, None)
+            time.sleep(3)
+
+        with open('data.json', 'w') as json_file:
+            json.dump(self.scraped_concepts, json_file)

config/api_router.py

@@ -1,17 +1,29 @@
-from care.facility.api.viewsets.facility_users import FacilityUserViewSet
 from django.conf import settings
 from django.conf.urls import include, url
 from rest_framework.routers import DefaultRouter, SimpleRouter
 from rest_framework_nested.routers import NestedSimpleRouter
 
-from care.facility.api.viewsets.ambulance import AmbulanceCreateViewSet, AmbulanceViewSet
-from care.facility.api.viewsets.asset import AssetLocationViewSet, AssetTransactionViewSet, AssetViewSet
-from care.facility.api.viewsets.bed import AssetBedViewSet, BedViewSet, ConsultationBedViewSet
+from care.facility.api.viewsets.ambulance import (
+    AmbulanceCreateViewSet,
+    AmbulanceViewSet,
+)
+from care.facility.api.viewsets.asset import (
+    AssetLocationViewSet,
+    AssetTransactionViewSet,
+    AssetViewSet,
+)
+from care.facility.api.viewsets.bed import (
+    AssetBedViewSet,
+    BedViewSet,
+    ConsultationBedViewSet,
+)
 from care.facility.api.viewsets.daily_round import DailyRoundsViewSet
 from care.facility.api.viewsets.facility import AllFacilityViewSet, FacilityViewSet
 from care.facility.api.viewsets.facility_capacity import FacilityCapacityViewSet
+from care.facility.api.viewsets.facility_users import FacilityUserViewSet
 from care.facility.api.viewsets.file_upload import FileUploadViewSet
 from care.facility.api.viewsets.hospital_doctor import HospitalDoctorViewSet
+from care.facility.api.viewsets.icd import ICDViewSet
 from care.facility.api.viewsets.inventory import (
     FacilityInventoryItemViewSet,
     FacilityInventoryLogViewSet,
@@ -41,14 +53,27 @@
     PrescriptionSupplierConsultationViewSet,
     PrescriptionSupplierViewSet,
 )
-from care.facility.api.viewsets.resources import ResourceRequestCommentViewSet, ResourceRequestViewSet
-from care.facility.api.viewsets.shifting import ShifitngRequestCommentViewSet, ShiftingViewSet
-from care.facility.summarisation.district.patient_summary import DistrictPatientSummaryViewSet
+from care.facility.api.viewsets.resources import (
+    ResourceRequestCommentViewSet,
+    ResourceRequestViewSet,
+)
+from care.facility.api.viewsets.shifting import (
+    ShifitngRequestCommentViewSet,
+    ShiftingViewSet,
+)
+from care.facility.summarisation.district.patient_summary import (
+    DistrictPatientSummaryViewSet,
+)
 from care.facility.summarisation.facility_capacity import FacilityCapacitySummaryViewSet
 from care.facility.summarisation.patient_summary import PatientSummaryViewSet
 from care.facility.summarisation.tests_summary import TestsSummaryViewSet
 from care.facility.summarisation.triage_summary import TriageSummaryViewSet
-from care.users.api.viewsets.lsg import DistrictViewSet, LocalBodyViewSet, StateViewSet, WardViewSet
+from care.users.api.viewsets.lsg import (
+    DistrictViewSet,
+    LocalBodyViewSet,
+    StateViewSet,
+    WardViewSet,
+)
 from care.users.api.viewsets.skill import SkillViewSet
 from care.users.api.viewsets.users import UserViewSet
 from care.users.api.viewsets.userskill import UserSkillViewSet
@@ -72,6 +97,8 @@
 router.register("ambulance/create", AmbulanceCreateViewSet)
 router.register("ambulance", AmbulanceViewSet)
 
+router.register("icd", ICDViewSet, basename="icd")
+
 router.register("patient/search", PatientSearchViewSet)
 
 router.register("otp/token", PatientMobileOTPViewSet)
@@ -89,7 +116,6 @@
 router.register("assetbed", AssetBedViewSet)
 router.register("consultationbed", ConsultationBedViewSet)
 
-
 router.register("pharmacy/consultation", PrescriptionSupplierConsultationViewSet)
 router.register("pharmacy/prescription", PrescriptionSupplierViewSet)
 
@@ -106,7 +132,8 @@
 router.register("patient_search", PatientScopedSearchViewSet)
 
 # Summarisation
-router.register("facility_summary", FacilityCapacitySummaryViewSet, basename="summary-facility")
+router.register("facility_summary", FacilityCapacitySummaryViewSet,
+                basename="summary-facility")
 router.register("patient_summary", PatientSummaryViewSet, basename="summary-patient")
 router.register("tests_summary", TestsSummaryViewSet, basename="summary-tests")
 router.register("triage_summary", TriageSummaryViewSet, basename="summary-triage")
@@ -119,7 +146,6 @@
     basename="district-summary-patient",
 )
 
-
 router.register("items", FacilityInventoryItemViewSet)
 # router.register("burn_rate", FacilityInventoryBurnRateViewSet)
 
@@ -151,13 +177,13 @@
 router.register("asset", AssetViewSet)
 router.register("asset_transaction", AssetTransactionViewSet)
 
-
 patient_nested_router = NestedSimpleRouter(router, r"patient", lookup="patient")
 patient_nested_router.register(r"test_sample", PatientSampleViewSet)
 patient_nested_router.register(r"investigation", PatientInvestigationSummaryViewSet)
 patient_nested_router.register(r"notes", PatientNotesViewSet)
 
-consultation_nested_router = NestedSimpleRouter(router, r"consultation", lookup="consultation")
+consultation_nested_router = NestedSimpleRouter(
+    router, r"consultation", lookup="consultation")
 consultation_nested_router.register(r"daily_rounds", DailyRoundsViewSet)
 consultation_nested_router.register(r"investigation", InvestigationValueViewSet)
 
@@ -172,7 +198,4 @@
     url(r"^", include(shifting_nested_router.urls)),
 ]
 
-
-## Importing Celery Tasks
-
-import care.facility.reports.admin_reports
+# Importing Celery Tasks

config/settings/base.py

@@ -6,6 +6,9 @@
 from datetime import timedelta
 
 import environ
+from healthy_django.healthcheck.celery_queue_length import (
+    DjangoCeleryQueueLengthHealthCheck,
+)
 from healthy_django.healthcheck.django_cache import DjangoCacheHealthCheck
 from healthy_django.healthcheck.django_database import DjangoDatabaseHealthCheck
 
@@ -420,6 +423,10 @@ def GETKEY(group, request):
 
 OTP_LENGTH = 5
 
+# ICD
+ICD_SCRAPER_ROOT_CONCEPTS_URL = "https://icd.who.int/browse11/l-m/en/JsonGetRootConcepts"
+ICD_SCRAPER_CHILD_CONCEPTS_URL = "https://icd.who.int/browse11/l-m/en/JsonGetChildrenConcepts"
+
 # SMS
 USE_SMS = False