support upload of 10 test cases

d-fine · Nov 20, 2024 · ba29a90 · ba29a90
1 parent eaaa88f
commit ba29a90
Show file tree

Hide file tree

Showing 20 changed files with 4,017 additions and 3,997 deletions.
diff --git a/data/jsons/concordia.json b/data/jsons/concordia.json
diff --git a/data/jsons/covestro.json b/data/jsons/covestro.json
diff --git a/data/jsons/deka.json b/data/jsons/deka.json
diff --git a/data/jsons/enbw.json b/data/jsons/enbw.json
diff --git a/data/jsons/enel.json b/data/jsons/enel.json
diff --git a/data/jsons/eon.json b/data/jsons/eon.json
diff --git a/data/jsons/iberdrola.json b/data/jsons/iberdrola.json
diff --git a/data/jsons/munichre.json b/data/jsons/munichre.json
diff --git a/data/jsons/rwe.json b/data/jsons/rwe.json
diff --git a/data/jsons/total.json b/data/jsons/total.json
diff --git a/...dfs/2023_DNK_Concordia_Versicherungen.pdf → data/pdfs/concordia.pdf b/...dfs/2023_DNK_Concordia_Versicherungen.pdf → data/pdfs/concordia.pdf
diff --git a/data/pdfs/covestro-ar23-entire.pdf → data/pdfs/covestro.pdf b/data/pdfs/covestro-ar23-entire.pdf → data/pdfs/covestro.pdf
diff --git a/...pdfs/Deka_Nachhaltigkeitsbericht_2023.pdf → data/pdfs/deka.pdf b/...pdfs/Deka_Nachhaltigkeitsbericht_2023.pdf → data/pdfs/deka.pdf
diff --git a/...s/integrierter-geschaeftsbericht-2023.pdf → data/pdfs/enbw.pdf b/...s/integrierter-geschaeftsbericht-2023.pdf → data/pdfs/enbw.pdf
diff --git a/data/pdfs/enel_report.pdf → data/pdfs/enel.pdf b/data/pdfs/enel_report.pdf → data/pdfs/enel.pdf
diff --git a/...pdfs/gsm24-sustainability-report-2023.pdf → data/pdfs/iberdrola.pdf b/...pdfs/gsm24-sustainability-report-2023.pdf → data/pdfs/iberdrola.pdf
diff --git a/...chRe-Konzerngeschaeftsbericht-2023-de.pdf → data/pdfs/munichre.pdf b/...chRe-Konzerngeschaeftsbericht-2023-de.pdf → data/pdfs/munichre.pdf
diff --git a/...dfs/2024-03-14-rwe-annual-report-2023.pdf → data/pdfs/rwe.pdf b/...dfs/2024-03-14-rwe-annual-report-2023.pdf → data/pdfs/rwe.pdf
diff --git a/...egistration-document-2023_2023_en_pdf.pdf → data/pdfs/total.pdf b/...egistration-document-2023_2023_en_pdf.pdf → data/pdfs/total.pdf
diff --git a/notebooks/provide_test_data.ipynb b/notebooks/provide_test_data.ipynb
@@ -2,23 +2,18 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "document_id='4abdfd0764559831fdd2e972abab0f34bc7300c650f6f789beea10ecb7d20251'\n",
-      "[CompanyIdAndName(company_name='E.ON SE', company_id='8863987f-df05-452f-8067-11f72e771345')]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%run base.ipynb\n",
     "import json\n",
     "from pathlib import Path\n",
     "\n",
+    "from dataland_backend.models.company_associated_data_nuclear_and_gas_data import (  # type: ignore\n",
+    "    CompanyAssociatedDataNuclearAndGasData,\n",
+    ")\n",
+    "\n",
     "from dataland_qa_lab.utils import config\n",
     "\n",
     "pdf_path = Path(\"../data/pdfs/\")\n",
@@ -27,35 +22,60 @@
     "conf = config.get_config()\n",
     "dataland_client = conf.dataland_client\n",
     "\n",
-    "dataset = dataland_client.company_api.get_companies_by_search_string(search_string=\"E.ON SE\", result_limit=1)\n",
-    "\n",
-    "# upload pdf file to dataland\n",
-    "if not dataland_client.documents_api.get_document(\n",
-    "    document_id=\"4abdfd0764559831fdd2e972abab0f34bc7300c650f6f789beea10ecb7d20251\"):\n",
+    "# list of companies to test\n",
+    "companies = [\"concordia\", \"covestro\", \"deka\", \"enbw\", \"enel\", \"eon\", \"iberdrola\", \"munichre\", \"rwe\", \"total\"]\n",
+    "# list of ids of corresponding pdfs\n",
+    "pdfs = [\n",
+    "    \"0a8eebb9e32d3c0a32a1083699352018afcbbe39458ab8441cd0c8985a466a59\",\n",
+    "    \"ebff9ec3cf12e715cb6ee1c55a1295656a87e1716a9b536b4fbf2a1b9312260c\",\n",
+    "    \"b31dfa1143e9e518cfdacd95b2d4f6c531e50bc33c0dabbbe35cccfe14dd83f3\",\n",
+    "    \"9c0a555a29683aedd2cd50ff7e837181a7fbb2d1c567d336897e2356fc17a595\",\n",
+    "    \"a58354fd0d2969d7c3161d6ba273c9ba4814866c0fc8ec0e220dc4ee6e87753c\",\n",
+    "    \"4abdfd0764559831fdd2e972abab0f34bc7300c650f6f789beea10ecb7d20251\",\n",
+    "    \"3305bd49f340b73919de891d166f7492cd61f59a9efdc1b84a0720db1f846fc2\",\n",
+    "    \"e974e3f3675386f17b67af4a5b03ee5a0a313c4d0b07d719c2cf5cb715ccbeb3\",\n",
+    "    \"eb119227edc8c66d672785619522cd6045b2faf37e63796207799c0e40fa66be\",\n",
+    "    \"dba48e9f5e7e6fc9862dd95159960eb2a270d6975f2457f443ca422e7449e7d6\",\n",
+    "]\n",
+    "for company, pdf_id in zip(companies, pdfs, strict=False):\n",
     "\n",
-    "    pdf_file_path = pdf_path / \"eon.pdf\"\n",
-    "    pdf_content = pdf_file_path.read_bytes()\n",
+    "    # if needed upload pdf file to dataland\n",
+    "    if not dataland_client.documents_api.get_document(document_id=pdf_id):\n",
+    "        pdf_file_path = pdf_path / f\"{company}.pdf\"\n",
+    "        pdf_content = pdf_file_path.read_bytes()\n",
     "\n",
-    "    print(dataland_client.documents_api.post_document(document=pdf_content))\n",
+    "        print(dataland_client.documents_api.post_document(document=pdf_content))\n",
     "\n",
-    "# change companyID in json file\n",
-    "company_id = dataset[0].company_id\n",
+    "    # get companyIDs of company to test\n",
+    "    if company == \"eon\":\n",
+    "        dataset = dataland_client.company_api.get_companies_by_search_string(search_string=\"E.ON SE\", result_limit=1)\n",
+    "    elif company == \"munichre\":\n",
+    "        dataset = dataland_client.company_api.get_companies_by_search_string(\n",
+    "            search_string=\"Münchener Rückversicherungs-Gesellschaft Aktiengesellschaft in München\", result_limit=1\n",
+    "        )\n",
+    "    else:\n",
+    "        dataset = dataland_client.company_api.get_companies_by_search_string(search_string=company, result_limit=1)\n",
     "\n",
-    "json_file_path = json_path / \"eon.json\"\n",
-    "with json_file_path.open(encoding=\"utf-8\") as f:\n",
-    "    json_data = json.load(f)\n",
+    "    company_id = dataset[0].company_id\n",
     "\n",
-    "json_data[\"companyID\"] = company_id\n",
+    "    # change companyID in json file\n",
+    "    json_file_path = json_path / f\"{company}.json\"\n",
     "\n",
-    "with json_file_path.open(\"w\", encoding=\"utf-8\") as f:\n",
-    "    json.dump(json_data, f, indent=4)\n",
-    "\n",
-    "# upload data to dataland\n",
-    "response = dataland_client.nuclear_and_gas_api.post_data(\n",
-    "    data=json_data\n",
-    ")\n",
+    "    with json_file_path.open(encoding=\"utf-8\") as f:\n",
+    "        json_data = json.load(f)\n",
+    "    json_data[\"companyId\"] = company_id\n",
+    "    json_str = json.dumps(json_data, indent=4)\n",
+    "    json_file_path.write_text(json_str, encoding=\"utf-8\")\n",
+    "    written_data = json_file_path.read_text(encoding=\"utf-8\")\n",
     "\n",
-    "print(dataset)"
+    "    # if needed upload document\n",
+    "    if not dataland_client.eu_taxonomy_nuclear_and_gas_api.get_all_company_nuclear_and_gas_data(company_id=company_id):\n",
+    "        nuclear_and_gas_data = CompanyAssociatedDataNuclearAndGasData.from_json(json_str)\n",
+    "        print(\n",
+    "            dataland_client.eu_taxonomy_nuclear_and_gas_api.post_company_associated_nuclear_and_gas_data(\n",
+    "                company_associated_data_nuclear_and_gas_data=nuclear_and_gas_data\n",
+    "            )\n",
+    "        )"
    ]
   },
   {