Expand the usage of TimesModel class (#165)

Co-authored-by: Siddharth Krishna <siddharth-krishna@users.noreply.github.com>

benchmarks.yml

@@ -248,8 +248,8 @@ benchmarks:
       - "SubRES_TMPL/SubRES_SRV_NewTechs_Trans.xlsx"
       - "SubRES_TMPL/SubRES_SUP_BioRefineries.xlsx"
       - "SubRES_TMPL/SubRES_SUP_BioRefineries_Trans.xlsx"
-      - "SubRES_TMPL/SubRES_SUP_Hydrogen.xlsx"
-      - "SubRES_TMPL/SubRES_SUP_Hydrogen_Trans.xlsx"
+      - "SubRES_TMPL/SubRES_SUP_H2NewTechs.xlsx"
+      - "SubRES_TMPL/SubRES_SUP_H2NewTechs_Trans.xlsx"
       - "SubRES_TMPL/SubRES_SYS_OtherNewTechs.xlsx"
       - "SubRES_TMPL/SubRES_SYS_OtherNewTechs_Trans.xlsx"
       - "SubRES_TMPL/SubRES_TRA_NewVehicles.xlsx"

utils/dd_to_csv.py

@@ -1,6 +1,7 @@
 import argparse
-import os
 from collections import defaultdict
+import json
+import os
 from pathlib import Path
 from typing import Dict, List, Tuple, Union
 
@@ -127,7 +128,7 @@ def save_data_with_headers(
 ) -> None:
     """
 
-    :param param_data: Dictionary containing key=param_name and val=dataframe for parameters or List[str] for sets
+    :param param_data_dict: Dictionary containing key=param_name and val=dataframe for parameters or List[str] for sets
     :param headers_data: Dictionary containing key=param_name and val=dataframes
     :param save_dir: Path to folder in which to save the tabular data files
     :return: None
@@ -154,7 +155,24 @@ def save_data_with_headers(
     return
 
 
-def convert_dd_to_tabular(basedir: str, output_dir: str) -> None:
+def generate_headers_by_attr() -> Dict[str, List[str]]:
+    with open("xl2times/config/times-info.json") as f:
+        attributes = json.load(f)
+
+    headers_by_attr = {}
+
+    for attr in attributes:
+        if attr["gams-cat"] == "parameter":
+            headers_by_attr[attr["name"]] = [*attr["indexes"], "VALUE"]
+        else:
+            headers_by_attr[attr["name"]] = attr["indexes"]
+
+    return headers_by_attr
+
+
+def convert_dd_to_tabular(
+    basedir: str, output_dir: str, headers_by_attr: Dict[str, List[str]]
+) -> None:
     dd_files = [p for p in Path(basedir).rglob("*.dd")]
 
     all_sets = defaultdict(list)
@@ -182,7 +200,9 @@ def convert_dd_to_tabular(basedir: str, output_dir: str) -> None:
 
     # Extract headers with key=param_name and value=List[attributes]
     lines = list(open("xl2times/config/times_mapping.txt", "r"))
-    headers_data = dict()
+    headers_data = headers_by_attr
+    # The following will overwrite data obtained from headers_by_attr
+    # TODO: Remove once migration is done?
     for line in lines:
         line = line.strip()
         if line != "":
@@ -205,4 +225,4 @@ def convert_dd_to_tabular(basedir: str, output_dir: str) -> None:
         "output_dir", type=str, help="Output directory to save the .csv files in."
     )
     args = args_parser.parse_args()
-    convert_dd_to_tabular(args.input_dir, args.output_dir)
+    convert_dd_to_tabular(args.input_dir, args.output_dir, generate_headers_by_attr())

xl2times/__main__.py

@@ -68,6 +68,7 @@ def convert_xl_to_times(
         ],
         transforms.process_regions,
         transforms.generate_dummy_processes,
+        transforms.process_time_slices,
         transforms.process_transform_insert_variants,
         transforms.process_transform_insert,
         transforms.process_processes,
@@ -77,7 +78,6 @@ def convert_xl_to_times(
         transforms.process_commodity_emissions,
         transforms.process_commodities,
         transforms.process_transform_availability,
-        transforms.process_time_slices,
         transforms.fill_in_missing_values,
         transforms.expand_rows_parallel,  # slow
         transforms.remove_invalid_values,
@@ -98,6 +98,7 @@ def convert_xl_to_times(
         transforms.convert_aliases,
         transforms.rename_cgs,
         transforms.fix_topology,
+        transforms.complete_dictionary,
         transforms.convert_to_string,
         lambda config, tables, model: dump_tables(
             tables, os.path.join(output_dir, "merged_tables.txt")

xl2times/config/times-info.json

@@ -279,20 +279,20 @@
     "name": "ALL_TS",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "ALL_TS"
     ],
     "mapping": [
-      "tbd"
+      "timeslice"
     ]
   },
   {
     "name": "ALL_REG",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "ALL_REG"
     ],
     "mapping": [
-      "tbd"
+      "region"
     ]
   },
   {
@@ -1040,10 +1040,10 @@
     "name": "COM_GRP",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "COM_GRP"
     ],
     "mapping": [
-      "tbd"
+      "commoditygroup"
     ]
   },
   {
@@ -1256,7 +1256,7 @@
     ],
     "mapping": [
       "region",
-      "tbd",
+      "csets",
       "commodity"
     ]
   },
@@ -1285,7 +1285,7 @@
     "mapping": [
       "region",
       "commodity",
-      "tbd"
+      "unit"
     ]
   },
   {
@@ -1320,10 +1320,10 @@
     "name": "CUR",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "CUR"
     ],
     "mapping": [
-      "tbd"
+      "currency"
     ]
   },
   {
@@ -2993,7 +2993,7 @@
     ],
     "mapping": [
       "region",
-      "tbd",
+      "ctype",
       "commodity"
     ]
   },
@@ -3035,10 +3035,10 @@
     "name": "PRC",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "PRC"
     ],
     "mapping": [
-      "tbd"
+      "process"
     ]
   },
   {
@@ -3069,8 +3069,8 @@
     "mapping": [
       "region",
       "process",
-      "other_indexes",
-      "tbd"
+      "primarycg",
+      "tact"
     ]
   },
   {
@@ -3113,8 +3113,8 @@
     "mapping": [
       "region",
       "process",
-      "other_indexes",
-      "tbd"
+      "primarycg",
+      "tcap"
     ]
   },
   {
@@ -3217,7 +3217,7 @@
     ],
     "mapping": [
       "region",
-      "tbd",
+      "sets",
       "process"
     ]
   },
@@ -3935,10 +3935,10 @@
     "name": "SPE",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "SPE"
     ],
     "mapping": [
-      "tbd"
+      "spe"
     ]
   },
   {
@@ -4209,10 +4209,10 @@
     "name": "TM_DEFVAL",
     "gams-cat": "parameter",
     "indexes": [
-      "*"
+      "TM_DEFVAL"
     ],
     "mapping": [
-      "tbd"
+      "tm_defval"
     ]
   },
   {
@@ -4785,10 +4785,10 @@
     "name": "UC_N",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "UC_N"
     ],
     "mapping": [
-      "tbd"
+      "uc_n"
     ]
   },
   {
@@ -5045,10 +5045,10 @@
     "name": "UNITS",
     "gams-cat": "set",
     "indexes": [
-      "*"
+      "UNITS"
     ],
     "mapping": [
-      "tbd"
+      "units"
     ]
   },
   {
@@ -5058,7 +5058,7 @@
       "UNITS"
     ],
     "mapping": [
-      "tbd"
+      "units"
     ]
   },
   {
@@ -5068,7 +5068,7 @@
       "UNITS"
     ],
     "mapping": [
-      "tbd"
+      "units"
     ]
   },
   {
@@ -5078,7 +5078,7 @@
       "UNITS"
     ],
     "mapping": [
-      "tbd"
+      "units"
     ]
   },
   {