Enhance custom industry

Snakefile

@@ -168,7 +168,7 @@ rule prepare_sector_network:
         shapes_path=pypsaearth(
             "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
         ),
-        pipelines="resources/custom_data/pipelines.csv"
+        pipelines="data_custom/pipelines.csv"
         if config["custom_data"]["gas_network"]
         else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv",
     output:
@@ -615,103 +615,59 @@ rule clean:
         shell("rm -r " + PYPSAEARTH_FOLDER + "/networks")
 
 
-if config["custom_data"].get("industry_demand", False) == True:
-
-    rule build_industrial_distribution_key:  #custom data
-        input:
-            regions_onshore=pypsaearth(
-                "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
-            ),
-            clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}.csv",
-            clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}.csv",
-            industrial_database="resources/custom_data/industrial_database.csv",
-            shapes_path=pypsaearth(
-                "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
-            ),
-            #shapes_path=PYPSAEARTH_FOLDER + "/resources/shapes/MAR2.geojson",
-        output:
-            industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
-        threads: 1
-        resources:
-            mem_mb=1000,
-        benchmark:
-            "benchmarks/build_industrial_distribution_key/s{simpl}_{clusters}"
-        script:
-            "scripts/build_industrial_distribution_key.py"
-
-    rule build_industry_demand:  #custom data
-        input:
-            industry_sector_ratios="resources/custom_data/industry_sector_ratios_{demand}_{planning_horizons}.csv",
-            industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
-            industrial_production_per_country_tomorrow="resources/custom_data/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv",
-            costs=CDIR
-            + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-            industry_growth_cagr="data/demand/industry_growth_cagr.csv",
-        output:
-            industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv",
-        threads: 1
-        resources:
-            mem_mb=1000,
-        benchmark:
-            "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv"
-        script:
-            "scripts/build_industry_demand.py"
-
+rule build_industrial_distribution_key:  #default data
+    input:
+        regions_onshore=pypsaearth(
+            "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
+        ),
+        clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}.csv",
+        clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}.csv",
+        industrial_database="data/industrial_database.csv",
+        shapes_path=pypsaearth(
+            "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
+        ),
+    output:
+        industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
+    threads: 1
+    resources:
+        mem_mb=1000,
+    benchmark:
+        "benchmarks/build_industrial_distribution_key_elec_s{simpl}_{clusters}"
+    script:
+        "scripts/build_industrial_distribution_key.py"
 
-if config["custom_data"].get("industry_demand", False) == False:
 
-    rule build_industrial_distribution_key:  #default data
-        input:
-            regions_onshore=pypsaearth(
-                "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
-            ),
-            clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}.csv",
-            clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}.csv",
-            industrial_database="data/industrial_database.csv",
-            shapes_path=pypsaearth(
-                "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson"
-            ),
-        output:
-            industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
-        threads: 1
-        resources:
-            mem_mb=1000,
-        benchmark:
-            "benchmarks/build_industrial_distribution_key_elec_s{simpl}_{clusters}"
-        script:
-            "scripts/build_industrial_distribution_key.py"
+rule build_base_industry_totals:  #default data
+    input:
+        #industrial_production_per_country="data/industrial_production_per_country.csv",
+        #unsd_path="data/demand/unsd/data/",
+        energy_totals_base="data/energy_totals_base.csv",
+    output:
+        base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv",
+    threads: 1
+    resources:
+        mem_mb=1000,
+    benchmark:
+        "benchmarks/build_base_industry_totals_{planning_horizons}_{demand}"
+    script:
+        "scripts/build_base_industry_totals.py"
 
-    rule build_base_industry_totals:  #default data
-        input:
-            #industrial_production_per_country="data/industrial_production_per_country.csv",
-            #unsd_path="data/demand/unsd/data/",
-            energy_totals_base="data/energy_totals_base.csv",
-        output:
-            base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv",
-        threads: 1
-        resources:
-            mem_mb=1000,
-        benchmark:
-            "benchmarks/build_base_industry_totals_{planning_horizons}_{demand}"
-        script:
-            "scripts/build_base_industry_totals.py"
 
-    rule build_industry_demand:  #default data
-        input:
-            industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
-            #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv",
-            #industrial_production_per_country="data/industrial_production_per_country.csv",
-            base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv",
-            industrial_database="data/industrial_database.csv",
-            costs=CDIR
-            + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
-            industry_growth_cagr="data/demand/industry_growth_cagr.csv",
-        output:
-            industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv",
-        threads: 1
-        resources:
-            mem_mb=1000,
-        benchmark:
-            "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv"
-        script:
-            "scripts/build_industry_demand.py"
+rule build_industry_demand:  #default data
+    input:
+        industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv",
+        #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv",
+        #industrial_production_per_country="data/industrial_production_per_country.csv",
+        base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv",
+        industrial_database="data/industrial_database.csv",
+        costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
+        industry_growth_cagr="data/demand/industry_growth_cagr.csv",
+    output:
+        industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv",
+    threads: 1
+    resources:
+        mem_mb=1000,
+    benchmark:
+        "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv"
+    script:
+        "scripts/build_industry_demand.py"

config.default.yaml

@@ -5,7 +5,7 @@ results_dir: results/
 summary_dir: results/
 costs_dir: data/ #TODO change to the equivalent of technology data
 
-run: test_h2_cons_yearly
+run: test_run
 
 foresight: overnight
 
@@ -55,7 +55,7 @@ fossil_reserves:
 
 
 export:
-  h2export: [120] # Yearly export demand in TWh
+  h2export: [10] # Yearly export demand in TWh
   store: true # [True, False] # specifies wether an export store to balance demand is implemented
   store_capital_costs: "no_costs" # ["standard_costs", "no_costs"] # specifies the costs of the export store. "standard_costs" takes CAPEX of "hydrogen storage tank type 1 including compressor"
   export_profile: "ship" # use "ship" or "constant"
@@ -70,6 +70,7 @@ custom_data:
   elec_demand: false
   heat_demand: false
   industry_demand: false
+  industry_database: false
   transport_demand: false
   water_costs: false
   h2_underground: false
@@ -171,6 +172,8 @@ sector:
     gas_network_repurposing: true # If true -> ["sector"]["gas"]["network"] is automatically false
     underground_storage: false
 
+  international_bunkers: false #Whether or not to count the emissions of international aviation and navigation
+
   oil:
     spatial_oil: true
 
@@ -313,13 +316,14 @@ sector:
     NZ_2050: 0.36
     DF_2050: 0.12
 
-  gadm_level: 2
+  gadm_level: 1
   h2_cavern: true
   marginal_cost_storage: 0
   methanation: true
   helmeth: true
   dac: true
   SMR: true
+  SMR CC: true
   cc_fraction: 0.9
   cc: true
   space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating.
@@ -556,3 +560,4 @@ plotting:
     H2 for industry: "#222222"
     H2 for shipping: "#6495ED"
     biomass EOP: "green"
+    biomass: "green"

data/AL_production.csv

@@ -241,7 +241,6 @@ TC,0,2019,no available information online assumed value of 0
 TV,0,2019,no available information online assumed value of 0
 UG,0,2019,no available information online assumed value of 0
 UA,0,2019,no available information online assumed value of 0
-AE,0,2019,no available information online assumed value of 0
 GB,0,2019,no available information online assumed value of 0
 US,0,2019,no available information online assumed value of 0
 UM,0,2019,no available information online assumed value of 0

data_custom/TEMPLATE_industrial_database.csv

@@ -0,0 +1,2 @@
+country,y,x,technology,capacity,unit,quality,location
+MA,32.992424,-7.622267,Industry NMM Cement,790, kt/yr,actual,Settat

data_custom/TEMPLATE_industry_demand_AB_2030.csv

@@ -0,0 +1,9 @@
+country,carrier,Industry Steel Primary Blast Furnace Open Hearth Furnace,Industry Steel Primary Blast Furnace Basic Oxygen Furnace,Industry Steel Primary DRI,Industry Steel Secondary EAF,Industry Steel Other,Industry Chemical Ammonia SMR,Industry Chemical Ammonia Other conventional,Industry Chemical Ammonia Renewable,Industry Chemical HVC Naphtha,Industry Chemical HVC LPG,Industry Chemical HVC Methanol,Industry Chemical Other,Industry NMM Cement,Industry NMM Other,Industry Food and tobacco,Industry Construction,Industry Mining,Industry Machinery,Industry Non ferrous metals Aluminium Primary,Industry Non ferrous metals Aluminium Secondary,Industry Non ferrous metals Other,Industry Paper and pulp Pulp Primary,Industry Paper and pulp Pulp Secondary,Industry Paper and pulp Paper,Industry Paper and pulp Other,Industry Transport equipment,Industry Textiles and leather,Industry Wood,Industry Miscellaneous
+MA,oil,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,10726146.18,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,397010.79,0.0,0.0,0.0,0.0,13258151.395154
+MA,gas,0.0,0.0,3601545.78,236922.08000000002,0.0,1788930.0,0.0,0.0,0.0,0.0,0.0,0.0,9281486.520000001,0.0,0.0,0.0,0.0,0.0,26202513.16,0.0,0.0,2514199.8,0.0,41960.49,0.0,0.0,0.0,0.0,207158615.483001
+MA,electricity,0.0,0.0,1031598.5399999999,962495.9500000001,0.0,93825.0,0.0,132050.0,1884697.5200000003,0.0,55432.72,0.0,3265708.22,0.0,0.0,0.0,0.0,0.0,11232826.1,0.0,0.0,838066.6,0.0,187208.34,0.0,0.0,0.0,0.0,55242297.464439005
+MA,coal,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,6875175.2,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+MA,heat,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+MA,biomass,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,171879.38,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,276211.484037
+MA,hydrogen,0.0,0.0,1158286.08,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,276211.484037
+MA,process emission,0.0,0.0,50065.41247790604,40962.59763502796,0.0,0.0,0.0,0.0,67875.68113871076,0.0,1385.228977747275,8.72479576360429e-05,9323308.507746331,0.03203319690797162,0.0,0.0,0.0,0.0,1658616.7007389446,0.0,0.18963180469598598,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

scripts/build_industrial_distribution_key.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """Build industrial distribution keys from hotmaps database."""
 
+import logging
 import os
 import uuid
 from distutils.version import StrictVersion
@@ -11,6 +12,7 @@
 from helpers import locate_bus, three_2_two_digits_country
 from shapely.geometry import Point
 
+logger = logging.getLogger(__name__)
 gpd_version = StrictVersion(gpd.__version__)
 
 
@@ -69,7 +71,7 @@ def build_nodal_distribution_key(
         regions_ct = regions.name[regions.name.str.contains(country)]
 
         facilities = industrial_database.query(
-            "country == @country and technology == @tech"
+            "country == @country and industry == @tech"
         )
         # TODO adapt for facilities with production values not emissions
         if not facilities.empty:
@@ -99,6 +101,7 @@ def match_technology(df):
         "Cement": "non-metallic minerals",
         "HVC": "chemical and petrochemical",
         "Paper": "paper pulp and print",
+        "Aluminium": "non-ferrous metals",
     }
 
     df["industry"] = df["technology"].map(industry_mapping)
@@ -114,8 +117,8 @@ def match_technology(df):
         snakemake = mock_snakemake(
             "build_industrial_distribution_key",
             simpl="",
-            clusters=38,
-            demand="EG",
+            clusters=12,
+            demand="AB",
             planning_horizons=2050,
         )
         sets_path_to_root("pypsa-earth-sec")
@@ -136,16 +139,30 @@ def match_technology(df):
             lambda name: three_2_two_digits_country(name[:3]) + name[3:]
         )
 
-    geo_locs = pd.read_csv(
-        snakemake.input.industrial_database,
-        sep=",",
-        header=0,
-        keep_default_na=False,  # , index_col=0
-    )
-    geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity)
+    if snakemake.config["custom_data"]["industry_database"]:
+        logger.info(
+            "Using custom industry database from 'data_custom/industrial_database.csv' instead of default"
+        )
+        geo_locs = pd.read_csv(
+            "data_custom/industrial_database.csv",
+            sep=",",
+            header=0,
+            keep_default_na=False,  # , index_col=0
+        )
+        geo_locs["industry"] = geo_locs["technology"]
+    else:
+        logger.info("Using default industry database")
+        geo_locs = pd.read_csv(
+            snakemake.input.industrial_database,
+            sep=",",
+            header=0,
+            keep_default_na=False,  # , index_col=0
+        )
+        geo_locs = geo_locs[geo_locs["country"].isin(countries)]
+        geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity)
 
-    # Call the function to add the "industry" column
-    df_with_industry = match_technology(geo_locs)
+        # Call the function to add the "industry" column
+        df_with_industry = match_technology(geo_locs)
 
     geo_locs.capacity = pd.to_numeric(geo_locs.capacity)