From 4d2705674e697adce46ab8bec4f3b65ae9b3fee6 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Fri, 15 Mar 2024 22:49:08 +0100 Subject: [PATCH 01/23] adapt build_industrial_distribution_key to better accomodate custom data --- scripts/build_industrial_distribution_key.py | 33 +++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index dd058188..f5d7a8a3 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Build industrial distribution keys from hotmaps database.""" +import logging import os import uuid from distutils.version import StrictVersion @@ -11,6 +12,7 @@ from helpers import locate_bus, three_2_two_digits_country from shapely.geometry import Point +logger = logging.getLogger(__name__) gpd_version = StrictVersion(gpd.__version__) @@ -69,7 +71,7 @@ def build_nodal_distribution_key( regions_ct = regions.name[regions.name.str.contains(country)] facilities = industrial_database.query( - "country == @country and technology == @tech" + "country == @country and industry == @tech" ) # TODO adapt for facilities with production values not emissions if not facilities.empty: @@ -99,6 +101,7 @@ def match_technology(df): "Cement": "non-metallic minerals", "HVC": "chemical and petrochemical", "Paper": "paper pulp and print", + "Aluminium": "non-ferrous metals", } df["industry"] = df["technology"].map(industry_mapping) @@ -114,8 +117,8 @@ def match_technology(df): snakemake = mock_snakemake( "build_industrial_distribution_key", simpl="", - clusters=38, - demand="EG", + clusters=12, + demand="AB", planning_horizons=2050, ) sets_path_to_root("pypsa-earth-sec") @@ -136,16 +139,30 @@ def match_technology(df): lambda name: three_2_two_digits_country(name[:3]) + name[3:] ) - geo_locs = pd.read_csv( + + if snakemake.config["custom_data"]["industry_database"]: + logger.info("Using custom industry database from 'data_custom/industrial_database.csv' instead of default") + geo_locs = pd.read_csv( + "data_custom/industrial_database.csv", + sep=",", + header=0, + keep_default_na=False, # , index_col=0 + ) + geo_locs["industry"] = geo_locs["technology"] + else: + logger.info("Using default industry database") + geo_locs = pd.read_csv( snakemake.input.industrial_database, sep=",", header=0, keep_default_na=False, # , index_col=0 - ) - geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity) + ) + geo_locs = geo_locs[geo_locs['country'].isin(countries)] + geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity) + + # Call the function to add the "industry" column + df_with_industry = match_technology(geo_locs) - # Call the function to add the "industry" column - df_with_industry = match_technology(geo_locs) geo_locs.capacity = pd.to_numeric(geo_locs.capacity) From 289462f8a3637714197d25923f1c700c50ce88f7 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Fri, 15 Mar 2024 22:49:46 +0100 Subject: [PATCH 02/23] adapt build_indusry_demand to better accomodate custom data --- scripts/build_industry_demand.py | 395 +++++++++++++++++-------------- 1 file changed, 214 insertions(+), 181 deletions(-) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index bd35e3c5..ce2f534c 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -40,7 +40,7 @@ def country_to_nodal(industrial_production, keys): mapping = sector key = keys.loc[buses, mapping] - print(sector) + #print(sector) nodal_production.loc[buses, sector] = ( industrial_production.at[country, sector] * key ) @@ -57,210 +57,243 @@ def country_to_nodal(industrial_production, keys): snakemake = mock_snakemake( "build_industry_demand", simpl="", - clusters=4, + clusters=12, planning_horizons=2030, demand="AB", ) sets_path_to_root("pypsa-earth-sec") - no_years = int(snakemake.wildcards.planning_horizons) - int( - snakemake.config["demand_data"]["base_year"] - ) + countries = snakemake.config["countries"] - cagr = read_csv_nafix(snakemake.input.industry_growth_cagr, index_col=0) - countries = snakemake.config["countries"] - # countries = ["EG", "BH"] + if snakemake.config["custom_data"]["industry_demand"]: - for country in countries: - if country not in cagr.index: - cagr.loc[country] = cagr.loc["DEFAULT"] - _logger.warning( - "No industry growth data for " - + country - + " using default data instead." - ) + _logger.info("Fetching custom industry demand data.. expecting file at 'data_custom/industry_demand_{0}_{1}.csv'".format(snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"])) - cagr = cagr[cagr.index.isin(countries)] + industry_demand = pd.read_csv("data_custom/industry_demand_{0}_{1}.csv".format(snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"]), index_col=[0,1]) + keys_path = snakemake.input.industrial_distribution_key - growth_factors = calculate_end_values(cagr) + dist_keys = pd.read_csv( + keys_path, index_col=0, keep_default_na=False, na_values=[""] + ) + production_base = pd.DataFrame(1, columns=industry_demand.columns, index=countries) + nodal_keys = country_to_nodal(production_base, dist_keys) + + nodal_df = pd.DataFrame() + + for country in countries: + nodal_production_tom_co = nodal_keys[ + nodal_keys.index.to_series().str.startswith(country) + ] + industry_base_totals_co = industry_demand.loc[country] + # final energy consumption per node and industry (TWh/a) + nodal_df_co = nodal_production_tom_co.dot(industry_base_totals_co.T) + nodal_df = pd.concat([nodal_df, nodal_df_co]) + + else: + no_years = int(snakemake.wildcards.planning_horizons) - int( + snakemake.config["demand_data"]["base_year"] + ) - industry_base_totals = read_csv_nafix( - snakemake.input["base_industry_totals"], index_col=[0, 1] - ) + cagr = read_csv_nafix(snakemake.input.industry_growth_cagr, index_col=0) - production_base = cagr.applymap(lambda x: 1) - production_tom = production_base * growth_factors - industry_totals = (production_tom * industry_base_totals).fillna(0) - industry_util_factor = snakemake.config["sector"]["industry_util_factor"] + # Building nodal industry production growth - # Load distribution keys - keys_path = snakemake.input.industrial_distribution_key + for country in countries: + if country not in cagr.index: + cagr.loc[country] = cagr.loc["DEFAULT"] + _logger.warning( + "No industry growth data for " + + country + + " using default data instead." + ) - dist_keys = pd.read_csv( - keys_path, index_col=0, keep_default_na=False, na_values=[""] - ) + cagr = cagr[cagr.index.isin(countries)] - # material demand per node and industry (kton/a) - nodal_production_tom = country_to_nodal(production_tom, dist_keys) - - clean_industry_list = [ - "iron and steel", - "chemical and petrochemical", - "non-ferrous metals", - "non-metallic minerals", - "transport equipment", - "machinery", - "mining and quarrying", - "food and tobacco", - "paper pulp and print", - "wood and wood products", - "textile and leather", - "construction", - "other", - ] - - emission_factors = { # Based on JR data following PyPSA-EUR - "iron and steel": 0.025, - "chemical and petrochemical": 0.51, # taken from HVC including process and feedstock - "non-ferrous metals": 1.5, # taken from Aluminum primary - "non-metallic minerals": 0.542, # taken for cement - "transport equipment": 0, - "machinery": 0, - "mining and quarrying": 0, # assumed - "food and tobacco": 0, - "paper pulp and print": 0, - "wood and wood products": 0, - "textile and leather": 0, - "construction": 0, # assumed - "other": 0, - } - - geo_locs = pd.read_csv( - snakemake.input.industrial_database, - sep=",", - header=0, - keep_default_na=False, - index_col=0, - ) - geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity) - - def match_technology(df): - industry_mapping = { - "Integrated steelworks": "iron and steel", - "DRI + Electric arc": "iron and steel", - "Electric arc": "iron and steel", - "Cement": "non-metallic minerals", - "HVC": "chemical and petrochemical", - "Paper": "paper pulp and print", - } + growth_factors = calculate_end_values(cagr) - df["industry"] = df["technology"].map(industry_mapping) - return df + industry_base_totals = read_csv_nafix( + snakemake.input["base_industry_totals"], index_col=[0, 1] + ) - geo_locs = match_technology(geo_locs).loc[countries] + production_base = cagr.applymap(lambda x: 1) + production_tom = production_base * growth_factors - AL = read_csv_nafix("data/AL_production.csv", index_col=0) - AL_prod_tom = AL["production[ktons/a]"].loc[countries] - AL_emissions = AL_prod_tom * emission_factors["non-ferrous metals"] + industry_totals = (production_tom * industry_base_totals).fillna(0) - Steel_emissions = ( - geo_locs[geo_locs.industry == "iron and steel"] - .groupby("country") - .sum() - .capacity - * 1000 - * emission_factors["iron and steel"] - * industry_util_factor - ) - NMM_emissions = ( - geo_locs[geo_locs.industry == "non-metallic minerals"] - .groupby("country") - .sum() - .capacity - * 1000 - * emission_factors["non-metallic minerals"] - * industry_util_factor - ) - refinery_emissons = ( - geo_locs[geo_locs.industry == "chemical and petrochemical"] - .groupby("country") - .sum() - .capacity - * emission_factors["chemical and petrochemical"] - * 0.136 - * 365 - * industry_util_factor - ) + industry_util_factor = snakemake.config["sector"]["industry_util_factor"] + + # Load distribution keys + keys_path = snakemake.input.industrial_distribution_key - for country in countries: - industry_base_totals.loc[(country, "process emissions"), :] = 0 - try: - industry_base_totals.loc[ - (country, "process emissions"), "non-metallic minerals" - ] = NMM_emissions.loc[country] - except KeyError: - pass - - try: - industry_base_totals.loc[ - (country, "process emissions"), "iron and steel" - ] = Steel_emissions.loc[country] - except KeyError: - pass # # Code to handle the KeyError - try: - industry_base_totals.loc[ - (country, "process emissions"), "non-ferrous metals" - ] = AL_emissions.loc[country] - except KeyError: - pass # Code to handle the KeyError - try: - industry_base_totals.loc[ - (country, "process emissions"), "chemical and petrochemical" - ] = refinery_emissons.loc[country] - except KeyError: - pass # Code to handle the KeyError - industry_base_totals = industry_base_totals.sort_index() - - all_carriers = [ - "electricity", - "gas", - "coal", - "oil", - "hydrogen", - "biomass", - "low-temperature heat", - ] - - for country in countries: - carriers_present = industry_base_totals.xs(country, level="country").index - missing_carriers = set(all_carriers) - set(carriers_present) - for carrier in missing_carriers: - # Add the missing carrier with a value of 0 - industry_base_totals.loc[(country, carrier), :] = 0 - - nodal_df = pd.DataFrame() - - for country in countries: - nodal_production_tom_co = nodal_production_tom[ - nodal_production_tom.index.to_series().str.startswith(country) + dist_keys = pd.read_csv( + keys_path, index_col=0, keep_default_na=False, na_values=[""] + ) + + # production of industries per node compared to current + nodal_production_tom = country_to_nodal(production_tom, dist_keys) + + + clean_industry_list = [ + "iron and steel", + "chemical and petrochemical", + "non-ferrous metals", + "non-metallic minerals", + "transport equipment", + "machinery", + "mining and quarrying", + "food and tobacco", + "paper pulp and print", + "wood and wood products", + "textile and leather", + "construction", + "other", + ] + + emission_factors = { # Based on JR data following PyPSA-EUR + "iron and steel": 0.025, + "chemical and petrochemical": 0.51, # taken from HVC including process and feedstock + "non-ferrous metals": 1.5, # taken from Aluminum primary + "non-metallic minerals": 0.542, # taken for cement + "transport equipment": 0, + "machinery": 0, + "mining and quarrying": 0, # assumed + "food and tobacco": 0, + "paper pulp and print": 0, + "wood and wood products": 0, + "textile and leather": 0, + "construction": 0, # assumed + "other": 0, + } + + geo_locs = pd.read_csv( + snakemake.input.industrial_database, + sep=",", + header=0, + keep_default_na=False, + index_col=0, + ) + geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity) + + def match_technology(df): + industry_mapping = { + "Integrated steelworks": "iron and steel", + "DRI + Electric arc": "iron and steel", + "Electric arc": "iron and steel", + "Cement": "non-metallic minerals", + "HVC": "chemical and petrochemical", + "Paper": "paper pulp and print", + } + + df["industry"] = df["technology"].map(industry_mapping) + return df + + # Calculating emissions + + geo_locs = match_technology(geo_locs).loc[countries] + + AL = read_csv_nafix("data/AL_production.csv", index_col=0) + AL_prod_tom = AL["production[ktons/a]"].loc[countries] + AL_emissions = AL_prod_tom * emission_factors["non-ferrous metals"] + + Steel_emissions = ( + geo_locs[geo_locs.industry == "iron and steel"] + .groupby("country") + .sum() + .capacity + * 1000 + * emission_factors["iron and steel"] + * industry_util_factor + ) + NMM_emissions = ( + geo_locs[geo_locs.industry == "non-metallic minerals"] + .groupby("country") + .sum() + .capacity + * 1000 + * emission_factors["non-metallic minerals"] + * industry_util_factor + ) + refinery_emissons = ( + geo_locs[geo_locs.industry == "chemical and petrochemical"] + .groupby("country") + .sum() + .capacity + * emission_factors["chemical and petrochemical"] + * 0.136 + * 365 + * industry_util_factor + ) + + for country in countries: + industry_base_totals.loc[(country, "process emissions"), :] = 0 + try: + industry_base_totals.loc[ + (country, "process emissions"), "non-metallic minerals" + ] = NMM_emissions.loc[country] + except KeyError: + pass + + try: + industry_base_totals.loc[ + (country, "process emissions"), "iron and steel" + ] = Steel_emissions.loc[country] + except KeyError: + pass + try: + industry_base_totals.loc[ + (country, "process emissions"), "non-ferrous metals" + ] = AL_emissions.loc[country] + except KeyError: + pass + try: + industry_base_totals.loc[ + (country, "process emissions"), "chemical and petrochemical" + ] = refinery_emissons.loc[country] + except KeyError: + pass + industry_base_totals = industry_base_totals.sort_index() + + all_carriers = [ + "electricity", + "gas", + "coal", + "oil", + "hydrogen", + "biomass", + "low-temperature heat", ] - industry_base_totals_co = industry_base_totals.loc[country] - # final energy consumption per node and industry (TWh/a) - nodal_df_co = nodal_production_tom_co.dot(industry_base_totals_co.T) - nodal_df = pd.concat([nodal_df, nodal_df_co]) - - rename_sectors = { - "elec": "electricity", - "biomass": "solid biomass", - "heat": "low-temperature heat", - } - nodal_df.rename(columns=rename_sectors, inplace=True) - - nodal_df.index.name = "TWh/a (MtCO2/a)" + + #Fill missing carriers with 0s + for country in countries: + carriers_present = industry_base_totals.xs(country, level="country").index + missing_carriers = set(all_carriers) - set(carriers_present) + for carrier in missing_carriers: + # Add the missing carrier with a value of 0 + industry_base_totals.loc[(country, carrier), :] = 0 + + nodal_df = pd.DataFrame() + + for country in countries: + nodal_production_tom_co = nodal_production_tom[ + nodal_production_tom.index.to_series().str.startswith(country) + ] + industry_base_totals_co = industry_base_totals.loc[country] + # final energy consumption per node and industry (TWh/a) + nodal_df_co = nodal_production_tom_co.dot(industry_base_totals_co.T) + nodal_df = pd.concat([nodal_df, nodal_df_co]) + + rename_sectors = { + "elec": "electricity", + "biomass": "solid biomass", + "heat": "low-temperature heat", + } + nodal_df.rename(columns=rename_sectors, inplace=True) + + nodal_df.index.name = "MWh/a (tCO2/a)" nodal_df.to_csv( snakemake.output.industrial_energy_demand_per_node, float_format="%.2f" From 136a25791b9153d3c37969117cd7162ab9fa6dcb Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Fri, 15 Mar 2024 22:50:28 +0100 Subject: [PATCH 03/23] adapt snakefile to better accomodate custom data --- Snakefile | 152 +++++++++++++++++++----------------------------------- 1 file changed, 53 insertions(+), 99 deletions(-) diff --git a/Snakefile b/Snakefile index e303037e..e6c4d5f4 100644 --- a/Snakefile +++ b/Snakefile @@ -614,104 +614,58 @@ rule clean: shell("rm -r " + PYPSAEARTH_FOLDER + "/resources") shell("rm -r " + PYPSAEARTH_FOLDER + "/networks") +rule build_industrial_distribution_key: #default data + input: + regions_onshore=pypsaearth( + "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" + ), + clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}.csv", + clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}.csv", + industrial_database="data/industrial_database.csv", + shapes_path=pypsaearth( + "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" + ), + output: + industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", + threads: 1 + resources: + mem_mb=1000, + benchmark: + "benchmarks/build_industrial_distribution_key_elec_s{simpl}_{clusters}" + script: + "scripts/build_industrial_distribution_key.py" -if config["custom_data"].get("industry_demand", False) == True: - - rule build_industrial_distribution_key: #custom data - input: - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}.csv", - clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}.csv", - industrial_database="resources/custom_data/industrial_database.csv", - shapes_path=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - #shapes_path=PYPSAEARTH_FOLDER + "/resources/shapes/MAR2.geojson", - output: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", - threads: 1 - resources: - mem_mb=1000, - benchmark: - "benchmarks/build_industrial_distribution_key/s{simpl}_{clusters}" - script: - "scripts/build_industrial_distribution_key.py" - - rule build_industry_demand: #custom data - input: - industry_sector_ratios="resources/custom_data/industry_sector_ratios_{demand}_{planning_horizons}.csv", - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", - industrial_production_per_country_tomorrow="resources/custom_data/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", - costs=CDIR - + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - industry_growth_cagr="data/demand/industry_growth_cagr.csv", - output: - industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", - threads: 1 - resources: - mem_mb=1000, - benchmark: - "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" - script: - "scripts/build_industry_demand.py" - - -if config["custom_data"].get("industry_demand", False) == False: - - rule build_industrial_distribution_key: #default data - input: - regions_onshore=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}.csv", - clustered_gdp_layout="resources/gdp_shares/gdp_layout_elec_s{simpl}_{clusters}.csv", - industrial_database="data/industrial_database.csv", - shapes_path=pypsaearth( - "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" - ), - output: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", - threads: 1 - resources: - mem_mb=1000, - benchmark: - "benchmarks/build_industrial_distribution_key_elec_s{simpl}_{clusters}" - script: - "scripts/build_industrial_distribution_key.py" - - rule build_base_industry_totals: #default data - input: - #industrial_production_per_country="data/industrial_production_per_country.csv", - #unsd_path="data/demand/unsd/data/", - energy_totals_base="data/energy_totals_base.csv", - output: - base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", - threads: 1 - resources: - mem_mb=1000, - benchmark: - "benchmarks/build_base_industry_totals_{planning_horizons}_{demand}" - script: - "scripts/build_base_industry_totals.py" +rule build_base_industry_totals: #default data + input: + #industrial_production_per_country="data/industrial_production_per_country.csv", + #unsd_path="data/demand/unsd/data/", + energy_totals_base="data/energy_totals_base.csv", + output: + base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", + threads: 1 + resources: + mem_mb=1000, + benchmark: + "benchmarks/build_base_industry_totals_{planning_horizons}_{demand}" + script: + "scripts/build_base_industry_totals.py" - rule build_industry_demand: #default data - input: - industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", - #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", - #industrial_production_per_country="data/industrial_production_per_country.csv", - base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", - industrial_database="data/industrial_database.csv", - costs=CDIR - + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), - industry_growth_cagr="data/demand/industry_growth_cagr.csv", - output: - industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", - threads: 1 - resources: - mem_mb=1000, - benchmark: - "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" - script: - "scripts/build_industry_demand.py" +rule build_industry_demand: #default data + input: + industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", + #industrial_production_per_country_tomorrow="resources/demand/industrial_production_per_country_tomorrow_{planning_horizons}_{demand}.csv", + #industrial_production_per_country="data/industrial_production_per_country.csv", + base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", + industrial_database="data/industrial_database.csv", + costs=CDIR + + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), + industry_growth_cagr="data/demand/industry_growth_cagr.csv", + output: + industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", + threads: 1 + resources: + mem_mb=1000, + benchmark: + "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv" + script: + "scripts/build_industry_demand.py" From 0cb27665c1b01c8b86957327d773acd2f6383006 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Fri, 15 Mar 2024 22:52:51 +0100 Subject: [PATCH 04/23] adapt config files --- config.default.yaml | 8 +++++--- test/config.test1.yaml | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 809d4f48..9631a24b 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -5,7 +5,7 @@ results_dir: results/ summary_dir: results/ costs_dir: data/ #TODO change to the equivalent of technology data -run: test_h2_cons_yearly +run: test_run foresight: overnight @@ -55,7 +55,7 @@ fossil_reserves: export: - h2export: [120] # Yearly export demand in TWh + h2export: [10] # Yearly export demand in TWh store: true # [True, False] # specifies wether an export store to balance demand is implemented store_capital_costs: "no_costs" # ["standard_costs", "no_costs"] # specifies the costs of the export store. "standard_costs" takes CAPEX of "hydrogen storage tank type 1 including compressor" export_profile: "ship" # use "ship" or "constant" @@ -70,6 +70,7 @@ custom_data: elec_demand: false heat_demand: false industry_demand: false + industry_database: false transport_demand: false water_costs: false h2_underground: false @@ -313,7 +314,7 @@ sector: NZ_2050: 0.36 DF_2050: 0.12 - gadm_level: 2 + gadm_level: 1 h2_cavern: true marginal_cost_storage: 0 methanation: true @@ -556,3 +557,4 @@ plotting: H2 for industry: "#222222" H2 for shipping: "#6495ED" biomass EOP: "green" + biomass: "green" diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 9a2fca64..cfcc95fe 100644 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -72,6 +72,7 @@ custom_data: elec_demand: false heat_demand: false industry_demand: false + industry_database: false transport_demand: false water_costs: false h2_underground: false From 7619ea732e5a176be73fa25f535906a2185adb6c Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Fri, 15 Mar 2024 23:03:19 +0100 Subject: [PATCH 05/23] add custom data templates to guide the user --- data_custom/TEMPLATE_industrial_database.csv | 2 ++ data_custom/TEMPLATE_industry_demand_AB_2030.csv | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 data_custom/TEMPLATE_industrial_database.csv create mode 100644 data_custom/TEMPLATE_industry_demand_AB_2030.csv diff --git a/data_custom/TEMPLATE_industrial_database.csv b/data_custom/TEMPLATE_industrial_database.csv new file mode 100644 index 00000000..902fef6f --- /dev/null +++ b/data_custom/TEMPLATE_industrial_database.csv @@ -0,0 +1,2 @@ +country,y,x,technology,capacity,unit,quality,location +MA,32.992424,-7.622267,Industry NMM Cement,790, kt/yr,actual,Settat diff --git a/data_custom/TEMPLATE_industry_demand_AB_2030.csv b/data_custom/TEMPLATE_industry_demand_AB_2030.csv new file mode 100644 index 00000000..a97d197d --- /dev/null +++ b/data_custom/TEMPLATE_industry_demand_AB_2030.csv @@ -0,0 +1,9 @@ +country,carrier,Industry Steel Primary Blast Furnace Open Hearth Furnace,Industry Steel Primary Blast Furnace Basic Oxygen Furnace,Industry Steel Primary DRI,Industry Steel Secondary EAF,Industry Steel Other,Industry Chemical Ammonia SMR,Industry Chemical Ammonia Other conventional,Industry Chemical Ammonia Renewable,Industry Chemical HVC Naphtha,Industry Chemical HVC LPG,Industry Chemical HVC Methanol,Industry Chemical Other,Industry NMM Cement,Industry NMM Other,Industry Food and tobacco,Industry Construction,Industry Mining,Industry Machinery,Industry Non ferrous metals Aluminium Primary,Industry Non ferrous metals Aluminium Secondary,Industry Non ferrous metals Other,Industry Paper and pulp Pulp Primary,Industry Paper and pulp Pulp Secondary,Industry Paper and pulp Paper,Industry Paper and pulp Other,Industry Transport equipment,Industry Textiles and leather,Industry Wood,Industry Miscellaneous +MA,oil,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,10726146.18,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,397010.79,0.0,0.0,0.0,0.0,13258151.395154 +MA,gas,0.0,0.0,3601545.78,236922.08000000002,0.0,1788930.0,0.0,0.0,0.0,0.0,0.0,0.0,9281486.520000001,0.0,0.0,0.0,0.0,0.0,26202513.16,0.0,0.0,2514199.8,0.0,41960.49,0.0,0.0,0.0,0.0,207158615.483001 +MA,electricity,0.0,0.0,1031598.5399999999,962495.9500000001,0.0,93825.0,0.0,132050.0,1884697.5200000003,0.0,55432.72,0.0,3265708.22,0.0,0.0,0.0,0.0,0.0,11232826.1,0.0,0.0,838066.6,0.0,187208.34,0.0,0.0,0.0,0.0,55242297.464439005 +MA,coal,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,6875175.2,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +MA,heat,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 +MA,biomass,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,171879.38,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,276211.484037 +MA,hydrogen,0.0,0.0,1158286.08,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,276211.484037 +MA,process emission,0.0,0.0,50065.41247790604,40962.59763502796,0.0,0.0,0.0,0.0,67875.68113871076,0.0,1385.228977747275,8.72479576360429e-05,9323308.507746331,0.03203319690797162,0.0,0.0,0.0,0.0,1658616.7007389446,0.0,0.18963180469598598,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 From 2d723ced72565d298294fc62dc4e1359e4e9dfe7 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Fri, 15 Mar 2024 23:03:48 +0100 Subject: [PATCH 06/23] remove duplications in AL production --- data/AL_production.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/data/AL_production.csv b/data/AL_production.csv index 706f643d..5f0f6734 100644 --- a/data/AL_production.csv +++ b/data/AL_production.csv @@ -241,7 +241,6 @@ TC,0,2019,no available information online assumed value of 0 TV,0,2019,no available information online assumed value of 0 UG,0,2019,no available information online assumed value of 0 UA,0,2019,no available information online assumed value of 0 -AE,0,2019,no available information online assumed value of 0 GB,0,2019,no available information online assumed value of 0 US,0,2019,no available information online assumed value of 0 UM,0,2019,no available information online assumed value of 0 From 0f2fefc3d2d6727100b4b7f02aa91b26f35eeeb7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 22:14:50 +0000 Subject: [PATCH 07/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Snakefile | 6 +++-- scripts/build_industrial_distribution_key.py | 24 ++++++++--------- scripts/build_industry_demand.py | 28 ++++++++++++-------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/Snakefile b/Snakefile index e6c4d5f4..b73625ce 100644 --- a/Snakefile +++ b/Snakefile @@ -614,6 +614,7 @@ rule clean: shell("rm -r " + PYPSAEARTH_FOLDER + "/resources") shell("rm -r " + PYPSAEARTH_FOLDER + "/networks") + rule build_industrial_distribution_key: #default data input: regions_onshore=pypsaearth( @@ -635,6 +636,7 @@ rule build_industrial_distribution_key: #default data script: "scripts/build_industrial_distribution_key.py" + rule build_base_industry_totals: #default data input: #industrial_production_per_country="data/industrial_production_per_country.csv", @@ -650,6 +652,7 @@ rule build_base_industry_totals: #default data script: "scripts/build_base_industry_totals.py" + rule build_industry_demand: #default data input: industrial_distribution_key="resources/demand/industrial_distribution_key_elec_s{simpl}_{clusters}.csv", @@ -657,8 +660,7 @@ rule build_industry_demand: #default data #industrial_production_per_country="data/industrial_production_per_country.csv", base_industry_totals="resources/demand/base_industry_totals_{planning_horizons}_{demand}.csv", industrial_database="data/industrial_database.csv", - costs=CDIR - + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), + costs=CDIR + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), industry_growth_cagr="data/demand/industry_growth_cagr.csv", output: industrial_energy_demand_per_node="resources/demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index f5d7a8a3..52cf6a8f 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -139,31 +139,31 @@ def match_technology(df): lambda name: three_2_two_digits_country(name[:3]) + name[3:] ) - if snakemake.config["custom_data"]["industry_database"]: - logger.info("Using custom industry database from 'data_custom/industrial_database.csv' instead of default") + logger.info( + "Using custom industry database from 'data_custom/industrial_database.csv' instead of default" + ) geo_locs = pd.read_csv( - "data_custom/industrial_database.csv", - sep=",", - header=0, - keep_default_na=False, # , index_col=0 + "data_custom/industrial_database.csv", + sep=",", + header=0, + keep_default_na=False, # , index_col=0 ) geo_locs["industry"] = geo_locs["technology"] else: logger.info("Using default industry database") geo_locs = pd.read_csv( - snakemake.input.industrial_database, - sep=",", - header=0, - keep_default_na=False, # , index_col=0 + snakemake.input.industrial_database, + sep=",", + header=0, + keep_default_na=False, # , index_col=0 ) - geo_locs = geo_locs[geo_locs['country'].isin(countries)] + geo_locs = geo_locs[geo_locs["country"].isin(countries)] geo_locs["capacity"] = pd.to_numeric(geo_locs.capacity) # Call the function to add the "industry" column df_with_industry = match_technology(geo_locs) - geo_locs.capacity = pd.to_numeric(geo_locs.capacity) gadm_clustering = snakemake.config["clustering_options"]["alternative_clustering"] diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index ce2f534c..b36c7198 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -40,7 +40,7 @@ def country_to_nodal(industrial_production, keys): mapping = sector key = keys.loc[buses, mapping] - #print(sector) + # print(sector) nodal_production.loc[buses, sector] = ( industrial_production.at[country, sector] * key ) @@ -66,18 +66,27 @@ def country_to_nodal(industrial_production, keys): countries = snakemake.config["countries"] - if snakemake.config["custom_data"]["industry_demand"]: + _logger.info( + "Fetching custom industry demand data.. expecting file at 'data_custom/industry_demand_{0}_{1}.csv'".format( + snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"] + ) + ) - _logger.info("Fetching custom industry demand data.. expecting file at 'data_custom/industry_demand_{0}_{1}.csv'".format(snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"])) - - industry_demand = pd.read_csv("data_custom/industry_demand_{0}_{1}.csv".format(snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"]), index_col=[0,1]) + industry_demand = pd.read_csv( + "data_custom/industry_demand_{0}_{1}.csv".format( + snakemake.wildcards["demand"], snakemake.wildcards["planning_horizons"] + ), + index_col=[0, 1], + ) keys_path = snakemake.input.industrial_distribution_key dist_keys = pd.read_csv( keys_path, index_col=0, keep_default_na=False, na_values=[""] ) - production_base = pd.DataFrame(1, columns=industry_demand.columns, index=countries) + production_base = pd.DataFrame( + 1, columns=industry_demand.columns, index=countries + ) nodal_keys = country_to_nodal(production_base, dist_keys) nodal_df = pd.DataFrame() @@ -98,9 +107,7 @@ def country_to_nodal(industrial_production, keys): cagr = read_csv_nafix(snakemake.input.industry_growth_cagr, index_col=0) - - - # Building nodal industry production growth + # Building nodal industry production growth for country in countries: if country not in cagr.index: @@ -136,7 +143,6 @@ def country_to_nodal(industrial_production, keys): # production of industries per node compared to current nodal_production_tom = country_to_nodal(production_tom, dist_keys) - clean_industry_list = [ "iron and steel", "chemical and petrochemical", @@ -267,7 +273,7 @@ def match_technology(df): "low-temperature heat", ] - #Fill missing carriers with 0s + # Fill missing carriers with 0s for country in countries: carriers_present = industry_base_totals.xs(country, level="country").index missing_carriers = set(all_carriers) - set(carriers_present) From fa971b4789cd6d11a2d498245482ae8dfc476e40 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Sun, 17 Mar 2024 14:59:18 +0100 Subject: [PATCH 08/23] fix renaming bug in build industry demand --- scripts/build_industry_demand.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index b36c7198..1f1d177d 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -57,7 +57,7 @@ def country_to_nodal(industrial_production, keys): snakemake = mock_snakemake( "build_industry_demand", simpl="", - clusters=12, + clusters=13, planning_horizons=2030, demand="AB", ) @@ -292,12 +292,12 @@ def match_technology(df): nodal_df_co = nodal_production_tom_co.dot(industry_base_totals_co.T) nodal_df = pd.concat([nodal_df, nodal_df_co]) - rename_sectors = { - "elec": "electricity", - "biomass": "solid biomass", - "heat": "low-temperature heat", - } - nodal_df.rename(columns=rename_sectors, inplace=True) + rename_sectors = { + "elec": "electricity", + "biomass": "solid biomass", + "heat": "low-temperature heat", + } + nodal_df.rename(columns=rename_sectors, inplace=True) nodal_df.index.name = "MWh/a (tCO2/a)" From 9c59ddba7c7ad07ba63d8bb666d88a7ad8c25f61 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Sun, 17 Mar 2024 15:01:03 +0100 Subject: [PATCH 09/23] adapt prepare sector nettwork to not expect 3-letter coc code for custom pipelines --- scripts/prepare_sector_network.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 2490d313..97ec2f37 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -133,7 +133,7 @@ def add_oil(n, costs): n.add("Carrier", "oil") # Set the "co2_emissions" of the carrier "oil" to 0, because the emissions of oil usage taken from the spatial.oil.nodes are accounted seperately (directly linked to the co2 atmosphere bus). Setting the carrier to 0 here avoids double counting. Be aware to link oil emissions to the co2 atmosphere bus. - # n.carriers.loc["oil", "co2_emissions"] = 0 + n.carriers.loc["oil", "co2_emissions"] = 0 # print("co2_emissions of oil set to 0 for testing") # TODO add logger.info n.madd( @@ -344,18 +344,18 @@ def add_hydrogen(n, costs): h2_links["bus0"] = buses_ordered.str[0] + "_AC" h2_links["bus1"] = buses_ordered.str[1] + "_AC" - # Conversion of GADM id to from 3 to 2-digit - h2_links["bus0"] = ( - h2_links["bus0"] - .str.split(".") - .apply(lambda id: three_2_two_digits_country(id[0]) + "." + id[1]) - ) - h2_links["bus1"] = ( - h2_links["bus1"] - .str.split(".") - .apply(lambda id: three_2_two_digits_country(id[0]) + "." + id[1]) - ) - + # # Conversion of GADM id to from 3 to 2-digit + # h2_links["bus0"] = ( + # h2_links["bus0"] + # .str.split(".") + # .apply(lambda id: three_2_two_digits_country(id[0]) + "." + id[1]) + # ) + # h2_links["bus1"] = ( + # h2_links["bus1"] + # .str.split(".") + # .apply(lambda id: three_2_two_digits_country(id[0]) + "." + id[1]) + # ) + # Create index column h2_links["buses_idx"] = ( "H2 pipeline " + h2_links["bus0"] + " -> " + h2_links["bus1"] @@ -2339,7 +2339,7 @@ def add_rail_transport(n, costs): snakemake = mock_snakemake( "prepare_sector_network", simpl="", - clusters="10", + clusters="13", ll="c1.0", opts="Co2L", planning_horizons="2030", From 8f98860ac1a2f785a7db8ca96e88e8f7c6bd6f3c Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Sun, 17 Mar 2024 15:01:55 +0100 Subject: [PATCH 10/23] change folder for custom pipelines --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index b73625ce..6d4d6e79 100644 --- a/Snakefile +++ b/Snakefile @@ -168,7 +168,7 @@ rule prepare_sector_network: shapes_path=pypsaearth( "resources/bus_regions/regions_onshore_elec_s{simpl}_{clusters}.geojson" ), - pipelines="resources/custom_data/pipelines.csv" + pipelines="data_custom/pipelines.csv" if config["custom_data"]["gas_network"] else "resources/gas_networks/gas_network_elec_s{simpl}_{clusters}.csv", output: From be57f448799e55309b5ac6528ed25f76883ac2f6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 17 Mar 2024 14:02:19 +0000 Subject: [PATCH 11/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 97ec2f37..a8ed071c 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -355,7 +355,7 @@ def add_hydrogen(n, costs): # .str.split(".") # .apply(lambda id: three_2_two_digits_country(id[0]) + "." + id[1]) # ) - + # Create index column h2_links["buses_idx"] = ( "H2 pipeline " + h2_links["bus0"] + " -> " + h2_links["bus1"] From c2f5b1ca7c3ee755cdb76ca69888247db8a1d266 Mon Sep 17 00:00:00 2001 From: Hazem-IEG Date: Sun, 17 Mar 2024 19:37:19 +0100 Subject: [PATCH 12/23] option to exclude international bunkers --- scripts/prepare_sector_network.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a8ed071c..9bec5a23 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -792,7 +792,12 @@ def add_co2(n, costs): def add_aviation(n, cost): - all_aviation = ["total international aviation", "total domestic aviation"] + + if snakemake.config["sector"]["international_bunkers"]: + all_aviation = ["total international aviation", "total domestic aviation"] + else: + all_aviation = ["total domestic aviation"] + aviation_demand = ( energy_totals.loc[countries, all_aviation].sum(axis=1).sum() # * 1e6 / 8760 @@ -938,7 +943,7 @@ def h2_hc_conversions(n, costs): lifetime=costs.at["helmeth", "lifetime"], ) - if options["SMR"]: + if options["SMR CC"]: n.madd( "Link", spatial.nodes, @@ -956,6 +961,8 @@ def h2_hc_conversions(n, costs): lifetime=costs.at["SMR CC", "lifetime"], ) + if options["SMR"]: + n.madd( "Link", nodes + " SMR", @@ -979,7 +986,11 @@ def add_shipping(n, costs): gadm_level = options["gadm_level"] - all_navigation = ["total international navigation", "total domestic navigation"] + if snakemake.config["sector"]["international_bunkers"]: + all_navigation = ["total international navigation", "total domestic navigation"] + else: + all_navigation = ["total domestic navigation"] + navigation_demand = ( energy_totals.loc[countries, all_navigation].sum(axis=1).sum() # * 1e6 / 8760 From 8add797c410a0ebdca070d4b5581be1c5fcaca46 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 17 Mar 2024 18:37:39 +0000 Subject: [PATCH 13/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_sector_network.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9bec5a23..a94a90e0 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -792,13 +792,11 @@ def add_co2(n, costs): def add_aviation(n, cost): - if snakemake.config["sector"]["international_bunkers"]: all_aviation = ["total international aviation", "total domestic aviation"] else: all_aviation = ["total domestic aviation"] - aviation_demand = ( energy_totals.loc[countries, all_aviation].sum(axis=1).sum() # * 1e6 / 8760 ) @@ -962,7 +960,6 @@ def h2_hc_conversions(n, costs): ) if options["SMR"]: - n.madd( "Link", nodes + " SMR", @@ -991,7 +988,6 @@ def add_shipping(n, costs): else: all_navigation = ["total domestic navigation"] - navigation_demand = ( energy_totals.loc[countries, all_navigation].sum(axis=1).sum() # * 1e6 / 8760 ) From 89a84142c56ea8687c0fd5dfbb665e48f0f20a93 Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:48:48 +0200 Subject: [PATCH 14/23] Update config.test1.yaml --- test/config.test1.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index cfcc95fe..6540ef73 100644 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -321,6 +321,7 @@ sector: helmeth: true dac: true SMR: true + SMR CC: true cc_fraction: 0.9 cc: true space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating. From 18b81cdaa102eb412b2e13a61a4c27afeca0a945 Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:49:19 +0200 Subject: [PATCH 15/23] Update config.default.yaml --- config.default.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.default.yaml b/config.default.yaml index 9631a24b..93ba14d3 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -321,6 +321,7 @@ sector: helmeth: true dac: true SMR: true + SMR CC: true cc_fraction: 0.9 cc: true space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating. From e50262e5b2b76cd48aa61d44e9af181490d723eb Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:38:46 +0200 Subject: [PATCH 16/23] Update config.default.yaml --- config.default.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.default.yaml b/config.default.yaml index 93ba14d3..7614ee71 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -172,6 +172,8 @@ sector: gas_network_repurposing: true # If true -> ["sector"]["gas"]["network"] is automatically false underground_storage: false + international_bunkers: false #Whether or not to count the demand of international aviation and navigation + oil: spatial_oil: true From 6c304fb3b6f6384ba72133fc4954fb091cbc7e34 Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:39:16 +0200 Subject: [PATCH 17/23] Update config.test1.yaml --- test/config.test1.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/config.test1.yaml b/test/config.test1.yaml index 6540ef73..caa95dc6 100644 --- a/test/config.test1.yaml +++ b/test/config.test1.yaml @@ -173,6 +173,8 @@ sector: gas_network_repurposing: true # If true -> ["sector"]["gas"]["network"] is automatically false underground_storage: false + international_bunkers: false #Whether or not to count the emissions of international aviation and navigation + oil: spatial_oil: true From 3da196911bf5c3c71ccaa9006df130dc1bb546e5 Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:39:46 +0200 Subject: [PATCH 18/23] Update config.default.yaml --- config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 7614ee71..772b2c04 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -172,7 +172,7 @@ sector: gas_network_repurposing: true # If true -> ["sector"]["gas"]["network"] is automatically false underground_storage: false - international_bunkers: false #Whether or not to count the demand of international aviation and navigation + international_bunkers: false #Whether or not to count the emissions of international aviation and navigation oil: spatial_oil: true From 6a16987a4bdda252062fb401c95c689a228c490c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Apr 2024 15:00:42 +0000 Subject: [PATCH 19/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_industry_demand.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index c86824be..62ba3bbd 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -140,7 +140,6 @@ def country_to_nodal(industrial_production, keys): keys_path, index_col=0, keep_default_na=False, na_values=[""] ) - # production of industries per node compared to current nodal_production_tom = country_to_nodal(production_tom, dist_keys) @@ -207,7 +206,7 @@ def match_technology(df): "production[ktons/a]" ].loc[countries] AL_emissions = AL_prod_tom * emission_factors["non-ferrous metals"] - + Steel_emissions = ( geo_locs[geo_locs.industry == "iron and steel"] .groupby("country") From f2d62c3f9fbab505b8ae2fe2056e386cb1bcd619 Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:29:17 +0200 Subject: [PATCH 20/23] Update prepare_gas_network.py --- scripts/prepare_gas_network.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index de0e20a9..b6a6ad6e 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -903,7 +903,18 @@ def plot_clustered_gas_network(pipelines, bus_regions_onshore): plot_gas_network(pipelines, country_borders, bus_regions_onshore) pipelines = cluster_gas_network(pipelines, bus_regions_onshore, length_factor=1.25) + + # Conversion of GADM id to from 3 to 2-digit + pipelines["bus0"] = ( + pipelines["bus0"] + .apply(lambda id: three_2_two_digits_country(id[:3]) + id[3:]) + ) + pipelines["bus1"] = ( + pipelines["bus1"] + .apply(lambda id: three_2_two_digits_country(id[:3]) + id[3:]) + ) + pipelines.to_csv(snakemake.output.clustered_gas_network, index=False) plot_clustered_gas_network(pipelines, bus_regions_onshore) From 1c4dfe6ba530e00287af5fa3b7a6be27f9ce6beb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 12:34:52 +0000 Subject: [PATCH 21/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_gas_network.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index 5fc949fb..9d6e13fe 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -903,22 +903,19 @@ def plot_clustered_gas_network(pipelines, bus_regions_onshore): if len(pipelines.loc[pipelines.amount_states_passed >= 2]) > 0: plot_gas_network(pipelines, country_borders, bus_regions_onshore) - pipelines = cluster_gas_network( pipelines, bus_regions_onshore, length_factor=1.25 ) - + # Conversion of GADM id to from 3 to 2-digit - pipelines["bus0"] = ( - pipelines["bus0"] - .apply(lambda id: three_2_two_digits_country(id[:3]) + id[3:]) + pipelines["bus0"] = pipelines["bus0"].apply( + lambda id: three_2_two_digits_country(id[:3]) + id[3:] ) - pipelines["bus1"] = ( - pipelines["bus1"] - .apply(lambda id: three_2_two_digits_country(id[:3]) + id[3:]) + pipelines["bus1"] = pipelines["bus1"].apply( + lambda id: three_2_two_digits_country(id[:3]) + id[3:] ) - + pipelines.to_csv(snakemake.output.clustered_gas_network, index=False) plot_clustered_gas_network(pipelines, bus_regions_onshore) From 2eed995818922faadcb4c368289ec9aa431db41d Mon Sep 17 00:00:00 2001 From: Hazem <87850910+hazemakhalek@users.noreply.github.com> Date: Thu, 2 May 2024 11:14:40 +0200 Subject: [PATCH 22/23] Update prepare_gas_network.py --- scripts/prepare_gas_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index 9d6e13fe..5a002423 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -827,7 +827,10 @@ def plot_clustered_gas_network(pipelines, bus_regions_onshore): # Create a new GeoDataFrame with centroids centroids = bus_regions_onshore.copy() centroids["geometry"] = centroids["geometry"].centroid - + centroids["gadm_id"] = ( + centroids["gadm_id"] + .apply(lambda id: three_2_two_digits_country(id[:3]) + id[3:]) + ) gdf1 = pd.merge( pipelines, centroids, left_on=["bus0"], right_on=["gadm_id"], how="left" ) From 6e56d84b5dc1f075c8395cd8d7c05ca8d692aa9d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 09:14:51 +0000 Subject: [PATCH 23/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_gas_network.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py index 5a002423..d08003ab 100644 --- a/scripts/prepare_gas_network.py +++ b/scripts/prepare_gas_network.py @@ -827,9 +827,8 @@ def plot_clustered_gas_network(pipelines, bus_regions_onshore): # Create a new GeoDataFrame with centroids centroids = bus_regions_onshore.copy() centroids["geometry"] = centroids["geometry"].centroid - centroids["gadm_id"] = ( - centroids["gadm_id"] - .apply(lambda id: three_2_two_digits_country(id[:3]) + id[3:]) + centroids["gadm_id"] = centroids["gadm_id"].apply( + lambda id: three_2_two_digits_country(id[:3]) + id[3:] ) gdf1 = pd.merge( pipelines, centroids, left_on=["bus0"], right_on=["gadm_id"], how="left"