From 823df52309d1a1a44bb449802c2fdbaee8cb0d95 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Mon, 11 Sep 2023 22:51:31 +0200
Subject: [PATCH 01/15] add backward compatible config in wildcards

---
 config/config.default.yaml        |  52 +++++++++++++
 rules/build_electricity.smk       |   4 +
 rules/build_sector.smk            |   2 +
 scripts/_helpers.py               |  14 ++++
 scripts/prepare_network.py        | 125 +++++++++++++++++-------------
 scripts/prepare_sector_network.py | 103 ++++++++++++++----------
 6 files changed, 207 insertions(+), 93 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index b162b75dc..fcc74981b 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -60,6 +60,14 @@ snapshots:
   end: "2014-01-01"
   inclusive: 'left'
 
+snapshot_opts:
+  average_every_nhours:
+    enable: false
+    hour: 2
+  time_segmentation:
+    enable: false
+    hour: 4380
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable
 enable:
   retrieve: auto
@@ -73,6 +81,22 @@ enable:
   retrieve_natura_raster: true
   custom_busmap: false
 
+enable_sector:
+  no_heat_district: false
+  land_transport: false
+  heating: false
+  waste_heat: false
+  biomass: false
+  biomass_transport: false
+  agriculture_machinery: false
+  industry: false
+  decentral: false
+  #wave_energy: false
+  #wave_energy_factor:
+  noH2network: false
+  carbon_budget: false
+  co2limit_sector: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
   2020: 0.701
@@ -83,10 +107,22 @@ co2_budget:
   2045: 0.032
   2050: 0.000
 
+co2_budget_opts:
+  from_descrete_value:
+    enable: true
+  from_beta_decay:
+    enable: false # TODO: move to own rule with sector-opts wildcard?
+    value: 1
+  from_exp_decay:
+    enable: false
+    value: 1
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
   voltages: [220., 300., 380.]
+  gaslimit_enable: false
   gaslimit: false
+  co2limit_enable: false
   co2limit: 7.75e+7
   co2base: 1.487e+9
   agg_p_nom_limits: data/agg_p_nom_minmax.csv
@@ -123,6 +159,17 @@ electricity:
       Onshore: [onwind]
       PV: [solar]
 
+  adjust_carrier: # This is the solar+c0.5 thing
+    enable: false
+    #solar:
+    #  p_nom_max:
+    #  capital_cost:
+    #  marginal_cost:
+
+  autarky:
+    enable: false
+    by_country: false
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
   default_cutout: europe-2013-era5
@@ -573,6 +620,11 @@ costs:
   emission_prices:
     co2: 0.
 
+  enable:
+    emission_prices: false
+    monthly_prices: false
+
+
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
   simplify_network:
diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index f9fdc3ac2..6b9cde24d 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -473,10 +473,14 @@ rule prepare_network:
         links=config["links"],
         lines=config["lines"],
         co2base=config["electricity"]["co2base"],
+        co2limit_enable=config["electricity"].get("co2limit_enable", False),
         co2limit=config["electricity"]["co2limit"],
+        gaslimit_enable=config["electricity"].get("gaslimit_enable", False),
         gaslimit=config["electricity"].get("gaslimit"),
         max_hours=config["electricity"]["max_hours"],
         costs=config["costs"],
+        snapshot_opts=config.get("snapshot_opts",{}),
+        autarky=config["electricity"].get("autarky",{}),
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 10a5f821c..9845ac136 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -705,6 +705,7 @@ rule build_transport_demand:
 
 rule prepare_sector_network:
     params:
+        enable_sector=config.get("enable_sector", {}),
         co2_budget=config["co2_budget"],
         conventional_carriers=config["existing_capacities"]["conventional_carriers"],
         foresight=config["foresight"],
@@ -717,6 +718,7 @@ rule prepare_sector_network:
         countries=config["countries"],
         emissions_scope=config["energy"]["emissions"],
         eurostat_report_year=config["energy"]["eurostat_report_year"],
+        snapshot_opts=config.get("snapshot_opts",{}),
         RDIR=RDIR,
     input:
         **build_retro_cost_output,
diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index fc7bc9e0b..714bf33fe 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -7,6 +7,7 @@
 import logging
 import os
 import urllib
+import re
 from pathlib import Path
 
 import pandas as pd
@@ -21,6 +22,19 @@
 REGION_COLS = ["geometry", "name", "x", "y", "country"]
 
 
+def get_opt(opts, expr, flags=None):
+    """
+    Return the first option matching the regular expression.
+    The regular expression is case-insensitive by default.
+    """
+    if flags is None:
+        flags = re.IGNORECASE
+    for o in opts:
+        match = re.match(expr, o, flags=flags)
+        if match:
+            return match.group(0)
+    return None
+
 # Define a context manager to temporarily mute print statements
 @contextlib.contextmanager
 def mute_print():
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index a5a00a3cb..c3fe74b19 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -63,7 +63,7 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, get_opt
 from add_electricity import load_costs, update_transmission_costs
 from pypsa.descriptors import expand_series
 
@@ -71,6 +71,18 @@
 
 logger = logging.getLogger(__name__)
 
+def find_opt(opts, expr):
+    """
+    Return if available the float after the expression.
+    """
+    for o in opts:
+        if expr in o:
+            m = re.findall("[0-9]*\.?[0-9]+$", o)
+            if len(m) > 0:
+                return True, float(m[0])
+            else:
+                return True, None
+    return False, None
 
 def add_co2limit(n, co2limit, Nyears=1.0):
     n.add(
@@ -296,43 +308,47 @@ def set_line_nom_max(
     )
 
     set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])
-
-    for o in opts:
-        m = re.match(r"^\d+h$", o, re.IGNORECASE)
-        if m is not None:
-            n = average_every_nhours(n, m.group(0))
-            break
-
-    for o in opts:
-        m = re.match(r"^\d+seg$", o, re.IGNORECASE)
-        if m is not None:
-            solver_name = snakemake.config["solving"]["solver"]["name"]
-            n = apply_time_segmentation(n, m.group(0)[:-3], solver_name)
-            break
-
-    for o in opts:
-        if "Co2L" in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                co2limit = float(m[0]) * snakemake.params.co2base
-                add_co2limit(n, co2limit, Nyears)
-                logger.info("Setting CO2 limit according to wildcard value.")
-            else:
-                add_co2limit(n, snakemake.params.co2limit, Nyears)
-                logger.info("Setting CO2 limit according to config value.")
-            break
-
-    for o in opts:
-        if "CH4L" in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                limit = float(m[0]) * 1e6
-                add_gaslimit(n, limit, Nyears)
-                logger.info("Setting gas usage limit according to wildcard value.")
-            else:
-                add_gaslimit(n, snakemake.params.gaslimit, Nyears)
-                logger.info("Setting gas usage limit according to config value.")
-            break
+    
+    # temporal averaging
+    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{})
+    nhours_enable_config = nhours_opts_config.get("enable",None)
+    nhours_config = str(nhours_opts_config.get("hour",None)) + "h"
+    nhours_wildcard = get_opt(opts, r"^\d+h$")
+    if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None):
+        nhours = nhours_wildcard or nhours_config
+        n = average_every_nhours(n, nhours)
+
+    # segments with package tsam
+    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{})
+    time_seg_enable_config = nhours_opts_config.get("enable",None)
+    time_seg_config = str(nhours_opts_config.get("hour",None)) + "seg"
+    time_seg_wildcard = get_opt(opts, r"^\d+seg$")
+    if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None):
+        time_seg = time_seg_wildcard or time_seg_config
+        solver_name = snakemake.config["solving"]["solver"]["name"]
+        n = apply_time_segmentation(n, time_seg, solver_name)
+
+    Co2L_config = snakemake.params.co2limit_enable and isinstance(snakemake.params.co2limit,float)
+    Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
+    if Co2L_wildcard or Co2L_config:
+        if co2limit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard?
+            co2limit = co2limit_wildcard * snakemake.params.co2base
+            add_co2limit(n, co2limit, Nyears)
+            logger.info("Setting CO2 limit according to wildcard value.")
+        else: 
+            add_co2limit(n, snakemake.params.co2limit, Nyears)
+            logger.info("Setting CO2 limit according to config value.")
+
+    CH4L_config = snakemake.params.gaslimit_enable and isinstance(snakemake.params.gaslimit,float)
+    CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L")
+    if CH4L_wildcard or CH4L_config:
+        if gaslimit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard?
+            gaslimit = gaslimit_wildcard * 1e6
+            add_gaslimit(n, gaslimit, Nyears)
+            logger.info("Setting gas usage limit according to wildcard value.")
+        else: 
+            add_gaslimit(n, snakemake.params.gaslimit, Nyears)
+            logger.info("Setting gas usage limit according to config value.")
 
     for o in opts:
         if "+" not in o:
@@ -353,21 +369,24 @@ def set_line_nom_max(
                     sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel, attr] *= factor
 
+    Ept_config = snakemake.params.costs.get("enable",{}).get("monthly_prices", False)
     for o in opts:
-        if "Ept" in o:
+        if "Ept" in o or Ept_config:
             logger.info(
                 "Setting time dependent emission prices according spot market price"
             )
             add_dynamic_emission_prices(n)
-        elif "Ep" in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                logger.info("Setting emission prices according to wildcard value.")
-                add_emission_prices(n, dict(co2=float(m[0])))
-            else:
-                logger.info("Setting emission prices according to config value.")
-                add_emission_prices(n, snakemake.params.costs["emission_prices"])
-            break
+            Ept_config = True
+
+    Ep_config = snakemake.params.costs.get("enable",{}).get("emission_prices", False)
+    Ep_wildcard, co2_wildcard = find_opt(opts, "Ep")
+    if (Ep_wildcard or Ep_config) and not Ept_config:
+        if co2_wildcard is not None:
+            logger.info("Setting emission prices according to wildcard value.")
+            add_emission_prices(n, dict(co2=co2_wildcard))
+        else: 
+            logger.info("Setting emission prices according to config value.")
+            add_emission_prices(n, snakemake.params.costs["emission_prices"])
 
     ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
     set_transmission_limit(n, ll_type, factor, costs, Nyears)
@@ -380,10 +399,12 @@ def set_line_nom_max(
         p_nom_max_ext=snakemake.params.links.get("max_extension", np.inf),
     )
 
-    if "ATK" in opts:
-        enforce_autarky(n)
-    elif "ATKc" in opts:
-        enforce_autarky(n, only_crossborder=True)
+    autarky_config = snakemake.params.autarky
+    if "ATK" in opts or autarky_config.get("enable", False):
+        only_crossborder = False
+        if "ATKc" in opts or autarky_config.get("by_country", False):
+            only_crossborder = True
+        enforce_autarky(n, only_crossborder=only_crossborder)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output[0])
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 11406bffc..af666f5db 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -17,7 +17,7 @@
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import generate_periodic_profiles, update_config_with_sector_opts
+from _helpers import generate_periodic_profiles, update_config_with_sector_opts, get_opt
 from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
@@ -161,11 +161,11 @@ def define_spatial(nodes, options):
 
 def emission_sectors_from_opts(opts):
     sectors = ["electricity"]
-    if "T" in opts:
+    if "T" in opts or opts_config.get("land_transport",False):
         sectors += ["rail non-elec", "road non-elec"]
-    if "H" in opts:
+    if "H" in opts or opts_config.get("heating",False):
         sectors += ["residential non-elec", "services non-elec"]
-    if "I" in opts:
+    if "I" in opts or opts_config.get("industry",False):
         sectors += [
             "industrial non-elec",
             "industrial processes",
@@ -174,7 +174,10 @@ def emission_sectors_from_opts(opts):
             "domestic navigation",
             "international navigation",
         ]
-    if "A" in opts:
+
+    heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False)
+
+    if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)):
         sectors += ["agriculture"]
 
     return sectors
@@ -3256,27 +3259,39 @@ def set_temporal_aggregation(n, opts, solver_name):
     """
     Aggregate network temporally.
     """
-    for o in opts:
-        # temporal averaging
-        m = re.match(r"^\d+h$", o, re.IGNORECASE)
-        if m is not None:
-            n = average_every_nhours(n, m.group(0))
-            break
-        # representative snapshots
-        m = re.match(r"(^\d+)sn$", o, re.IGNORECASE)
-        if m is not None:
-            sn = int(m[1])
-            logger.info(f"Use every {sn} snapshot as representative")
-            n.set_snapshots(n.snapshots[::sn])
-            n.snapshot_weightings *= sn
-            break
-        # segments with package tsam
-        m = re.match(r"^(\d+)seg$", o, re.IGNORECASE)
-        if m is not None:
-            segments = int(m[1])
-            logger.info(f"Use temporal segmentation with {segments} segments")
-            n = apply_time_segmentation(n, segments, solver_name=solver_name)
-            break
+    # temporal averaging
+    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{})
+    nhours_enable_config = nhours_opts_config.get("enable",None)
+    nhours_config = str(nhours_opts_config.get("hour",None)) + "H"
+    nhours_wildcard = get_opt(opts, r"^\d+h$")
+    if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None):
+        nhours = nhours_wildcard or nhours_config
+        n = average_every_nhours(n, nhours)
+        return n
+
+    # representative snapshots
+    snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots",{})
+    snapshots_enable_config = snapshots_opts_config.get("enable",None)
+    snapshots_config = snapshots_opts_config.get("hour",None)
+    snapshots_wildcard = get_opt(opts, r"(^\d+)sn$")
+    if snapshots_wildcard is not None or (snapshots_enable_config and snapshots_config is not None):
+        sn = int(snapshots_wildcard[:-2]) or snapshots_config
+        logger.info(f"Use every {sn} snapshot as representative")
+        n.set_snapshots(n.snapshots[::sn])
+        n.snapshot_weightings *= sn
+        return n
+
+    # segments with package tsam
+    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{})
+    time_seg_enable_config = nhours_opts_config.get("enable",None)
+    time_seg_config = nhours_opts_config.get("hour",None)
+    time_seg_wildcard = get_opt(opts, r"^(\d+)seg$")
+    if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None):
+        segments = int(time_seg_wildcard[:-3]) or time_seg_config
+        logger.info(f"Use temporal segmentation with {segments} segments")
+        n = apply_time_segmentation(n, segments, solver_name=solver_name)
+        return n
+
     return n
 
 
@@ -3303,6 +3318,10 @@ def set_temporal_aggregation(n, opts, solver_name):
 
     opts = snakemake.wildcards.sector_opts.split("-")
 
+    opts_config = snakemake.params.enable_sector
+
+    heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False)
+
     investment_year = int(snakemake.wildcards.planning_horizons[-4:])
 
     n = pypsa.Network(snakemake.input.network)
@@ -3340,51 +3359,53 @@ def set_temporal_aggregation(n, opts, solver_name):
 
     # TODO merge with opts cost adjustment below
     for o in opts:
-        if o[:4] == "wave":
+        if o[:4] == "wave": # TODO: add config wildcard options or depreciated?
             wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-"))
             logger.info(
                 f"Including wave generators with cost factor of {wave_cost_factor}"
             )
             add_wave(n, wave_cost_factor)
-        if o[:4] == "dist":
+        if o[:4] == "dist": # TODO: add config wildcard options
             options["electricity_distribution_grid"] = True
             options["electricity_distribution_grid_cost_factor"] = float(
                 o[4:].replace("p", ".").replace("m", "-")
             )
-        if o == "biomasstransport":
+    for o in opts:
+        if o == "biomasstransport" or opts_config.get("biomass_transport",False):
             options["biomass_transport"] = True
+            break
 
-    if "nodistrict" in opts:
+    if "nodistrict" in opts or opts_config.get("no_heat_district",False):
         options["district_heating"]["progress"] = 0.0
 
-    if "T" in opts:
+    if "T" in opts or opts_config.get("land_transport",False):
         add_land_transport(n, costs)
 
-    if "H" in opts:
+    if "H" in opts or opts_config.get("heating",False):
         add_heat(n, costs)
 
-    if "B" in opts:
+    if "B" in opts or opts_config.get("biomass",False):
         add_biomass(n, costs)
 
     if options["ammonia"]:
         add_ammonia(n, costs)
 
-    if "I" in opts:
+    if "I" in opts or opts_config.get("industry",False):
         add_industry(n, costs)
 
-    if "I" in opts and "H" in opts:
+    if ("I" in opts and "H" in opts) or (heat_and_industry and opts_config.get("waste_heat",False)):
         add_waste_heat(n)
 
-    if "A" in opts:  # requires H and I
+    if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)):  # requires H and I
         add_agriculture(n, costs)
 
     if options["dac"]:
         add_dac(n, costs)
 
-    if "decentral" in opts:
+    if "decentral" in opts or opts_config.get("decentral",False):
         decentral(n)
 
-    if "noH2network" in opts:
+    if "noH2network" in opts or opts_config.get("noH2network",False):
         remove_h2_network(n)
 
     if options["co2network"]:
@@ -3399,7 +3420,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     limit_type = "config"
     limit = get(snakemake.params.co2_budget, investment_year)
     for o in opts:
-        if "cb" not in o:
+        if "cb" not in o or opts_config.get("carbon_budget",False) is False:
             continue
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv"
@@ -3419,7 +3440,7 @@ def set_temporal_aggregation(n, opts, solver_name):
         limit = co2_cap.loc[investment_year]
         break
     for o in opts:
-        if "Co2L" not in o:
+        if "Co2L" not in o or opts_config.get("co2limit_sector",False) is False:
             continue
         limit_type = "wildcard"
         limit = o[o.find("Co2L") + 4 :]
@@ -3428,7 +3449,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     logger.info(f"Add CO2 limit from {limit_type}")
     add_co2limit(n, nyears, limit)
 
-    for o in opts:
+    for o in opts: # TODO: add config wildcard options or depreciated?
         if not o[:10] == "linemaxext":
             continue
         maxext = float(o[10:]) * 1e3

From 3eed3410448c93426af03e2fce23aaa7db986e73 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 11 Sep 2023 21:03:58 +0000
Subject: [PATCH 02/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/build_electricity.smk       |  4 +-
 rules/build_sector.smk            |  2 +-
 scripts/_helpers.py               |  4 +-
 scripts/prepare_network.py        | 50 +++++++++++-------
 scripts/prepare_sector_network.py | 84 ++++++++++++++++++-------------
 5 files changed, 88 insertions(+), 56 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index 6b9cde24d..b15796c90 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -479,8 +479,8 @@ rule prepare_network:
         gaslimit=config["electricity"].get("gaslimit"),
         max_hours=config["electricity"]["max_hours"],
         costs=config["costs"],
-        snapshot_opts=config.get("snapshot_opts",{}),
-        autarky=config["electricity"].get("autarky",{}),
+        snapshot_opts=config.get("snapshot_opts", {}),
+        autarky=config["electricity"].get("autarky", {}),
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 9845ac136..1bee1ed04 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -718,7 +718,7 @@ rule prepare_sector_network:
         countries=config["countries"],
         emissions_scope=config["energy"]["emissions"],
         eurostat_report_year=config["energy"]["eurostat_report_year"],
-        snapshot_opts=config.get("snapshot_opts",{}),
+        snapshot_opts=config.get("snapshot_opts", {}),
         RDIR=RDIR,
     input:
         **build_retro_cost_output,
diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 714bf33fe..01349e08b 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -6,8 +6,8 @@
 import contextlib
 import logging
 import os
-import urllib
 import re
+import urllib
 from pathlib import Path
 
 import pandas as pd
@@ -25,6 +25,7 @@
 def get_opt(opts, expr, flags=None):
     """
     Return the first option matching the regular expression.
+
     The regular expression is case-insensitive by default.
     """
     if flags is None:
@@ -35,6 +36,7 @@ def get_opt(opts, expr, flags=None):
             return match.group(0)
     return None
 
+
 # Define a context manager to temporarily mute print statements
 @contextlib.contextmanager
 def mute_print():
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index c3fe74b19..2a2d73f81 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -71,6 +71,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 def find_opt(opts, expr):
     """
     Return if available the float after the expression.
@@ -84,6 +85,7 @@ def find_opt(opts, expr):
                 return True, None
     return False, None
 
+
 def add_co2limit(n, co2limit, Nyears=1.0):
     n.add(
         "GlobalConstraint",
@@ -308,45 +310,57 @@ def set_line_nom_max(
     )
 
     set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])
-    
+
     # temporal averaging
-    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{})
-    nhours_enable_config = nhours_opts_config.get("enable",None)
-    nhours_config = str(nhours_opts_config.get("hour",None)) + "h"
+    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {})
+    nhours_enable_config = nhours_opts_config.get("enable", None)
+    nhours_config = str(nhours_opts_config.get("hour", None)) + "h"
     nhours_wildcard = get_opt(opts, r"^\d+h$")
-    if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None):
+    if nhours_wildcard is not None or (
+        nhours_enable_config and nhours_config is not None
+    ):
         nhours = nhours_wildcard or nhours_config
         n = average_every_nhours(n, nhours)
 
     # segments with package tsam
-    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{})
-    time_seg_enable_config = nhours_opts_config.get("enable",None)
-    time_seg_config = str(nhours_opts_config.get("hour",None)) + "seg"
+    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {})
+    time_seg_enable_config = nhours_opts_config.get("enable", None)
+    time_seg_config = str(nhours_opts_config.get("hour", None)) + "seg"
     time_seg_wildcard = get_opt(opts, r"^\d+seg$")
-    if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None):
+    if time_seg_wildcard is not None or (
+        time_seg_enable_config and time_seg_config is not None
+    ):
         time_seg = time_seg_wildcard or time_seg_config
         solver_name = snakemake.config["solving"]["solver"]["name"]
         n = apply_time_segmentation(n, time_seg, solver_name)
 
-    Co2L_config = snakemake.params.co2limit_enable and isinstance(snakemake.params.co2limit,float)
+    Co2L_config = snakemake.params.co2limit_enable and isinstance(
+        snakemake.params.co2limit, float
+    )
     Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
     if Co2L_wildcard or Co2L_config:
-        if co2limit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard?
+        if (
+            co2limit_wildcard is not None
+        ):  # TODO: what if you wat to determine the factor through the wildcard?
             co2limit = co2limit_wildcard * snakemake.params.co2base
             add_co2limit(n, co2limit, Nyears)
             logger.info("Setting CO2 limit according to wildcard value.")
-        else: 
+        else:
             add_co2limit(n, snakemake.params.co2limit, Nyears)
             logger.info("Setting CO2 limit according to config value.")
 
-    CH4L_config = snakemake.params.gaslimit_enable and isinstance(snakemake.params.gaslimit,float)
+    CH4L_config = snakemake.params.gaslimit_enable and isinstance(
+        snakemake.params.gaslimit, float
+    )
     CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L")
     if CH4L_wildcard or CH4L_config:
-        if gaslimit_wildcard is not None: # TODO: what if you wat to determine the factor through the wildcard?
+        if (
+            gaslimit_wildcard is not None
+        ):  # TODO: what if you wat to determine the factor through the wildcard?
             gaslimit = gaslimit_wildcard * 1e6
             add_gaslimit(n, gaslimit, Nyears)
             logger.info("Setting gas usage limit according to wildcard value.")
-        else: 
+        else:
             add_gaslimit(n, snakemake.params.gaslimit, Nyears)
             logger.info("Setting gas usage limit according to config value.")
 
@@ -369,7 +383,7 @@ def set_line_nom_max(
                     sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel, attr] *= factor
 
-    Ept_config = snakemake.params.costs.get("enable",{}).get("monthly_prices", False)
+    Ept_config = snakemake.params.costs.get("enable", {}).get("monthly_prices", False)
     for o in opts:
         if "Ept" in o or Ept_config:
             logger.info(
@@ -378,13 +392,13 @@ def set_line_nom_max(
             add_dynamic_emission_prices(n)
             Ept_config = True
 
-    Ep_config = snakemake.params.costs.get("enable",{}).get("emission_prices", False)
+    Ep_config = snakemake.params.costs.get("enable", {}).get("emission_prices", False)
     Ep_wildcard, co2_wildcard = find_opt(opts, "Ep")
     if (Ep_wildcard or Ep_config) and not Ept_config:
         if co2_wildcard is not None:
             logger.info("Setting emission prices according to wildcard value.")
             add_emission_prices(n, dict(co2=co2_wildcard))
-        else: 
+        else:
             logger.info("Setting emission prices according to config value.")
             add_emission_prices(n, snakemake.params.costs["emission_prices"])
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index af666f5db..a2ffc3da2 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -17,7 +17,7 @@
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import generate_periodic_profiles, update_config_with_sector_opts, get_opt
+from _helpers import generate_periodic_profiles, get_opt, update_config_with_sector_opts
 from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
@@ -161,11 +161,11 @@ def define_spatial(nodes, options):
 
 def emission_sectors_from_opts(opts):
     sectors = ["electricity"]
-    if "T" in opts or opts_config.get("land_transport",False):
+    if "T" in opts or opts_config.get("land_transport", False):
         sectors += ["rail non-elec", "road non-elec"]
-    if "H" in opts or opts_config.get("heating",False):
+    if "H" in opts or opts_config.get("heating", False):
         sectors += ["residential non-elec", "services non-elec"]
-    if "I" in opts or opts_config.get("industry",False):
+    if "I" in opts or opts_config.get("industry", False):
         sectors += [
             "industrial non-elec",
             "industrial processes",
@@ -175,9 +175,13 @@ def emission_sectors_from_opts(opts):
             "international navigation",
         ]
 
-    heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False)
+    heat_and_industry = opts_config.get("industry", False) and opts_config.get(
+        "heating", False
+    )
 
-    if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)):
+    if ("I" in opts and "H" in opts and "A" in opts) or (
+        heat_and_industry and opts_config.get("agriculture_machinery", False)
+    ):
         sectors += ["agriculture"]
 
     return sectors
@@ -3260,21 +3264,25 @@ def set_temporal_aggregation(n, opts, solver_name):
     Aggregate network temporally.
     """
     # temporal averaging
-    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours",{})
-    nhours_enable_config = nhours_opts_config.get("enable",None)
-    nhours_config = str(nhours_opts_config.get("hour",None)) + "H"
+    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {})
+    nhours_enable_config = nhours_opts_config.get("enable", None)
+    nhours_config = str(nhours_opts_config.get("hour", None)) + "H"
     nhours_wildcard = get_opt(opts, r"^\d+h$")
-    if nhours_wildcard is not None or (nhours_enable_config and nhours_config is not None):
+    if nhours_wildcard is not None or (
+        nhours_enable_config and nhours_config is not None
+    ):
         nhours = nhours_wildcard or nhours_config
         n = average_every_nhours(n, nhours)
         return n
 
     # representative snapshots
-    snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots",{})
-    snapshots_enable_config = snapshots_opts_config.get("enable",None)
-    snapshots_config = snapshots_opts_config.get("hour",None)
+    snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots", {})
+    snapshots_enable_config = snapshots_opts_config.get("enable", None)
+    snapshots_config = snapshots_opts_config.get("hour", None)
     snapshots_wildcard = get_opt(opts, r"(^\d+)sn$")
-    if snapshots_wildcard is not None or (snapshots_enable_config and snapshots_config is not None):
+    if snapshots_wildcard is not None or (
+        snapshots_enable_config and snapshots_config is not None
+    ):
         sn = int(snapshots_wildcard[:-2]) or snapshots_config
         logger.info(f"Use every {sn} snapshot as representative")
         n.set_snapshots(n.snapshots[::sn])
@@ -3282,11 +3290,13 @@ def set_temporal_aggregation(n, opts, solver_name):
         return n
 
     # segments with package tsam
-    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation",{})
-    time_seg_enable_config = nhours_opts_config.get("enable",None)
-    time_seg_config = nhours_opts_config.get("hour",None)
+    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {})
+    time_seg_enable_config = nhours_opts_config.get("enable", None)
+    time_seg_config = nhours_opts_config.get("hour", None)
     time_seg_wildcard = get_opt(opts, r"^(\d+)seg$")
-    if time_seg_wildcard is not None or (time_seg_enable_config and time_seg_config is not None):
+    if time_seg_wildcard is not None or (
+        time_seg_enable_config and time_seg_config is not None
+    ):
         segments = int(time_seg_wildcard[:-3]) or time_seg_config
         logger.info(f"Use temporal segmentation with {segments} segments")
         n = apply_time_segmentation(n, segments, solver_name=solver_name)
@@ -3320,7 +3330,9 @@ def set_temporal_aggregation(n, opts, solver_name):
 
     opts_config = snakemake.params.enable_sector
 
-    heat_and_industry = opts_config.get("industry",False) and opts_config.get("heating",False)
+    heat_and_industry = opts_config.get("industry", False) and opts_config.get(
+        "heating", False
+    )
 
     investment_year = int(snakemake.wildcards.planning_horizons[-4:])
 
@@ -3359,53 +3371,57 @@ def set_temporal_aggregation(n, opts, solver_name):
 
     # TODO merge with opts cost adjustment below
     for o in opts:
-        if o[:4] == "wave": # TODO: add config wildcard options or depreciated?
+        if o[:4] == "wave":  # TODO: add config wildcard options or depreciated?
             wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-"))
             logger.info(
                 f"Including wave generators with cost factor of {wave_cost_factor}"
             )
             add_wave(n, wave_cost_factor)
-        if o[:4] == "dist": # TODO: add config wildcard options
+        if o[:4] == "dist":  # TODO: add config wildcard options
             options["electricity_distribution_grid"] = True
             options["electricity_distribution_grid_cost_factor"] = float(
                 o[4:].replace("p", ".").replace("m", "-")
             )
     for o in opts:
-        if o == "biomasstransport" or opts_config.get("biomass_transport",False):
+        if o == "biomasstransport" or opts_config.get("biomass_transport", False):
             options["biomass_transport"] = True
             break
 
-    if "nodistrict" in opts or opts_config.get("no_heat_district",False):
+    if "nodistrict" in opts or opts_config.get("no_heat_district", False):
         options["district_heating"]["progress"] = 0.0
 
-    if "T" in opts or opts_config.get("land_transport",False):
+    if "T" in opts or opts_config.get("land_transport", False):
         add_land_transport(n, costs)
 
-    if "H" in opts or opts_config.get("heating",False):
+    if "H" in opts or opts_config.get("heating", False):
         add_heat(n, costs)
 
-    if "B" in opts or opts_config.get("biomass",False):
+    if "B" in opts or opts_config.get("biomass", False):
         add_biomass(n, costs)
 
     if options["ammonia"]:
         add_ammonia(n, costs)
 
-    if "I" in opts or opts_config.get("industry",False):
+    if "I" in opts or opts_config.get("industry", False):
         add_industry(n, costs)
 
-    if ("I" in opts and "H" in opts) or (heat_and_industry and opts_config.get("waste_heat",False)):
+    if ("I" in opts and "H" in opts) or (
+        heat_and_industry and opts_config.get("waste_heat", False)
+    ):
         add_waste_heat(n)
 
-    if ("I" in opts and "H" in opts and"A" in opts) or (heat_and_industry and opts_config.get("agriculture_machinery",False)):  # requires H and I
+    if ("I" in opts and "H" in opts and "A" in opts) or (
+        heat_and_industry and opts_config.get("agriculture_machinery", False)
+    ):  # requires H and I
         add_agriculture(n, costs)
 
     if options["dac"]:
         add_dac(n, costs)
 
-    if "decentral" in opts or opts_config.get("decentral",False):
+    if "decentral" in opts or opts_config.get("decentral", False):
         decentral(n)
 
-    if "noH2network" in opts or opts_config.get("noH2network",False):
+    if "noH2network" in opts or opts_config.get("noH2network", False):
         remove_h2_network(n)
 
     if options["co2network"]:
@@ -3420,7 +3436,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     limit_type = "config"
     limit = get(snakemake.params.co2_budget, investment_year)
     for o in opts:
-        if "cb" not in o or opts_config.get("carbon_budget",False) is False:
+        if "cb" not in o or opts_config.get("carbon_budget", False) is False:
             continue
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv"
@@ -3440,7 +3456,7 @@ def set_temporal_aggregation(n, opts, solver_name):
         limit = co2_cap.loc[investment_year]
         break
     for o in opts:
-        if "Co2L" not in o or opts_config.get("co2limit_sector",False) is False:
+        if "Co2L" not in o or opts_config.get("co2limit_sector", False) is False:
             continue
         limit_type = "wildcard"
         limit = o[o.find("Co2L") + 4 :]
@@ -3449,7 +3465,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     logger.info(f"Add CO2 limit from {limit_type}")
     add_co2limit(n, nyears, limit)
 
-    for o in opts: # TODO: add config wildcard options or depreciated?
+    for o in opts:  # TODO: add config wildcard options or depreciated?
         if not o[:10] == "linemaxext":
             continue
         maxext = float(o[10:]) * 1e3

From 7be8cc0773380915dccc06c227574528611a2ff9 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Fri, 15 Sep 2023 11:55:52 +0200
Subject: [PATCH 03/15] undo changes in sector networks and simplify config

---
 config/config.default.yaml        |  50 ++-----------
 rules/build_electricity.smk       |  23 +++++-
 rules/build_sector.smk            |  26 +++++--
 rules/postprocess.smk             |   6 +-
 rules/validate.smk                |  18 ++++-
 scripts/prepare_network.py        |  26 +++----
 scripts/prepare_sector_network.py | 119 ++++++++++--------------------
 7 files changed, 114 insertions(+), 154 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index fcc74981b..75de94375 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -59,14 +59,9 @@ snapshots:
   start: "2013-01-01"
   end: "2014-01-01"
   inclusive: 'left'
-
-snapshot_opts:
-  average_every_nhours:
-    enable: false
-    hour: 2
-  time_segmentation:
-    enable: false
-    hour: 4380
+  resolution: false
+  segmentation: false
+  #representative: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable
 enable:
@@ -81,21 +76,6 @@ enable:
   retrieve_natura_raster: true
   custom_busmap: false
 
-enable_sector:
-  no_heat_district: false
-  land_transport: false
-  heating: false
-  waste_heat: false
-  biomass: false
-  biomass_transport: false
-  agriculture_machinery: false
-  industry: false
-  decentral: false
-  #wave_energy: false
-  #wave_energy_factor:
-  noH2network: false
-  carbon_budget: false
-  co2limit_sector: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
 co2_budget:
@@ -107,16 +87,6 @@ co2_budget:
   2045: 0.032
   2050: 0.000
 
-co2_budget_opts:
-  from_descrete_value:
-    enable: true
-  from_beta_decay:
-    enable: false # TODO: move to own rule with sector-opts wildcard?
-    value: 1
-  from_exp_decay:
-    enable: false
-    value: 1
-
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
   voltages: [220., 300., 380.]
@@ -159,13 +129,6 @@ electricity:
       Onshore: [onwind]
       PV: [solar]
 
-  adjust_carrier: # This is the solar+c0.5 thing
-    enable: false
-    #solar:
-    #  p_nom_max:
-    #  capital_cost:
-    #  marginal_cost:
-
   autarky:
     enable: false
     by_country: false
@@ -618,12 +581,9 @@ costs:
     battery: 0.
     battery inverter: 0.
   emission_prices:
+    enable: false
     co2: 0.
-
-  enable:
-    emission_prices: false
-    monthly_prices: false
-
+    co2_monthly_prices: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index b15796c90..bfbc1bd2d 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -20,7 +20,11 @@ if config["enable"].get("prepare_links_p_nom", False):
 
 rule build_electricity_demand:
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         countries=config["countries"],
         load=config["load"],
     input:
@@ -61,7 +65,11 @@ rule build_powerplants:
 rule base_network:
     params:
         countries=config["countries"],
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         lines=config["lines"],
         links=config["links"],
         transformers=config["transformers"],
@@ -144,7 +152,11 @@ if config["enable"].get("build_cutout", False):
 
     rule build_cutout:
         params:
-            snapshots=config["snapshots"],
+            snapshots={
+                "start":config["snapshots"]["start"],
+                "end":config["snapshots"]["end"],
+                "inclusive":config["snapshots"]["inclusive"],
+            },
             cutouts=config["atlite"]["cutouts"],
         input:
             regions_onshore=RESOURCES + "regions_onshore.geojson",
@@ -470,6 +482,10 @@ rule add_extra_components:
 
 rule prepare_network:
     params:
+        snapshots={
+            "resolution":config["snapshots"].get("resolution", False),
+            "segmentation":config["snapshots"].get("segmentation", False),
+        },
         links=config["links"],
         lines=config["lines"],
         co2base=config["electricity"]["co2base"],
@@ -479,7 +495,6 @@ rule prepare_network:
         gaslimit=config["electricity"].get("gaslimit"),
         max_hours=config["electricity"]["max_hours"],
         costs=config["costs"],
-        snapshot_opts=config.get("snapshot_opts", {}),
         autarky=config["electricity"].get("autarky", {}),
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 1bee1ed04..56a6b5326 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -141,7 +141,11 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]):
 
 rule build_heat_demands:
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -163,7 +167,11 @@ rule build_heat_demands:
 
 rule build_temperature_profiles:
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -215,7 +223,11 @@ rule build_cop_profiles:
 
 rule build_solar_thermal_profiles:
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         solar_thermal=config["solar_thermal"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
@@ -677,7 +689,11 @@ rule build_shipping_demand:
 
 rule build_transport_demand:
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         sector=config["sector"],
     input:
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
@@ -705,7 +721,6 @@ rule build_transport_demand:
 
 rule prepare_sector_network:
     params:
-        enable_sector=config.get("enable_sector", {}),
         co2_budget=config["co2_budget"],
         conventional_carriers=config["existing_capacities"]["conventional_carriers"],
         foresight=config["foresight"],
@@ -718,7 +733,6 @@ rule prepare_sector_network:
         countries=config["countries"],
         emissions_scope=config["energy"]["emissions"],
         eurostat_report_year=config["energy"]["eurostat_report_year"],
-        snapshot_opts=config.get("snapshot_opts", {}),
         RDIR=RDIR,
     input:
         **build_retro_cost_output,
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index cf0038a3d..42f0722eb 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -55,7 +55,11 @@ rule make_summary:
     params:
         foresight=config["foresight"],
         costs=config["costs"],
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         scenario=config["scenario"],
         RDIR=RDIR,
     input:
diff --git a/rules/validate.smk b/rules/validate.smk
index cfb8c959a..ece9bebd5 100644
--- a/rules/validate.smk
+++ b/rules/validate.smk
@@ -17,7 +17,11 @@ rule build_electricity_production:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         countries=config["countries"],
     output:
         RESOURCES + "historical_electricity_production.csv",
@@ -35,7 +39,11 @@ rule build_cross_border_flows:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         countries=config["countries"],
     input:
         network=RESOURCES + "networks/base.nc",
@@ -55,7 +63,11 @@ rule build_electricity_prices:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots=config["snapshots"],
+        snapshots={
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+        },
         countries=config["countries"],
     output:
         RESOURCES + "historical_electricity_prices.csv",
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 2a2d73f81..f89db0249 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -312,24 +312,16 @@ def set_line_nom_max(
     set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])
 
     # temporal averaging
-    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {})
-    nhours_enable_config = nhours_opts_config.get("enable", None)
-    nhours_config = str(nhours_opts_config.get("hour", None)) + "h"
+    nhours_config = snakemake.params.snapshots.get("resolution",False)
     nhours_wildcard = get_opt(opts, r"^\d+h$")
-    if nhours_wildcard is not None or (
-        nhours_enable_config and nhours_config is not None
-    ):
+    if nhours_wildcard is not None or isinstance(nhours_config, str):
         nhours = nhours_wildcard or nhours_config
         n = average_every_nhours(n, nhours)
 
     # segments with package tsam
-    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {})
-    time_seg_enable_config = nhours_opts_config.get("enable", None)
-    time_seg_config = str(nhours_opts_config.get("hour", None)) + "seg"
+    time_seg_config = snakemake.params.snapshots.get("segmentation",False)
     time_seg_wildcard = get_opt(opts, r"^\d+seg$")
-    if time_seg_wildcard is not None or (
-        time_seg_enable_config and time_seg_config is not None
-    ):
+    if time_seg_wildcard is not None or isinstance(time_seg_config, str):
         time_seg = time_seg_wildcard or time_seg_config
         solver_name = snakemake.config["solving"]["solver"]["name"]
         n = apply_time_segmentation(n, time_seg, solver_name)
@@ -383,7 +375,7 @@ def set_line_nom_max(
                     sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel, attr] *= factor
 
-    Ept_config = snakemake.params.costs.get("enable", {}).get("monthly_prices", False)
+    Ept_config = snakemake.params.costs["emission_prices"].get("co2_monthly_prices", False)
     for o in opts:
         if "Ept" in o or Ept_config:
             logger.info(
@@ -392,15 +384,15 @@ def set_line_nom_max(
             add_dynamic_emission_prices(n)
             Ept_config = True
 
-    Ep_config = snakemake.params.costs.get("enable", {}).get("emission_prices", False)
+    Ep_config = snakemake.params.costs["emission_prices"].get("enable", False)
     Ep_wildcard, co2_wildcard = find_opt(opts, "Ep")
     if (Ep_wildcard or Ep_config) and not Ept_config:
         if co2_wildcard is not None:
-            logger.info("Setting emission prices according to wildcard value.")
+            logger.info("Setting CO2 prices according to wildcard value.")
             add_emission_prices(n, dict(co2=co2_wildcard))
         else:
-            logger.info("Setting emission prices according to config value.")
-            add_emission_prices(n, snakemake.params.costs["emission_prices"])
+            logger.info("Setting CO2 prices according to config value.")
+            add_emission_prices(n, dict(co2=snakemake.params.costs["emission_prices"]["co2"]))
 
     ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
     set_transmission_limit(n, ll_type, factor, costs, Nyears)
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index a2ffc3da2..11406bffc 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -17,7 +17,7 @@
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import generate_periodic_profiles, get_opt, update_config_with_sector_opts
+from _helpers import generate_periodic_profiles, update_config_with_sector_opts
 from add_electricity import calculate_annuity, sanitize_carriers
 from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2
 from networkx.algorithms import complement
@@ -161,11 +161,11 @@ def define_spatial(nodes, options):
 
 def emission_sectors_from_opts(opts):
     sectors = ["electricity"]
-    if "T" in opts or opts_config.get("land_transport", False):
+    if "T" in opts:
         sectors += ["rail non-elec", "road non-elec"]
-    if "H" in opts or opts_config.get("heating", False):
+    if "H" in opts:
         sectors += ["residential non-elec", "services non-elec"]
-    if "I" in opts or opts_config.get("industry", False):
+    if "I" in opts:
         sectors += [
             "industrial non-elec",
             "industrial processes",
@@ -174,14 +174,7 @@ def emission_sectors_from_opts(opts):
             "domestic navigation",
             "international navigation",
         ]
-
-    heat_and_industry = opts_config.get("industry", False) and opts_config.get(
-        "heating", False
-    )
-
-    if ("I" in opts and "H" in opts and "A" in opts) or (
-        heat_and_industry and opts_config.get("agriculture_machinery", False)
-    ):
+    if "A" in opts:
         sectors += ["agriculture"]
 
     return sectors
@@ -3263,45 +3256,27 @@ def set_temporal_aggregation(n, opts, solver_name):
     """
     Aggregate network temporally.
     """
-    # temporal averaging
-    nhours_opts_config = snakemake.params.snapshot_opts.get("average_every_nhours", {})
-    nhours_enable_config = nhours_opts_config.get("enable", None)
-    nhours_config = str(nhours_opts_config.get("hour", None)) + "H"
-    nhours_wildcard = get_opt(opts, r"^\d+h$")
-    if nhours_wildcard is not None or (
-        nhours_enable_config and nhours_config is not None
-    ):
-        nhours = nhours_wildcard or nhours_config
-        n = average_every_nhours(n, nhours)
-        return n
-
-    # representative snapshots
-    snapshots_opts_config = snakemake.params.snapshot_opts.get("set_snapshots", {})
-    snapshots_enable_config = snapshots_opts_config.get("enable", None)
-    snapshots_config = snapshots_opts_config.get("hour", None)
-    snapshots_wildcard = get_opt(opts, r"(^\d+)sn$")
-    if snapshots_wildcard is not None or (
-        snapshots_enable_config and snapshots_config is not None
-    ):
-        sn = int(snapshots_wildcard[:-2]) or snapshots_config
-        logger.info(f"Use every {sn} snapshot as representative")
-        n.set_snapshots(n.snapshots[::sn])
-        n.snapshot_weightings *= sn
-        return n
-
-    # segments with package tsam
-    time_seg_opts_config = snakemake.params.snapshot_opts.get("time_segmentation", {})
-    time_seg_enable_config = nhours_opts_config.get("enable", None)
-    time_seg_config = nhours_opts_config.get("hour", None)
-    time_seg_wildcard = get_opt(opts, r"^(\d+)seg$")
-    if time_seg_wildcard is not None or (
-        time_seg_enable_config and time_seg_config is not None
-    ):
-        segments = int(time_seg_wildcard[:-3]) or time_seg_config
-        logger.info(f"Use temporal segmentation with {segments} segments")
-        n = apply_time_segmentation(n, segments, solver_name=solver_name)
-        return n
-
+    for o in opts:
+        # temporal averaging
+        m = re.match(r"^\d+h$", o, re.IGNORECASE)
+        if m is not None:
+            n = average_every_nhours(n, m.group(0))
+            break
+        # representative snapshots
+        m = re.match(r"(^\d+)sn$", o, re.IGNORECASE)
+        if m is not None:
+            sn = int(m[1])
+            logger.info(f"Use every {sn} snapshot as representative")
+            n.set_snapshots(n.snapshots[::sn])
+            n.snapshot_weightings *= sn
+            break
+        # segments with package tsam
+        m = re.match(r"^(\d+)seg$", o, re.IGNORECASE)
+        if m is not None:
+            segments = int(m[1])
+            logger.info(f"Use temporal segmentation with {segments} segments")
+            n = apply_time_segmentation(n, segments, solver_name=solver_name)
+            break
     return n
 
 
@@ -3328,12 +3303,6 @@ def set_temporal_aggregation(n, opts, solver_name):
 
     opts = snakemake.wildcards.sector_opts.split("-")
 
-    opts_config = snakemake.params.enable_sector
-
-    heat_and_industry = opts_config.get("industry", False) and opts_config.get(
-        "heating", False
-    )
-
     investment_year = int(snakemake.wildcards.planning_horizons[-4:])
 
     n = pypsa.Network(snakemake.input.network)
@@ -3371,57 +3340,51 @@ def set_temporal_aggregation(n, opts, solver_name):
 
     # TODO merge with opts cost adjustment below
     for o in opts:
-        if o[:4] == "wave":  # TODO: add config wildcard options or depreciated?
+        if o[:4] == "wave":
             wave_cost_factor = float(o[4:].replace("p", ".").replace("m", "-"))
             logger.info(
                 f"Including wave generators with cost factor of {wave_cost_factor}"
             )
             add_wave(n, wave_cost_factor)
-        if o[:4] == "dist":  # TODO: add config wildcard options
+        if o[:4] == "dist":
             options["electricity_distribution_grid"] = True
             options["electricity_distribution_grid_cost_factor"] = float(
                 o[4:].replace("p", ".").replace("m", "-")
             )
-    for o in opts:
-        if o == "biomasstransport" or opts_config.get("biomass_transport", False):
+        if o == "biomasstransport":
             options["biomass_transport"] = True
-            break
 
-    if "nodistrict" in opts or opts_config.get("no_heat_district", False):
+    if "nodistrict" in opts:
         options["district_heating"]["progress"] = 0.0
 
-    if "T" in opts or opts_config.get("land_transport", False):
+    if "T" in opts:
         add_land_transport(n, costs)
 
-    if "H" in opts or opts_config.get("heating", False):
+    if "H" in opts:
         add_heat(n, costs)
 
-    if "B" in opts or opts_config.get("biomass", False):
+    if "B" in opts:
         add_biomass(n, costs)
 
     if options["ammonia"]:
         add_ammonia(n, costs)
 
-    if "I" in opts or opts_config.get("industry", False):
+    if "I" in opts:
         add_industry(n, costs)
 
-    if ("I" in opts and "H" in opts) or (
-        heat_and_industry and opts_config.get("waste_heat", False)
-    ):
+    if "I" in opts and "H" in opts:
         add_waste_heat(n)
 
-    if ("I" in opts and "H" in opts and "A" in opts) or (
-        heat_and_industry and opts_config.get("agriculture_machinery", False)
-    ):  # requires H and I
+    if "A" in opts:  # requires H and I
         add_agriculture(n, costs)
 
     if options["dac"]:
         add_dac(n, costs)
 
-    if "decentral" in opts or opts_config.get("decentral", False):
+    if "decentral" in opts:
         decentral(n)
 
-    if "noH2network" in opts or opts_config.get("noH2network", False):
+    if "noH2network" in opts:
         remove_h2_network(n)
 
     if options["co2network"]:
@@ -3436,7 +3399,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     limit_type = "config"
     limit = get(snakemake.params.co2_budget, investment_year)
     for o in opts:
-        if "cb" not in o or opts_config.get("carbon_budget", False) is False:
+        if "cb" not in o:
             continue
         limit_type = "carbon budget"
         fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv"
@@ -3456,7 +3419,7 @@ def set_temporal_aggregation(n, opts, solver_name):
         limit = co2_cap.loc[investment_year]
         break
     for o in opts:
-        if "Co2L" not in o or opts_config.get("co2limit_sector", False) is False:
+        if "Co2L" not in o:
             continue
         limit_type = "wildcard"
         limit = o[o.find("Co2L") + 4 :]
@@ -3465,7 +3428,7 @@ def set_temporal_aggregation(n, opts, solver_name):
     logger.info(f"Add CO2 limit from {limit_type}")
     add_co2limit(n, nyears, limit)
 
-    for o in opts:  # TODO: add config wildcard options or depreciated?
+    for o in opts:
         if not o[:10] == "linemaxext":
             continue
         maxext = float(o[10:]) * 1e3

From 668ec9efad33af2897484f9bba8dc0c1352603aa Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 15 Sep 2023 09:56:15 +0000
Subject: [PATCH 04/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/build_electricity.smk | 22 +++++++++++-----------
 rules/build_sector.smk      | 24 ++++++++++++------------
 rules/postprocess.smk       |  6 +++---
 rules/validate.smk          | 18 +++++++++---------
 scripts/prepare_network.py  | 12 ++++++++----
 5 files changed, 43 insertions(+), 39 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index bfbc1bd2d..c52404bf9 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -21,9 +21,9 @@ if config["enable"].get("prepare_links_p_nom", False):
 rule build_electricity_demand:
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         countries=config["countries"],
         load=config["load"],
@@ -66,9 +66,9 @@ rule base_network:
     params:
         countries=config["countries"],
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         lines=config["lines"],
         links=config["links"],
@@ -153,9 +153,9 @@ if config["enable"].get("build_cutout", False):
     rule build_cutout:
         params:
             snapshots={
-                "start":config["snapshots"]["start"],
-                "end":config["snapshots"]["end"],
-                "inclusive":config["snapshots"]["inclusive"],
+                "start": config["snapshots"]["start"],
+                "end": config["snapshots"]["end"],
+                "inclusive": config["snapshots"]["inclusive"],
             },
             cutouts=config["atlite"]["cutouts"],
         input:
@@ -483,8 +483,8 @@ rule add_extra_components:
 rule prepare_network:
     params:
         snapshots={
-            "resolution":config["snapshots"].get("resolution", False),
-            "segmentation":config["snapshots"].get("segmentation", False),
+            "resolution": config["snapshots"].get("resolution", False),
+            "segmentation": config["snapshots"].get("segmentation", False),
         },
         links=config["links"],
         lines=config["lines"],
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 56a6b5326..53a19852f 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -142,9 +142,9 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]):
 rule build_heat_demands:
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
@@ -168,9 +168,9 @@ rule build_heat_demands:
 rule build_temperature_profiles:
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
@@ -224,9 +224,9 @@ rule build_cop_profiles:
 rule build_solar_thermal_profiles:
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         solar_thermal=config["solar_thermal"],
     input:
@@ -690,9 +690,9 @@ rule build_shipping_demand:
 rule build_transport_demand:
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         sector=config["sector"],
     input:
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index 42f0722eb..02ac219ae 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -56,9 +56,9 @@ rule make_summary:
         foresight=config["foresight"],
         costs=config["costs"],
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         scenario=config["scenario"],
         RDIR=RDIR,
diff --git a/rules/validate.smk b/rules/validate.smk
index ece9bebd5..1c0fe10a8 100644
--- a/rules/validate.smk
+++ b/rules/validate.smk
@@ -18,9 +18,9 @@ rule build_electricity_production:
     """
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         countries=config["countries"],
     output:
@@ -40,9 +40,9 @@ rule build_cross_border_flows:
     """
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         countries=config["countries"],
     input:
@@ -64,9 +64,9 @@ rule build_electricity_prices:
     """
     params:
         snapshots={
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
+            "start": config["snapshots"]["start"],
+            "end": config["snapshots"]["end"],
+            "inclusive": config["snapshots"]["inclusive"],
         },
         countries=config["countries"],
     output:
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index f89db0249..30ac41bf2 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -312,14 +312,14 @@ def set_line_nom_max(
     set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"])
 
     # temporal averaging
-    nhours_config = snakemake.params.snapshots.get("resolution",False)
+    nhours_config = snakemake.params.snapshots.get("resolution", False)
     nhours_wildcard = get_opt(opts, r"^\d+h$")
     if nhours_wildcard is not None or isinstance(nhours_config, str):
         nhours = nhours_wildcard or nhours_config
         n = average_every_nhours(n, nhours)
 
     # segments with package tsam
-    time_seg_config = snakemake.params.snapshots.get("segmentation",False)
+    time_seg_config = snakemake.params.snapshots.get("segmentation", False)
     time_seg_wildcard = get_opt(opts, r"^\d+seg$")
     if time_seg_wildcard is not None or isinstance(time_seg_config, str):
         time_seg = time_seg_wildcard or time_seg_config
@@ -375,7 +375,9 @@ def set_line_nom_max(
                     sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel, attr] *= factor
 
-    Ept_config = snakemake.params.costs["emission_prices"].get("co2_monthly_prices", False)
+    Ept_config = snakemake.params.costs["emission_prices"].get(
+        "co2_monthly_prices", False
+    )
     for o in opts:
         if "Ept" in o or Ept_config:
             logger.info(
@@ -392,7 +394,9 @@ def set_line_nom_max(
             add_emission_prices(n, dict(co2=co2_wildcard))
         else:
             logger.info("Setting CO2 prices according to config value.")
-            add_emission_prices(n, dict(co2=snakemake.params.costs["emission_prices"]["co2"]))
+            add_emission_prices(
+                n, dict(co2=snakemake.params.costs["emission_prices"]["co2"])
+            )
 
     ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:]
     set_transmission_limit(n, ll_type, factor, costs, Nyears)

From a14e751ed86afc669456cde779c2f82a88cf1230 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Fri, 15 Sep 2023 12:16:53 +0200
Subject: [PATCH 05/15] fix snapshot bugs

---
 scripts/base_network.py             | 7 ++++++-
 scripts/build_line_rating.py        | 8 +++++++-
 scripts/build_renewable_profiles.py | 7 ++++++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/scripts/base_network.py b/scripts/base_network.py
index b4ac1d8c3..f40b0395a 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -709,11 +709,16 @@ def base_network(
     transformers = _set_electrical_parameters_transformers(transformers, config)
     links = _set_electrical_parameters_links(links, config, links_p_nom)
     converters = _set_electrical_parameters_converters(converters, config)
+    snapshots = {
+            "start":config["snapshots"]["start"],
+            "end":config["snapshots"]["end"],
+            "inclusive":config["snapshots"]["inclusive"],
+            }
 
     n = pypsa.Network()
     n.name = "PyPSA-Eur"
 
-    n.set_snapshots(pd.date_range(freq="h", **config["snapshots"]))
+    n.set_snapshots(pd.date_range(freq="h", **snapshots))
     n.madd("Carrier", ["AC", "DC"])
 
     n.import_components_from_dataframe(buses, "Bus")
diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py
index 7f842d437..cf6b56c4d 100755
--- a/scripts/build_line_rating.py
+++ b/scripts/build_line_rating.py
@@ -148,8 +148,14 @@ def calculate_line_rating(n, cutout):
         )
     configure_logging(snakemake)
 
+    snapshots = {
+        "start":snakemake.config["snapshots"]["start"],
+        "end":snakemake.config["snapshots"]["end"],
+        "inclusive":snakemake.config["snapshots"]["inclusive"],
+        }
+
     n = pypsa.Network(snakemake.input.base_network)
-    time = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    time = pd.date_range(freq="h", **snapshots)
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time)
 
     da = calculate_line_rating(n, cutout)
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 7b08325b4..863ba4e4c 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -211,6 +211,11 @@
     correction_factor = params.get("correction_factor", 1.0)
     capacity_per_sqkm = params["capacity_per_sqkm"]
     p_nom_max_meth = params.get("potential", "conservative")
+    snapshots = {
+        "start":snakemake.config["snapshots"]["start"],
+        "end":snakemake.config["snapshots"]["end"],
+        "inclusive":snakemake.config["snapshots"]["inclusive"],
+        }
 
     if isinstance(params.get("corine", {}), list):
         params["corine"] = {"grid_codes": params["corine"]}
@@ -223,7 +228,7 @@
     else:
         client = None
 
-    sns = pd.date_range(freq="h", **snakemake.config["snapshots"])
+    sns = pd.date_range(freq="h", **snapshots)
     cutout = atlite.Cutout(snakemake.input.cutout).sel(time=sns)
     regions = gpd.read_file(snakemake.input.regions)
     assert not regions.empty, (

From b2c19eda40e6409257e616ccc9995ed3063733bd Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 15 Sep 2023 10:18:01 +0000
Subject: [PATCH 06/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/base_network.py             | 8 ++++----
 scripts/build_line_rating.py        | 8 ++++----
 scripts/build_renewable_profiles.py | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/scripts/base_network.py b/scripts/base_network.py
index f40b0395a..a176028fe 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -710,10 +710,10 @@ def base_network(
     links = _set_electrical_parameters_links(links, config, links_p_nom)
     converters = _set_electrical_parameters_converters(converters, config)
     snapshots = {
-            "start":config["snapshots"]["start"],
-            "end":config["snapshots"]["end"],
-            "inclusive":config["snapshots"]["inclusive"],
-            }
+        "start": config["snapshots"]["start"],
+        "end": config["snapshots"]["end"],
+        "inclusive": config["snapshots"]["inclusive"],
+    }
 
     n = pypsa.Network()
     n.name = "PyPSA-Eur"
diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py
index cf6b56c4d..51c779e5d 100755
--- a/scripts/build_line_rating.py
+++ b/scripts/build_line_rating.py
@@ -149,10 +149,10 @@ def calculate_line_rating(n, cutout):
     configure_logging(snakemake)
 
     snapshots = {
-        "start":snakemake.config["snapshots"]["start"],
-        "end":snakemake.config["snapshots"]["end"],
-        "inclusive":snakemake.config["snapshots"]["inclusive"],
-        }
+        "start": snakemake.config["snapshots"]["start"],
+        "end": snakemake.config["snapshots"]["end"],
+        "inclusive": snakemake.config["snapshots"]["inclusive"],
+    }
 
     n = pypsa.Network(snakemake.input.base_network)
     time = pd.date_range(freq="h", **snapshots)
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 863ba4e4c..1d3b9956d 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -212,10 +212,10 @@
     capacity_per_sqkm = params["capacity_per_sqkm"]
     p_nom_max_meth = params.get("potential", "conservative")
     snapshots = {
-        "start":snakemake.config["snapshots"]["start"],
-        "end":snakemake.config["snapshots"]["end"],
-        "inclusive":snakemake.config["snapshots"]["inclusive"],
-        }
+        "start": snakemake.config["snapshots"]["start"],
+        "end": snakemake.config["snapshots"]["end"],
+        "inclusive": snakemake.config["snapshots"]["inclusive"],
+    }
 
     if isinstance(params.get("corine", {}), list):
         params["corine"] = {"grid_codes": params["corine"]}

From 01cd1bbb74a5af95b89f9b492ffd0c6190f3d1dd Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Sat, 23 Sep 2023 18:39:11 +0200
Subject: [PATCH 07/15] add Fabians suggestion

---
 rules/build_electricity.smk         | 21 ++++---------
 rules/build_sector.smk              | 24 +++------------
 rules/postprocess.smk               |  6 +---
 rules/validate.smk                  | 18 ++---------
 scripts/base_network.py             |  6 +---
 scripts/build_line_rating.py        |  6 +---
 scripts/build_renewable_profiles.py |  6 +---
 scripts/prepare_network.py          | 48 ++++++++++++-----------------
 8 files changed, 36 insertions(+), 99 deletions(-)

diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
index c52404bf9..dc5ce1c66 100644
--- a/rules/build_electricity.smk
+++ b/rules/build_electricity.smk
@@ -20,11 +20,7 @@ if config["enable"].get("prepare_links_p_nom", False):
 
 rule build_electricity_demand:
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         countries=config["countries"],
         load=config["load"],
     input:
@@ -65,11 +61,7 @@ rule build_powerplants:
 rule base_network:
     params:
         countries=config["countries"],
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         lines=config["lines"],
         links=config["links"],
         transformers=config["transformers"],
@@ -152,11 +144,7 @@ if config["enable"].get("build_cutout", False):
 
     rule build_cutout:
         params:
-            snapshots={
-                "start": config["snapshots"]["start"],
-                "end": config["snapshots"]["end"],
-                "inclusive": config["snapshots"]["inclusive"],
-            },
+            snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
             cutouts=config["atlite"]["cutouts"],
         input:
             regions_onshore=RESOURCES + "regions_onshore.geojson",
@@ -220,6 +208,7 @@ rule build_ship_raster:
 
 rule build_renewable_profiles:
     params:
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         renewable=config["renewable"],
     input:
         base_network=RESOURCES + "networks/base.nc",
@@ -310,6 +299,8 @@ rule build_hydro_profile:
 if config["lines"]["dynamic_line_rating"]["activate"]:
 
     rule build_line_rating:
+        params:
+            snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         input:
             base_network=RESOURCES + "networks/base.nc",
             cutout="cutouts/"
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index 53a19852f..c148bc9ff 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -141,11 +141,7 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]):
 
 rule build_heat_demands:
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -167,11 +163,7 @@ rule build_heat_demands:
 
 rule build_temperature_profiles:
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -223,11 +215,7 @@ rule build_cop_profiles:
 
 rule build_solar_thermal_profiles:
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         solar_thermal=config["solar_thermal"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
@@ -689,11 +677,7 @@ rule build_shipping_demand:
 
 rule build_transport_demand:
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         sector=config["sector"],
     input:
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
diff --git a/rules/postprocess.smk b/rules/postprocess.smk
index 02ac219ae..795ea1b17 100644
--- a/rules/postprocess.smk
+++ b/rules/postprocess.smk
@@ -55,11 +55,7 @@ rule make_summary:
     params:
         foresight=config["foresight"],
         costs=config["costs"],
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         scenario=config["scenario"],
         RDIR=RDIR,
     input:
diff --git a/rules/validate.smk b/rules/validate.smk
index 1c0fe10a8..0fa1f607c 100644
--- a/rules/validate.smk
+++ b/rules/validate.smk
@@ -17,11 +17,7 @@ rule build_electricity_production:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         countries=config["countries"],
     output:
         RESOURCES + "historical_electricity_production.csv",
@@ -39,11 +35,7 @@ rule build_cross_border_flows:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         countries=config["countries"],
     input:
         network=RESOURCES + "networks/base.nc",
@@ -63,11 +55,7 @@ rule build_electricity_prices:
     The data is used for validation of the optimization results.
     """
     params:
-        snapshots={
-            "start": config["snapshots"]["start"],
-            "end": config["snapshots"]["end"],
-            "inclusive": config["snapshots"]["inclusive"],
-        },
+        snapshots={k: config["snapshots"][k] for k in ["start", "end", "inclusive"]},
         countries=config["countries"],
     output:
         RESOURCES + "historical_electricity_prices.csv",
diff --git a/scripts/base_network.py b/scripts/base_network.py
index a176028fe..372f9a206 100644
--- a/scripts/base_network.py
+++ b/scripts/base_network.py
@@ -709,11 +709,7 @@ def base_network(
     transformers = _set_electrical_parameters_transformers(transformers, config)
     links = _set_electrical_parameters_links(links, config, links_p_nom)
     converters = _set_electrical_parameters_converters(converters, config)
-    snapshots = {
-        "start": config["snapshots"]["start"],
-        "end": config["snapshots"]["end"],
-        "inclusive": config["snapshots"]["inclusive"],
-    }
+    snapshots = snakemake.params.snapshots
 
     n = pypsa.Network()
     n.name = "PyPSA-Eur"
diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py
index 51c779e5d..1767ebc8f 100755
--- a/scripts/build_line_rating.py
+++ b/scripts/build_line_rating.py
@@ -148,11 +148,7 @@ def calculate_line_rating(n, cutout):
         )
     configure_logging(snakemake)
 
-    snapshots = {
-        "start": snakemake.config["snapshots"]["start"],
-        "end": snakemake.config["snapshots"]["end"],
-        "inclusive": snakemake.config["snapshots"]["inclusive"],
-    }
+    snapshots = snakemake.params.snapshots
 
     n = pypsa.Network(snakemake.input.base_network)
     time = pd.date_range(freq="h", **snapshots)
diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py
index 1d3b9956d..a84322199 100644
--- a/scripts/build_renewable_profiles.py
+++ b/scripts/build_renewable_profiles.py
@@ -211,11 +211,7 @@
     correction_factor = params.get("correction_factor", 1.0)
     capacity_per_sqkm = params["capacity_per_sqkm"]
     p_nom_max_meth = params.get("potential", "conservative")
-    snapshots = {
-        "start": snakemake.config["snapshots"]["start"],
-        "end": snakemake.config["snapshots"]["end"],
-        "inclusive": snakemake.config["snapshots"]["inclusive"],
-    }
+    snapshots = snakemake.params.snapshots
 
     if isinstance(params.get("corine", {}), list):
         params["corine"] = {"grid_codes": params["corine"]}
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 30ac41bf2..2a15cb0cb 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -314,26 +314,22 @@ def set_line_nom_max(
     # temporal averaging
     nhours_config = snakemake.params.snapshots.get("resolution", False)
     nhours_wildcard = get_opt(opts, r"^\d+h$")
-    if nhours_wildcard is not None or isinstance(nhours_config, str):
-        nhours = nhours_wildcard or nhours_config
+    nhours = nhours_wildcard or nhours_config
+    if nhours:
         n = average_every_nhours(n, nhours)
 
     # segments with package tsam
     time_seg_config = snakemake.params.snapshots.get("segmentation", False)
     time_seg_wildcard = get_opt(opts, r"^\d+seg$")
-    if time_seg_wildcard is not None or isinstance(time_seg_config, str):
-        time_seg = time_seg_wildcard or time_seg_config
+    time_seg = time_seg_wildcard or time_seg_config
+    if time_seg:
         solver_name = snakemake.config["solving"]["solver"]["name"]
         n = apply_time_segmentation(n, time_seg, solver_name)
 
-    Co2L_config = snakemake.params.co2limit_enable and isinstance(
-        snakemake.params.co2limit, float
-    )
+    Co2L_config = snakemake.params.co2limit_enable
     Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
     if Co2L_wildcard or Co2L_config:
-        if (
-            co2limit_wildcard is not None
-        ):  # TODO: what if you wat to determine the factor through the wildcard?
+        if co2limit_wildcard is not None:
             co2limit = co2limit_wildcard * snakemake.params.co2base
             add_co2limit(n, co2limit, Nyears)
             logger.info("Setting CO2 limit according to wildcard value.")
@@ -341,14 +337,10 @@ def set_line_nom_max(
             add_co2limit(n, snakemake.params.co2limit, Nyears)
             logger.info("Setting CO2 limit according to config value.")
 
-    CH4L_config = snakemake.params.gaslimit_enable and isinstance(
-        snakemake.params.gaslimit, float
-    )
+    CH4L_config = snakemake.params.gaslimit_enable
     CH4L_wildcard, gaslimit_wildcard = find_opt(opts, "CH4L")
     if CH4L_wildcard or CH4L_config:
-        if (
-            gaslimit_wildcard is not None
-        ):  # TODO: what if you wat to determine the factor through the wildcard?
+        if gaslimit_wildcard is not None:
             gaslimit = gaslimit_wildcard * 1e6
             add_gaslimit(n, gaslimit, Nyears)
             logger.info("Setting gas usage limit according to wildcard value.")
@@ -375,20 +367,18 @@ def set_line_nom_max(
                     sel = c.df.carrier.str.contains(carrier)
                     c.df.loc[sel, attr] *= factor
 
-    Ept_config = snakemake.params.costs["emission_prices"].get(
-        "co2_monthly_prices", False
-    )
-    for o in opts:
-        if "Ept" in o or Ept_config:
-            logger.info(
-                "Setting time dependent emission prices according spot market price"
-            )
-            add_dynamic_emission_prices(n)
-            Ept_config = True
-
-    Ep_config = snakemake.params.costs["emission_prices"].get("enable", False)
+    emission_prices = snakemake.params.costs["emission_prices"]
+    Ept_config = emission_prices.get("co2_monthly_prices", False)
+    Ept_wildcard = "Ept" in opts
+    Ep_config = emission_prices.get("enable", False)
     Ep_wildcard, co2_wildcard = find_opt(opts, "Ep")
-    if (Ep_wildcard or Ep_config) and not Ept_config:
+
+    if Ept_wildcard or Ept_config:
+        logger.info(
+            "Setting time dependent emission prices according spot market price"
+        )
+        add_dynamic_emission_prices(n)
+    elif Ep_wildcard or Ep_config:
         if co2_wildcard is not None:
             logger.info("Setting CO2 prices according to wildcard value.")
             add_emission_prices(n, dict(co2=co2_wildcard))

From 05535590348118ef14a17652b5f7f7f8635bfa3b Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Sat, 23 Sep 2023 19:00:01 +0200
Subject: [PATCH 08/15] add documentations

---
 doc/configtables/costs.csv       | 20 +++++++++++---------
 doc/configtables/electricity.csv |  5 +++++
 doc/configtables/snapshots.csv   | 10 ++++++----
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/doc/configtables/costs.csv b/doc/configtables/costs.csv
index 9797d77e4..e307985df 100644
--- a/doc/configtables/costs.csv
+++ b/doc/configtables/costs.csv
@@ -1,9 +1,11 @@
-,Unit,Values,Description
-year,--,"YYYY; e.g. '2030'","Year for which to retrieve cost assumptions of ``resources/costs.csv``."
-version,--,"vX.X.X; e.g. 'v0.5.0'","Version of ``technology-data`` repository to use."
-rooftop_share,--,float,"Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV)."
-fill_values,--,float,"Default values if not specified for a technology in ``resources/costs.csv``."
-capital_cost,EUR/MW,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-marginal_cost,EUR/MWh,"Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.","For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
-emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs."
--- co2,EUR/t,float,"Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``."
+,Unit,Values,Description
+year,--,YYYY; e.g. '2030',Year for which to retrieve cost assumptions of ``resources/costs.csv``.
+version,--,vX.X.X; e.g. 'v0.5.0',Version of ``technology-data`` repository to use.
+rooftop_share,--,float,Share of rooftop PV when calculating capital cost of solar (joint rooftop and utility-scale PV).
+fill_values,--,float,Default values if not specified for a technology in ``resources/costs.csv``.
+capital_cost,EUR/MW,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their capital investment costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+marginal_cost,EUR/MWh,Keys should be in the 'technology' column of ``resources/costs.csv``. Values can be any float.,"For the given technologies, assumptions about their marginal operating costs are set to the corresponding value. Optional; overwrites cost assumptions from ``resources/costs.csv``."
+emission_prices,,,Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs.
+-- enable,bool,true or false,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well)
+-- co2,EUR/t,float,Exogenous price of carbon-dioxide added to the marginal costs of fossil-fuelled generators according to their carbon intensity. Added through the keyword ``Ep`` in the ``{opts}`` wildcard only in the rule :mod:`prepare_network``.
+-- co2_monthly_price,bool,true or false,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``
diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv
index 4c04fee66..00eec0c43 100644
--- a/doc/configtables/electricity.csv
+++ b/doc/configtables/electricity.csv
@@ -1,6 +1,8 @@
 ,Unit,Values,Description
 voltages,kV,"Any subset of {220., 300., 380.}",Voltage levels to consider
+gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
+co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``.
 co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions
 co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard.
 agg_p_nom_limits,file,path,Reference to ``.csv`` file specifying per carrier generator nominal capacity constraints for individual countries if ``'CCL'`` is in ``{opts}`` wildcard. Defaults to ``data/agg_p_nom_minmax.csv``.
@@ -34,3 +36,6 @@ estimate_renewable_capacities,,,
 -- -- Offshore,--,"Any subset of {offwind-ac, offwind-dc}","List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) onshore technology."
 -- -- Offshore,--,{onwind},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) offshore technology."
 -- -- PV,--,{solar},"List of PyPSA-Eur carriers that is considered as (IRENA, OPSD) PV technology."
+autarky,,,
+-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links.
+-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled.
diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv
index d60c78dc0..769e6cb09 100644
--- a/doc/configtables/snapshots.csv
+++ b/doc/configtables/snapshots.csv
@@ -1,4 +1,6 @@
-,Unit,Values,Description
-start,--,"str or datetime-like; e.g. YYYY-MM-DD","Left bound of date range"
-end,--,"str or datetime-like; e.g. YYYY-MM-DD","Right bound of date range"
-inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
+,Unit,Values,Description
+start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range
+end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range
+inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
+resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots
+segmentation,--,"{false,``nSEG``; e.g. ``4380SEG``}","Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load."

From 0d1339b1d3fabebf22cc1f5e3bfe5fcda6677d70 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Wed, 27 Sep 2023 08:22:28 +0200
Subject: [PATCH 09/15] add wildcard options in config for solve network

---
 config/config.default.yaml   |  6 ++++++
 doc/configtables/opts.csv    | 26 +++++++++++++-------------
 doc/configtables/solving.csv |  5 +++++
 scripts/solve_network.py     | 21 ++++++++++++++-------
 4 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 75de94375..620c04af8 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -622,6 +622,12 @@ solving:
     transmission_losses: 0
     linearized_unit_commitment: true
     horizon: 365
+    
+  constraints:
+    CCL: false
+    EQ: false
+    BAU: false
+    SAFE: false
 
   solver:
     name: gurobi
diff --git a/doc/configtables/opts.csv b/doc/configtables/opts.csv
index 8c8a706fb..b133c718d 100644
--- a/doc/configtables/opts.csv
+++ b/doc/configtables/opts.csv
@@ -1,13 +1,13 @@
-Trigger, Description, Definition, Status
-``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
-``nSEG``; e.g. ``4380SEG``, "Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
-``Co2L``, Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
-``Ep``, Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
-``Ept``, Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use
-``CCL``, Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
-``EQ``, "Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.", ``solve_network``, In active use
-``ATK``, "Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links.", ``prepare_network``, In active use
-``BAU``, Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
-``SAFE``, Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
-``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
-``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
+Trigger, Description, Definition, Status
+``nH``; i.e. ``2H``-``6H``, Resample the time-resolution by averaging over every ``n`` snapshots, ``prepare_network``: `average_every_nhours() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L110>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L146>`__), In active use
+``nSEG``; e.g. ``4380SEG``,"Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load.", ``prepare_network``: apply_time_segmentation(), In active use
+``Co2L``,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit``. If a float is appended an overall emission limit relative to the emission level given in ``electricity: co2base`` is added (e.g. ``Co2L0.05`` limits emissisions to 5% of what is given in ``electricity: co2base``), ``prepare_network``: `add_co2limit() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L19>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L154>`__, In active use
+``Ep``,Add cost for a carbon-dioxide price configured in ``costs: emission_prices: co2`` to ``marginal_cost`` of generators (other emission types listed in ``network.carriers`` possible as well), ``prepare_network``: `add_emission_prices() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L24>`_ and its `caller <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/prepare_network.py#L158>`__, In active use
+``Ept``,Add monthly cost for a carbon-dioxide price based on historical values built by the rule ``build_monthly_prices``, In active use,
+``CCL``,Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``., ``solve_network``, In active use
+``EQ``,Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption., ``solve_network``, In active use
+``ATK``,Require each node to be autarkic. Example: ``ATK`` removes all lines and links. ``ATKc`` removes all cross-border lines and links., ``prepare_network``, In active use
+``BAU``,Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``, ``solve_network``: `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L66>`__, Untested
+``SAFE``,Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network., ``solve_network`` `add_opts_constraints() <https://github.com/PyPSA/pypsa-eur/blob/6b964540ed39d44079cdabddee8333f486d0cd63/scripts/solve_network.py#L73>`__, Untested
+``carrier+{c|p|m}factor``,"Alter the capital cost (``c``), installable potential (``p``) or marginal costs (``m``) of a carrier by a factor. Example: ``solar+c0.5`` reduces the capital cost of solar to 50\% of original values.", ``prepare_network``, In active use
+``CH4L``,"Add an overall absolute gas limit. If configured in ``electricity: gaslimit`` it is given in MWh thermal, if a float is appended, the overall gaslimit is assumed to be given in TWh thermal (e.g. ``CH4L200`` limits gas dispatch to 200 TWh termal)", ``prepare_network``: ``add_gaslimit()``, In active use
diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv
index 45d50d844..344bf73f9 100644
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@@ -12,6 +12,11 @@ options,,,
 -- transmission_losses,int,[0-9],"Add piecewise linear approximation of transmission losses based on n tangents. Defaults to 0, which means losses are ignored."
 -- linearized_unit_commitment,bool,"{'true','false'}",Whether to optimise using the linearized unit commitment formulation.
 -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100.
+constraints ,,,
+-- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``.
+-- EQ,bool/string,"{'false',`EQn(c| )``; i.e. ``EQ0.5``-``EQ0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.
+-- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``
+-- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network.
 solver,,,
 -- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow.
 -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings.
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 836544b4b..c1c170ce8 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -33,7 +33,7 @@
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import configure_logging, update_config_with_sector_opts
+from _helpers import configure_logging, update_config_with_sector_opts, get_opt
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
@@ -542,6 +542,7 @@ def add_chp_constraints(n):
     # back-pressure
     if not electric.empty:
         lhs = (
+
             p.loc[:, heat] * (n.links.efficiency[heat] * n.links.c_b[electric].values)
             - p.loc[:, electric] * n.links.efficiency[electric]
         )
@@ -580,18 +581,24 @@ def extra_functionality(n, snapshots):
     """
     opts = n.opts
     config = n.config
-    if "BAU" in opts and n.generators.p_nom_extendable.any():
+    constraints = config["solving"].get("constraints", {})
+    if ("BAU" in opts or constraints.get("BAU",False)) and n.generators.p_nom_extendable.any():
         add_BAU_constraints(n, config)
-    if "SAFE" in opts and n.generators.p_nom_extendable.any():
+    if ("SAFE" in opts or constraints.get("SAFE",False)) and n.generators.p_nom_extendable.any():
         add_SAFE_constraints(n, config)
-    if "CCL" in opts and n.generators.p_nom_extendable.any():
+    if ("CCL" in opts or constraints.get("CCL",False)) and n.generators.p_nom_extendable.any():
         add_CCL_constraints(n, config)
+
     reserve = config["electricity"].get("operational_reserve", {})
     if reserve.get("activate"):
         add_operational_reserve_margin(n, snapshots, config)
-    for o in opts:
-        if "EQ" in o:
-            add_EQ_constraints(n, o)
+
+    EQ_config = constraints.get("EQ",False)
+    EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")
+    EQ_o = EQ_wildcard or EQ_config
+    if EQ_o:
+        add_EQ_constraints(n, EQ_o)
+
     add_battery_constraints(n)
     add_pipe_retrofit_constraint(n)
 

From 1e87cf0eebd447d98676f361829bd05b898013dc Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 27 Sep 2023 06:22:55 +0000
Subject: [PATCH 10/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 config/config.default.yaml |  2 +-
 scripts/solve_network.py   | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 620c04af8..76490e6be 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -622,7 +622,7 @@ solving:
     transmission_losses: 0
     linearized_unit_commitment: true
     horizon: 365
-    
+
   constraints:
     CCL: false
     EQ: false
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index c1c170ce8..ca542bbae 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -33,7 +33,7 @@
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import configure_logging, update_config_with_sector_opts, get_opt
+from _helpers import configure_logging, get_opt, update_config_with_sector_opts
 
 logger = logging.getLogger(__name__)
 pypsa.pf.logger.setLevel(logging.WARNING)
@@ -542,7 +542,6 @@ def add_chp_constraints(n):
     # back-pressure
     if not electric.empty:
         lhs = (
-
             p.loc[:, heat] * (n.links.efficiency[heat] * n.links.c_b[electric].values)
             - p.loc[:, electric] * n.links.efficiency[electric]
         )
@@ -582,18 +581,24 @@ def extra_functionality(n, snapshots):
     opts = n.opts
     config = n.config
     constraints = config["solving"].get("constraints", {})
-    if ("BAU" in opts or constraints.get("BAU",False)) and n.generators.p_nom_extendable.any():
+    if (
+        "BAU" in opts or constraints.get("BAU", False)
+    ) and n.generators.p_nom_extendable.any():
         add_BAU_constraints(n, config)
-    if ("SAFE" in opts or constraints.get("SAFE",False)) and n.generators.p_nom_extendable.any():
+    if (
+        "SAFE" in opts or constraints.get("SAFE", False)
+    ) and n.generators.p_nom_extendable.any():
         add_SAFE_constraints(n, config)
-    if ("CCL" in opts or constraints.get("CCL",False)) and n.generators.p_nom_extendable.any():
+    if (
+        "CCL" in opts or constraints.get("CCL", False)
+    ) and n.generators.p_nom_extendable.any():
         add_CCL_constraints(n, config)
 
     reserve = config["electricity"].get("operational_reserve", {})
     if reserve.get("activate"):
         add_operational_reserve_margin(n, snapshots, config)
 
-    EQ_config = constraints.get("EQ",False)
+    EQ_config = constraints.get("EQ", False)
     EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")
     EQ_o = EQ_wildcard or EQ_config
     if EQ_o:

From a3a7e19b07a51ca2c0a9a8f8de4f9573a6194647 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Thu, 28 Sep 2023 21:11:22 +0200
Subject: [PATCH 11/15] add finishing touches

---
 doc/configtables/snapshots.csv |  2 +-
 doc/configtables/solving.csv   |  2 +-
 scripts/_helpers.py            | 12 ++++++++++++
 scripts/prepare_network.py     | 18 ++----------------
 scripts/solve_network.py       |  4 ++--
 5 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/doc/configtables/snapshots.csv b/doc/configtables/snapshots.csv
index 769e6cb09..4a3e12128 100644
--- a/doc/configtables/snapshots.csv
+++ b/doc/configtables/snapshots.csv
@@ -3,4 +3,4 @@ start,--,str or datetime-like; e.g. YYYY-MM-DD,Left bound of date range
 end,--,str or datetime-like; e.g. YYYY-MM-DD,Right bound of date range
 inclusive,--,"One of {'neither', 'both', ‘left’, ‘right’}","Make the time interval closed to the ``left``, ``right``, or both sides ``both`` or neither side ``None``."
 resolution ,--,"{false,``nH``; i.e. ``2H``-``6H``}",Resample the time-resolution by averaging over every ``n`` snapshots
-segmentation,--,"{false,``nSEG``; e.g. ``4380SEG``}","Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load."
+segmentation,--,"{false,``n``; e.g. ``4380``}","Apply time series segmentation with `tsam <https://tsam.readthedocs.io/en/latest/index.html>`_ package to ``n`` adjacent snapshots of varying lengths based on capacity factors of varying renewables, hydro inflow and load."
diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv
index 344bf73f9..940deba91 100644
--- a/doc/configtables/solving.csv
+++ b/doc/configtables/solving.csv
@@ -14,7 +14,7 @@ options,,,
 -- horizon,--,int,Number of snapshots to consider in each iteration. Defaults to 100.
 constraints ,,,
 -- CCL,bool,"{'true','false'}",Add minimum and maximum levels of generator nominal capacity per carrier for individual countries. These can be specified in the file linked at ``electricity: agg_p_nom_limits`` in the configuration. File defaults to ``data/agg_p_nom_minmax.csv``.
--- EQ,bool/string,"{'false',`EQn(c| )``; i.e. ``EQ0.5``-``EQ0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.
+-- EQ,bool/string,"{'false',`n(c| )``; i.e. ``0.5``-``0.7c``}",Require each country or node to on average produce a minimal share of its total consumption itself. Example: ``EQ0.5c`` demands each country to produce on average at least 50% of its consumption; ``EQ0.5`` demands each node to produce on average at least 50% of its consumption.
 -- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities``
 -- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network.
 solver,,,
diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 01349e08b..559997ad8 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -36,6 +36,18 @@ def get_opt(opts, expr, flags=None):
             return match.group(0)
     return None
 
+def find_opt(opts, expr):
+    """
+    Return if available the float after the expression.
+    """
+    for o in opts:
+        if expr in o:
+            m = re.findall("[0-9]*\.?[0-9]+$", o)
+            if len(m) > 0:
+                return True, float(m[0])
+            else:
+                return True, None
+    return False, None
 
 # Define a context manager to temporarily mute print statements
 @contextlib.contextmanager
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 2a15cb0cb..c91097ba7 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -63,7 +63,7 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, get_opt
+from _helpers import configure_logging, get_opt, find_opt
 from add_electricity import load_costs, update_transmission_costs
 from pypsa.descriptors import expand_series
 
@@ -72,20 +72,6 @@
 logger = logging.getLogger(__name__)
 
 
-def find_opt(opts, expr):
-    """
-    Return if available the float after the expression.
-    """
-    for o in opts:
-        if expr in o:
-            m = re.findall("[0-9]*\.?[0-9]+$", o)
-            if len(m) > 0:
-                return True, float(m[0])
-            else:
-                return True, None
-    return False, None
-
-
 def add_co2limit(n, co2limit, Nyears=1.0):
     n.add(
         "GlobalConstraint",
@@ -320,7 +306,7 @@ def set_line_nom_max(
 
     # segments with package tsam
     time_seg_config = snakemake.params.snapshots.get("segmentation", False)
-    time_seg_wildcard = get_opt(opts, r"^\d+seg$")
+    time_seg_wildcard = get_opt(opts, r"^\d+seg$")[:-3]
     time_seg = time_seg_wildcard or time_seg_config
     if time_seg:
         solver_name = snakemake.config["solving"]["solver"]["name"]
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index ca542bbae..2335c86d5 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -598,8 +598,8 @@ def extra_functionality(n, snapshots):
     if reserve.get("activate"):
         add_operational_reserve_margin(n, snapshots, config)
 
-    EQ_config = constraints.get("EQ", False)
-    EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")
+    EQ_config = constraints.get("EQ",False)
+    EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")[2:]
     EQ_o = EQ_wildcard or EQ_config
     if EQ_o:
         add_EQ_constraints(n, EQ_o)

From d75b0ae8abfea19c1da69085af58003a653ac547 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 28 Sep 2023 19:11:51 +0000
Subject: [PATCH 12/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/_helpers.py        | 2 ++
 scripts/prepare_network.py | 2 +-
 scripts/solve_network.py   | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/_helpers.py b/scripts/_helpers.py
index 559997ad8..a32e8f05f 100644
--- a/scripts/_helpers.py
+++ b/scripts/_helpers.py
@@ -36,6 +36,7 @@ def get_opt(opts, expr, flags=None):
             return match.group(0)
     return None
 
+
 def find_opt(opts, expr):
     """
     Return if available the float after the expression.
@@ -49,6 +50,7 @@ def find_opt(opts, expr):
                 return True, None
     return False, None
 
+
 # Define a context manager to temporarily mute print statements
 @contextlib.contextmanager
 def mute_print():
diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index c91097ba7..7ddf805b0 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -63,7 +63,7 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, get_opt, find_opt
+from _helpers import configure_logging, find_opt, get_opt
 from add_electricity import load_costs, update_transmission_costs
 from pypsa.descriptors import expand_series
 
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 2335c86d5..08bb51d59 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -598,7 +598,7 @@ def extra_functionality(n, snapshots):
     if reserve.get("activate"):
         add_operational_reserve_margin(n, snapshots, config)
 
-    EQ_config = constraints.get("EQ",False)
+    EQ_config = constraints.get("EQ", False)
     EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")[2:]
     EQ_o = EQ_wildcard or EQ_config
     if EQ_o:

From 4ac664b8718d36c87d655d523d8f6263d8c7e358 Mon Sep 17 00:00:00 2001
From: virio-andreyana <arsyan.virio@gmail.com>
Date: Fri, 29 Sep 2023 21:24:29 +0200
Subject: [PATCH 13/15] use .replace to fix type error

---
 scripts/prepare_network.py | 4 ++--
 scripts/solve_network.py   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 7ddf805b0..596684535 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -306,11 +306,11 @@ def set_line_nom_max(
 
     # segments with package tsam
     time_seg_config = snakemake.params.snapshots.get("segmentation", False)
-    time_seg_wildcard = get_opt(opts, r"^\d+seg$")[:-3]
+    time_seg_wildcard = get_opt(opts, r"^\d+seg$")
     time_seg = time_seg_wildcard or time_seg_config
     if time_seg:
         solver_name = snakemake.config["solving"]["solver"]["name"]
-        n = apply_time_segmentation(n, time_seg, solver_name)
+        n = apply_time_segmentation(n, time_seg.replace("seg",""), solver_name)
 
     Co2L_config = snakemake.params.co2limit_enable
     Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 08bb51d59..963a85a9a 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -599,10 +599,10 @@ def extra_functionality(n, snapshots):
         add_operational_reserve_margin(n, snapshots, config)
 
     EQ_config = constraints.get("EQ", False)
-    EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")[2:]
+    EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")
     EQ_o = EQ_wildcard or EQ_config
     if EQ_o:
-        add_EQ_constraints(n, EQ_o)
+        add_EQ_constraints(n, EQ_o.replace("EQ",""))
 
     add_battery_constraints(n)
     add_pipe_retrofit_constraint(n)

From d2f8dc25fd3fa7497d3c3c8b6a7d820f593fc09b Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 29 Sep 2023 19:24:53 +0000
Subject: [PATCH 14/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_network.py | 2 +-
 scripts/solve_network.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py
index 596684535..f2c6556e5 100755
--- a/scripts/prepare_network.py
+++ b/scripts/prepare_network.py
@@ -310,7 +310,7 @@ def set_line_nom_max(
     time_seg = time_seg_wildcard or time_seg_config
     if time_seg:
         solver_name = snakemake.config["solving"]["solver"]["name"]
-        n = apply_time_segmentation(n, time_seg.replace("seg",""), solver_name)
+        n = apply_time_segmentation(n, time_seg.replace("seg", ""), solver_name)
 
     Co2L_config = snakemake.params.co2limit_enable
     Co2L_wildcard, co2limit_wildcard = find_opt(opts, "Co2L")
diff --git a/scripts/solve_network.py b/scripts/solve_network.py
index 963a85a9a..012e440b0 100644
--- a/scripts/solve_network.py
+++ b/scripts/solve_network.py
@@ -602,7 +602,7 @@ def extra_functionality(n, snapshots):
     EQ_wildcard = get_opt(opts, r"^EQ+[0-9]*\.?[0-9]+(c|)")
     EQ_o = EQ_wildcard or EQ_config
     if EQ_o:
-        add_EQ_constraints(n, EQ_o.replace("EQ",""))
+        add_EQ_constraints(n, EQ_o.replace("EQ", ""))
 
     add_battery_constraints(n)
     add_pipe_retrofit_constraint(n)

From 558763e40ce651062449f1b113d3e306bd4ccce1 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 5 Jan 2024 14:32:37 +0100
Subject: [PATCH 15/15] add release notes

---
 config/config.default.yaml | 1 -
 doc/release_notes.rst      | 5 +++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index a5e495392..b9e8dc917 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -90,7 +90,6 @@ co2_budget:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
-  voltages: [220., 300., 380.]
   voltages: [220., 300., 380., 500., 750.]
   gaslimit_enable: false
   gaslimit: false
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 88f05854f..7a045866d 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -10,6 +10,11 @@ Release Notes
 Upcoming Release
 ================
 
+* More wildcard options now have a corresponding config entry. If the wildcard
+  is given, then its value is used. If the wildcard is not given but the options
+  in config are enabled, then the value from config is used. If neither is
+  given, the options are skipped.
+
 * Distinguish between stored and sequestered CO2. Stored CO2 is stored
   overground in tanks and can be used for CCU (e.g. methanolisation).
   Sequestered CO2 is stored underground and can no longer be used for CCU. This