Exclude shared resources master

Snakefile

@@ -24,9 +24,11 @@ run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)
 
-logs = path_provider("logs/", RDIR, run["shared_resources"])
-benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
-resources = path_provider("resources/", RDIR, run["shared_resources"])
+shared_resources = run["shared_resources"]["policy"]
+exclude_from_shared = run["shared_resources"]["exclude"]
+logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
+benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
+resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
 
 CDIR = "" if run["shared_cutouts"] else RDIR
 RESULTS = "results/" + RDIR

config/config.default.yaml

@@ -26,7 +26,9 @@ run:
     enable: false
     file: config/scenarios.yaml
   disable_progressbar: false
-  shared_resources: false
+  shared_resources:
+    policy: false
+    exclude: []
   shared_cutouts: true
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight

config/config.entsoe-all.yaml

@@ -5,7 +5,8 @@
 run:
   name: "entsoe-all"
   disable_progressbar: true
-  shared_resources: false
+  shared_resources:
+    policy: false
   shared_cutouts: true
 
 scenario:

config/test/config.electricity.yaml

@@ -8,7 +8,8 @@ tutorial: true
 run:
   name: "test-elec" # use this to keep track of runs with different settings
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 scenario:

config/test/config.myopic.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-myopic"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 foresight: myopic

config/test/config.overnight.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-overnight"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 

config/test/config.perfect.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-perfect"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 foresight: perfect

config/test/config.scenarios.yaml

@@ -12,7 +12,8 @@ run:
     enable: true
     file: "config/test/scenarios.yaml"
   disable_progressbar: true
-  shared_resources: base
+  shared_resources:
+    policy: base
   shared_cutouts: true
 
 scenario:

doc/configtables/run.csv

@@ -5,5 +5,7 @@ scenarios,,,
 -- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``."
 -- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory."
 disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
-shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
+shared_resources,,,
+-- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
+-- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."

doc/release_notes.rst

@@ -14,6 +14,21 @@ Upcoming Release
 
 * Add floating wind technology for water depths below 60m
 
+* Add config ``run: shared_resources: exclude:`` to specify additional files
+  that should be excluded from shared resources with the setting ``run:
+  shared_resources: base``. The function ``_helpers/get_run_path()`` now takes
+  an additional keyword argument ``exclude_from_shared`` with a list of files
+  that should not be shared. This keyword argument accepts a list of strings
+  where the string only needs to match the start of a filename (e.g.
+  ``"transport_data"`` would exclude both ``transport_data.csv`` and
+  ``transport_data_{simpl}_{clusters}.csv`` from being shared across scenarios.
+
+* Move switch ``run: shared_resources:`` to ``run: shared_resources: policy:``.
+
+* Add config land_transport_demand_factor to model growth in land transport demand for different time horizons.
+
+* Allow dictionary for the config aviation_demand_factor.
+
 * Group existing capacities to the earlier grouping_year for consistency with optimized capacities.
 
 * Update data bundle:

scripts/_helpers.py

@@ -67,7 +67,7 @@ def get_rdir(run):
     return RDIR
 
 
-def get_run_path(fn, dir, rdir, shared_resources):
+def get_run_path(fn, dir, rdir, shared_resources, exclude_from_shared):
     """
     Dynamically provide paths based on shared resources and filename.
 
@@ -87,6 +87,8 @@ def get_run_path(fn, dir, rdir, shared_resources):
         - If string is "base", special handling for shared "base" resources (see notes).
         - If random string other than "base", this folder is used instead of the `rdir` keyword.
         - If boolean, directly specifies if the resource is shared.
+    exclude_from_shared: list
+        List of filenames to exclude from shared resources. Only relevant if shared_resources is "base".
 
     Returns
     -------
@@ -104,10 +106,12 @@ def get_run_path(fn, dir, rdir, shared_resources):
         existing_wildcards = set(re.findall(pattern, fn))
         irrelevant_wildcards = {"technology", "year", "scope", "kind"}
         no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards
-        no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith(
-            "add_electricity"
+        not_shared_rule = (
+            not fn.startswith("networks/elec")
+            and not fn.startswith("add_electricity")
+            and not any(fn.startswith(ex) for ex in exclude_from_shared)
         )
-        is_shared = no_relevant_wildcards and no_elec_rule
+        is_shared = no_relevant_wildcards and not_shared_rule
         rdir = "" if is_shared else rdir
     elif isinstance(shared_resources, str):
         rdir = shared_resources + "/"
@@ -121,7 +125,7 @@ def get_run_path(fn, dir, rdir, shared_resources):
     return f"{dir}{rdir}{fn}"
 
 
-def path_provider(dir, rdir, shared_resources):
+def path_provider(dir, rdir, shared_resources, exclude_from_shared):
     """
     Returns a partial function that dynamically provides paths based on shared
     resources and the filename.
@@ -132,7 +136,13 @@ def path_provider(dir, rdir, shared_resources):
         A partial function that takes a filename as input and
         returns the path to the file based on the shared_resources parameter.
     """
-    return partial(get_run_path, dir=dir, rdir=rdir, shared_resources=shared_resources)
+    return partial(
+        get_run_path,
+        dir=dir,
+        rdir=rdir,
+        shared_resources=shared_resources,
+        exclude_from_shared=exclude_from_shared,
+    )
 
 
 def get_opt(opts, expr, flags=None):