Add test and option to disable online retrieve rules

config/config.default.yaml

@@ -67,6 +67,7 @@ snapshots:
   inclusive: 'left' # include start, not end
 
 enable:
+  retrieve: auto
   prepare_links_p_nom: false
   retrieve_databundle: true
   retrieve_sector_databundle: true

doc/configtables/enable.csv

@@ -1,4 +1,5 @@
 ,Unit,Values,Description
+enable,str or bool,"{auto, true, false}","Switch to include (true) or exclude (false) the retrieve_* rules of snakemake into the workflow; 'auto' sets true|false based on availability of an internet connection to prevent issues with snakemake failing due to lack of internet connection."
 prepare_links_p_nom,bool,"{true, false}","Switch to retrieve current HVDC projects from `Wikipedia <https://en.wikipedia.org/wiki/List_of_HVDC_projects>`_"
 retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zenodo via the rule :mod:`retrieve_databundle` or whether to keep a custom databundle located in the corresponding folder."
 retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder."

doc/release_notes.rst

@@ -40,6 +40,10 @@ Upcoming Release
   e.g. by setting ``solving: options: transmission_losses: 2`` for an
   approximation with two tangents.
 
+* Added configuration option ``enable: retrieve:`` to control whether data
+  retrieval rules from snakemake are enabled or not. Th default setting ``auto``
+  will automatically detect and enable/disable the rules based on internet connectivity.
+
 
 PyPSA-Eur 0.8.0 (18th March 2023)
 =================================

rules/common.smk

@@ -23,6 +23,22 @@ def memory(w):
         return int(factor * (10000 + 195 * int(w.clusters)))
 
 
+# Check if the workflow has access to the internet by trying to access the HEAD of specified url
+def has_internet_access(url="www.zenodo.org") -> bool:
+    import http.client as http_client
+
+    # based on answer and comments from
+    # https://stackoverflow.com/a/29854274/11318472
+    conn = http_client.HTTPConnection(url, timeout=5)  # need access to zenodo anyway
+    try:
+        conn.request("HEAD", "/")
+        return True
+    except:
+        return False
+    finally:
+        conn.close()
+
+
 def input_eurostat(w):
     # 2016 includes BA, 2017 does not
     report_year = config["energy"]["eurostat_report_year"]

rules/retrieve.smk

@@ -2,7 +2,14 @@
 #
 # SPDX-License-Identifier: MIT
 
-if config["enable"].get("retrieve_databundle", True):
+if config["enable"].get("retrieve", "auto") == "auto":
+    config["enable"]["retrieve"] = has_internet_access()
+
+if config["enable"]["retrieve"] is False:
+    print("Datafile downloads disabled in config[retrieve] or no internet access.")
+
+
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", True):
     datafiles = [
         "ch_cantons.csv",
         "je-e-21.03.02.xls",
@@ -32,7 +39,7 @@ if config["enable"].get("retrieve_databundle", True):
             "../scripts/retrieve_databundle.py"
 
 
-if config["enable"].get("retrieve_cutout", True):
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True):
 
     rule retrieve_cutout:
         input:
@@ -51,7 +58,7 @@ if config["enable"].get("retrieve_cutout", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_cost_data", True):
+if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", True):
 
     rule retrieve_cost_data:
         input:
@@ -73,7 +80,9 @@ if config["enable"].get("retrieve_cost_data", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_natura_raster", True):
+if config["enable"]["retrieve"] and config["enable"].get(
+    "retrieve_natura_raster", True
+):
 
     rule retrieve_natura_raster:
         input:
@@ -93,7 +102,9 @@ if config["enable"].get("retrieve_natura_raster", True):
             move(input[0], output[0])
 
 
-if config["enable"].get("retrieve_sector_databundle", True):
+if config["enable"]["retrieve"] and config["enable"].get(
+    "retrieve_sector_databundle", True
+):
     datafiles = [
         "data/eea/UNFCCC_v23.csv",
         "data/switzerland-sfoe/switzerland-new_format.csv",
@@ -120,7 +131,9 @@ if config["enable"].get("retrieve_sector_databundle", True):
             "../scripts/retrieve_sector_databundle.py"
 
 
-if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
+if config["enable"]["retrieve"] and (
+    config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]
+):
     datafiles = [
         "IGGIELGN_LNGs.geojson",
         "IGGIELGN_BorderPoints.geojson",
@@ -140,37 +153,41 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]:
             "../scripts/retrieve_gas_infrastructure_data.py"
 
 
-rule retrieve_electricity_demand:
-    input:
-        HTTP.remote(
-            "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv",
-            keep_local=True,
-            static=True,
-        ),
-    output:
-        "data/load_raw.csv",
-    log:
-        LOGS + "retrieve_electricity_demand.log",
-    resources:
-        mem_mb=5000,
-    retries: 2
-    run:
-        move(input[0], output[0])
-
-
-rule retrieve_ship_raster:
-    input:
-        HTTP.remote(
-            "https://zenodo.org/record/6953563/files/shipdensity_global.zip",
-            keep_local=True,
-            static=True,
-        ),
-    output:
-        "data/shipdensity_global.zip",
-    log:
-        LOGS + "retrieve_ship_raster.log",
-    resources:
-        mem_mb=5000,
-    retries: 2
-    run:
-        move(input[0], output[0])
+if config["enable"]["retrieve"]:
+
+    rule retrieve_electricity_demand:
+        input:
+            HTTP.remote(
+                "data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv",
+                keep_local=True,
+                static=True,
+            ),
+        output:
+            "data/load_raw.csv",
+        log:
+            LOGS + "retrieve_electricity_demand.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        run:
+            move(input[0], output[0])
+
+
+if config["enable"]["retrieve"]:
+
+    rule retrieve_ship_raster:
+        input:
+            HTTP.remote(
+                "https://zenodo.org/record/6953563/files/shipdensity_global.zip",
+                keep_local=True,
+                static=True,
+            ),
+        output:
+            "data/shipdensity_global.zip",
+        log:
+            LOGS + "retrieve_ship_raster.log",
+        resources:
+            mem_mb=5000,
+        retries: 2
+        run:
+            move(input[0], output[0])