Add pre-commit CI

.pre-commit-config.yaml

@@ -0,0 +1,83 @@
+exclude: "^LICENSES"
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: check-merge-conflict
+  - id: end-of-file-fixer
+  - id: fix-encoding-pragma
+  - id: mixed-line-ending
+  - id: trailing-whitespace
+  - id: check-added-large-files
+    args: ["--maxkb=2000"]
+
+  # Sort package imports alphabetically
+- repo: https://github.com/PyCQA/isort
+  rev: 5.10.1
+  hooks:
+  - id: isort
+    args: ["--profile", "black", "--filter-files"]
+
+  # Convert relative imports to absolute imports
+- repo: https://github.com/MarcoGorelli/absolufy-imports
+  rev: v0.3.1
+  hooks:
+  - id: absolufy-imports
+
+  # Find common spelling mistakes in comments and docstrings
+- repo: https://github.com/codespell-project/codespell
+  rev: v2.2.1
+  hooks:
+  - id: codespell
+    args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom'] # Ignore capital case words, e.g. country codes
+    types_or: [python, rst, markdown]
+    files: ^(scripts|doc)/
+
+  # Make docstrings PEP 257 compliant
+- repo: https://github.com/myint/docformatter
+  rev: v1.5.0
+  hooks:
+  - id: docformatter
+    args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"]
+
+- repo: https://github.com/keewis/blackdoc
+  rev: v0.3.5
+  hooks:
+  - id: blackdoc
+
+  # Formatting with "black" coding style
+- repo: https://github.com/psf/black
+  rev: 22.8.0
+  hooks:
+      # Format Python files
+  - id: black
+      # Format Jupyter Python notebooks
+  - id: black-jupyter
+
+  # Remove output from Jupyter notebooks
+- repo: https://github.com/aflc/pre-commit-jupyter
+  rev: v1.2.1
+  hooks:
+  - id: jupyter-notebook-cleanup
+    args: ["--remove-kernel-metadata"]
+
+  # Do YAML formatting (before the linter checks it for misses)
+- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+  rev: v2.4.0
+  hooks:
+  - id: pretty-format-yaml
+    args: [--autofix, --indent, "2", --preserve-quotes]
+
+  # Format Snakemake rule / workflow files
+- repo: https://github.com/snakemake/snakefmt
+  rev: 0.4.4
+  hooks:
+  - id: snakefmt
+
+  # For cleaning jupyter notebooks
+- repo: https://github.com/aflc/pre-commit-jupyter
+  rev: v1.2.1
+  hooks:
+  - id: jupyter-notebook-cleanup
+    exclude: examples/solve-on-remote.ipynb

Snakefile

@@ -76,7 +76,8 @@ rule retrieve_load_data:
     input: HTTP.remote("data.open-power-system-data.org/time_series/2019-06-05/time_series_60min_singleindex.csv", keep_local=True, static=True)
     output: "data/load_raw.csv"
     resources: mem_mb=5000
-    run: move(input[0], output[0])
+    run:
+        move(input[0], output[0])
 
 
 rule build_load_data:
@@ -170,15 +171,17 @@ if config['enable'].get('retrieve_cutout', True):
         output: "cutouts/" + CDIR + "{cutout}.nc"
         log: "logs/" + CDIR + "retrieve_cutout_{cutout}.log"
         resources: mem_mb=5000
-        run: move(input[0], output[0])
+        run:
+            move(input[0], output[0])
 
 if config['enable'].get('retrieve_cost_data', True):
     rule retrieve_cost_data:
         input: HTTP.remote(f"raw.githubusercontent.com/PyPSA/technology-data/{config['costs']['version']}/outputs/costs_{config['costs']['year']}.csv", keep_local=True)
         output: COSTS
         log: "logs/" + RDIR + "retrieve_cost_data.log"
         resources: mem_mb=5000
-        run: move(input[0], output[0])
+        run:
+            move(input[0], output[0])
 
 if config['enable'].get('build_natura_raster', False):
     rule build_natura_raster:
@@ -196,14 +199,16 @@ if config['enable'].get('retrieve_natura_raster', True):
         input: HTTP.remote("zenodo.org/record/4706686/files/natura.tiff", keep_local=True, static=True)
         output: "resources/" + RDIR + "natura.tiff"
         resources: mem_mb=5000
-        run: move(input[0], output[0])
+        run:
+            move(input[0], output[0])
 
 
 rule retrieve_ship_raster:
     input: HTTP.remote("https://zenodo.org/record/6953563/files/shipdensity_global.zip", keep_local=True, static=True)
     output: "data/shipdensity_global.zip"
     resources: mem_mb=5000
-    run: move(input[0], output[0])
+    run:
+        move(input[0], output[0])
 
 
 rule build_ship_raster:

doc/cloudcomputing.rst

@@ -72,7 +72,7 @@ Step 3 - Installation of Cloud SDK
 
 - Download Google Cloud SDK `SDK <https://cloud.google.com/sdk>`_. Check that you are logged in in your Google account. The link should lead you to the Windows installation of Google Cloud SDK.
 - Follow the "Quickstart for Windows - Before you begin" steps.
-- After the successfull installation and initialization, close the Google Cloud SDK reopen it again. Type the following command into the "Google Cloud SDK Shell":
+- After the successful installation and initialization, close the Google Cloud SDK reopen it again. Type the following command into the "Google Cloud SDK Shell":
 
     .. code:: bash
 
@@ -107,7 +107,7 @@ Make sure that your instance is operating for the next steps.
 - Click on the advanced setting. SSH -> Authentication.
 - Option 1. Click on the Tools button and "Install Public Key into Server..". Somewhere in your folder structure must be a public key. I found it with the following folder syntax on my local windows computer -> :\Users\...\.ssh (there should be a PKK file).
 - Option 2. Click on the Tools button and "Generate new key pair...". Save the private key at a folder you remember and add it to the "private key file" field in WinSCP. Upload the public key to the metadeta of your instance.
-- Click ok and save. Then click Login. If successfull WinSCP will open on the left side your local computer folder structure and on the right side the folder strucutre of your VM. (If you followed Option 2 and its not initially working. Stop your instance, refresh the website, reopen the WinSCP field. Afterwards your your Login should be successfull)
+- Click ok and save. Then click Login. If successful WinSCP will open on the left side your local computer folder structure and on the right side the folder structure of your VM. (If you followed Option 2 and its not initially working. Stop your instance, refresh the website, reopen the WinSCP field. Afterwards your your Login should be successful)
 
 If you had struggle with the above steps, you could also try `this video <https://www.youtube.com/watch?v=lYx1oQkEF0E>`_.
 

doc/contributing.rst

@@ -16,10 +16,20 @@ to our `GitHub repository <https://github.com/PyPSA/PyPSA-Eur>`_.
 * If you start working on a feature in the code, let us know by opening an issue or a draft pull request.
   This helps all of us to keep an overview on what is being done and helps to avoid a situation where we
   are doing the same work twice in parallel.
-* We encourage you to use the `PEP 8 coding style <https://www.python.org/dev/peps/pep-0008/>`_.
+
+For linting, formatting and checking your code contributions
+against our guidelines (e.g. we use `Black <https://github.com/psf/black>`_ as code style
+use `pre-commit <https://pre-commit.com/index.html>`_:
+
+1. Installation ``conda install -c conda-forge pre-commit`` or ``pip install pre-commit``
+2. Usage:
+    * To automatically activate ``pre-commit`` on every ``git commit``: Run ``pre-commit install``
+    * To manually run it: ``pre-commit run --all``
+
+Note that installing `pre-commit` locally is not strictly necessary. If you create a Pull Request the `pre-commit CI` will be triggered automatically and take care of the checks.
 
 For all code contributions we follow the four eyes principle (two person principle), i.e. all suggested code
-including our own are reviewed by a second person before they are incoporated into our repository.
+including our own are reviewed by a second person before they are incorporated into our repository.
 
 If you are unfamiliar with pull requests, the GitHub help pages have a nice `guide <https://help.github.com/en/articles/about-pull-requests>`_.
 

doc/release_notes.rst

@@ -188,7 +188,7 @@ PyPSA-Eur 0.5.0 (27th July 2022)
 
 * Enable parallel computing with new dask version.
 
-* Restore compatibility of ``mock_snakemake`` with lastest Snakemake versions.
+* Restore compatibility of ``mock_snakemake`` with latest Snakemake versions.
 
 * Script ``build_bus_regions``: move voronoi partition from vresutils to script.
 
@@ -277,7 +277,7 @@ PyPSA-Eur 0.4.0 (22th September 2021)
   [`#261 <https://github.com/PyPSA/pypsa-eur/pull/261>`_].
 
 * The tutorial cutout was renamed from ``cutouts/europe-2013-era5.nc`` to
-  ``cutouts/be-03-2013-era5.nc`` to accomodate tutorial and productive
+  ``cutouts/be-03-2013-era5.nc`` to accommodate tutorial and productive
   cutouts side-by-side.
 
 * The flag ``keep_all_available_areas`` in the configuration for renewable
@@ -470,7 +470,7 @@ PyPSA-Eur 0.2.0 (8th June 2020)
 
 * Removed the ``id`` column for custom power plants in ``data/custom_powerplants.csv`` to avoid custom power plants with conflicting ids getting attached to the wrong bus [`#131 <https://github.com/PyPSA/pypsa-eur/pull/131>`_].
 
-* Add option ``renewables: {carrier}: keep_all_available_areas:`` to use all availabe weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed  [`#150 <https://github.com/PyPSA/pypsa-eur/pull/150>`_].
+* Add option ``renewables: {carrier}: keep_all_available_areas:`` to use all available weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed  [`#150 <https://github.com/PyPSA/pypsa-eur/pull/150>`_].
 
 * Added a function ``_helpers.load_network()`` which loads a network with overridden components specified in ``snakemake.config['override_components']`` [`#128 <https://github.com/PyPSA/pypsa-eur/pull/128>`_].
 

doc/tutorial.rst

@@ -119,13 +119,9 @@ clustered down to 6 buses and every 24 hours aggregated to one snapshot. The com
 
 orders ``snakemake`` to run the script ``solve_network`` that produces the solved network and stores it in ``.../pypsa-eur/results/networks`` with the name ``elec_s_6_ec_lcopt_Co2L-24H.nc``:
 
-.. code::
-
-    rule solve_network:
-        input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
-        output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
-        [...]
-        script: "scripts/solve_network.py"
+.. literalinclude:: ../Snakefile
+   :start-at: rule solve_network:
+   :end-before: rule solve_operations_network:
 
 .. until https://github.com/snakemake/snakemake/issues/46 closed
 
@@ -245,7 +241,7 @@ Once the whole worktree is finished, it should show state so in the terminal:
 
 You will notice that many intermediate stages are saved, namely the outputs of each individual ``snakemake`` rule.
 
-You can produce any output file occuring in the ``Snakefile`` by running
+You can produce any output file occurring in the ``Snakefile`` by running
 
 .. code:: bash
 

doc/wildcards.rst

@@ -123,7 +123,7 @@ These cutouts will be stored in a folder specified by ``{cutout}``.
 The ``{technology}`` wildcard
 =============================
 
-The ``{technology}`` wildcard specifies for which renewable energy technology to produce availablity time
+The ``{technology}`` wildcard specifies for which renewable energy technology to produce availability time
 series and potentials using the rule :mod:`build_renewable_profiles`.
 It can take the values ``onwind``, ``offwind-ac``, ``offwind-dc``, and ``solar`` but **not** ``hydro``
 (since hydroelectric plant profiles are created by a different rule).

scripts/add_electricity.py

@@ -582,7 +582,7 @@ def add_nice_carrier_names(n, config):
     extendable_carriers = snakemake.config['electricity']['extendable_carriers']
     if not (set(renewable_carriers) & set(extendable_carriers['Generator'])):
         logger.warning("No renewables found in config entry `extendable_carriers`. "
-                       "In future versions, these have to be explicitely listed. "
+                       "In future versions, these have to be explicitly listed. "
                        "Falling back to all renewables.")
 
     conventional_carriers = snakemake.config["electricity"]["conventional_carriers"]

scripts/cluster_network.py

@@ -286,7 +286,7 @@ def fix_country_assignment_for_hac(n):
                 logger.info(
                     f"overwriting country `{country}` of bus `{disconnected_bus}` "
                     f"to new country `{new_country}`, because it is disconnected "
-                    "from its inital inter-country transmission grid."
+                    "from its initial inter-country transmission grid."
                 )
                 n.buses.at[disconnected_bus, "country"] = new_country
         return n

scripts/make_summary.py

@@ -33,9 +33,9 @@
 Description
 -----------
 
-The following rule can be used to summarize the results in seperate .csv files:
+The following rule can be used to summarize the results in separate .csv files:
 
-.. code::
+.. code:: bash
 
     snakemake results/summaries/elec_s_all_lall_Co2L-3H_all
                                          clusters
@@ -46,7 +46,7 @@
 the line volume/cost cap field can be set to one of the following:
 * ``lv1.25`` for a particular line volume extension by 25%
 * ``lc1.25`` for a line cost extension by 25 %
-* ``lall`` for all evalutated caps
+* ``lall`` for all evaluated caps
 * ``lvall`` for all line volume caps
 * ``lcall`` for all line cost caps
 

scripts/simplify_network.py

@@ -77,7 +77,7 @@
 
 1. Create an equivalent transmission network in which all voltage levels are mapped to the 380 kV level by the function ``simplify_network(...)``.
 
-2. DC only sub-networks that are connected at only two buses to the AC network are reduced to a single representative link in the function ``simplify_links(...)``. The components attached to buses in between are moved to the nearest endpoint. The grid connection cost of offshore wind generators are added to the captial costs of the generator.
+2. DC only sub-networks that are connected at only two buses to the AC network are reduced to a single representative link in the function ``simplify_links(...)``. The components attached to buses in between are moved to the nearest endpoint. The grid connection cost of offshore wind generators are added to the capital costs of the generator.
 
 3. Stub lines and links, i.e. dead-ends of the network, are sequentially removed from the network in the function ``remove_stubs(...)``. Components are moved along.
 
@@ -112,7 +112,7 @@ def simplify_network_to_380(n):
     its voltage level, line type and number of parallel bundles (num_parallel).
 
     Transformers are removed and connected components are moved from their
-    starting bus to their ending bus. The corresponing starting buses are
+    starting bus to their ending bus. The corresponding starting buses are
     removed as well.
     """
     logger.info("Mapping all network lines onto a single 380kV layer")