v0.10.0

README.rst

@@ -53,7 +53,10 @@ A simple script could look like this:
 
    import slurminade
 
-   slurminade.update_default_configuration(partition="alg")  # global options for slurm
+   slurminade.update_default_configuration(
+      partition="alg",
+      exclusive=True
+      )  # global options for slurm
 
    # If no slurm environment is found, the functions are called directly to make scripts
    # compatible with any environment.
@@ -84,26 +87,25 @@ A simple script could look like this:
 
 
    if __name__ == "__main__":
-       jid = prepare.distribute()
+      prepare.distribute()
+      slurminade.join()  # make sure that no job runs before prepare has finished
+      with slurminade.JobBundling(max_size=20):  # automatically bundles up to 20 tasks
+         # run 100x f after `prepare` has finished
+         for i in range(100):
+            f.distribute(i)
 
-       with slurminade.Batch(max_size=20) as batch:  # automatically bundles up to 20 tasks
-           # run 100x f after `prepare` has finished
-           for i in range(100):
-               f.wait_for(jid).distribute(i)  # no job id while in batch!
-
-           # clean up after the previous jobs have finished
-           jids = batch.flush()  # flush returns a list with all job ids.
-           clean_up.wait_for(jids).distribute()
+      slurminade.join()  # make sure that the clean up jobs runs after all f-jobs have finished
+      clean_up.distribute()
 
 If slurm is not available, ``distribute`` results in a local function
 call. Analogous for ``srun`` and ``sbatch`` (giving some extra value on
 top of just forwarding to *simple_slurm*).
 
 .. warning::
-   Always use ``Batch`` when distributing many small tasks to few nodes. Slurm
+   Always use ``JobBundling`` when distributing many small tasks to few nodes. Slurm
    jobs have a certain overhead and you do not want to spam your
    infrastructure with too many jobs. However, function calls
-   joined by ``Batch`` are considered as a single job by slurm, thus,
+   joined by ``JobBundling`` are considered as a single job by slurm, thus,
    not shared across nodes.
 
 **What are the limitations of slurminade?** Slurminade reconstructs the
@@ -343,7 +345,7 @@ Project structure
 
 The project is reasonably easy:
 
--  batch.py: Contains code for bundling tasks, so we don’t spam slurm
+-  bundling.py: Contains code for bundling tasks, so we don’t spam slurm
    with too many.
 -  conf.py: Contains code for managing the configuration of slurm.
 -  dispatcher.py: Contains code for actually dispatching tasks to slurm.
@@ -358,6 +360,7 @@ The project is reasonably easy:
 Changes
 -------
 
+-  0.10.0: `Batch` is now named `JobBundling`. There is a method `join` for easier synchronization. `exec` allows to executed commands just like `srun` and `sbatch`, but uniform syntax with other slurmified functions. Functions can now also be called with `distribute_and_wait`. If you call `python3 -m slurminade.check --partition YOUR_PARTITION --constraint YOUR_CONSTRAINT` you can check if your slurm configuration is running correctly.
 -  0.9.0: Lots of improvements.
 -  0.8.1: Bugfix and automatic detection of wrong usage when using ``Batch`` with ``wait_for``.
 -  0.8.0: Added extensive logging and improved typing.

examples/example_1.py

@@ -1,7 +1,7 @@
 import datetime
 
 import slurminade
-from slurminade import Batch
+from slurminade import JobBundling
 
 slurminade.update_default_configuration(partition="alg", constraint="alggen02")
 
@@ -15,6 +15,6 @@ def f(hello_world):
 
 if __name__ == "__main__":
     jid = f.distribute(f"Hello World from slurminade! {datetime.datetime.now()!s}")
-    with Batch(20) as batch:
+    with JobBundling(20) as batch:
         f.distribute("hello 1!")
         f.distribute("hello 2!")

examples/example_3.py

@@ -23,7 +23,7 @@ def clean_up():
 if __name__ == "__main__":
     jid = prepare.distribute()
 
-    with slurminade.Batch(max_size=20) as batch:  # automatically bundles up to 20 tasks
+    with slurminade.JobBundling(max_size=20) as batch:  # automatically bundles up to 20 tasks
         # run 10x f after prepare has finished
         for i in range(100):
             f.wait_for(jid).distribute(i)

pyproject.toml

@@ -7,7 +7,7 @@ where = ["src"]
 
 [project]
 name = "slurminade"
-version = "1.0.0"
+version = "0.10.0"
 authors = [
     { name = "TU Braunschweig, IBR, Algorithms Group (Dominik Krupke)", email = "krupke@ibr.cs.tu-bs.de" },
 ]

src/slurminade/__init__.py

@@ -62,7 +62,7 @@ def clean_up():
     allow_recursive_distribution,
     disable_warning_on_repeated_flushes,
 )
-from .batch import Batch
+from .bundling import JobBundling
 from .dispatcher import (
     srun,
     sbatch,
@@ -83,7 +83,7 @@ def clean_up():
     "set_dispatch_limit",
     "allow_recursive_distribution",
     "disable_warning_on_repeated_flushes",
-    "Batch",
+    "JobBundling",
     "srun",
     "join",
     "sbatch",

src/slurminade/dispatcher.py

@@ -231,7 +231,9 @@ def _dispatch(
         logging.getLogger("slurminade").debug(command)
         if block:
             ret = slurm.srun(command)
-            logging.getLogger("slurminade").info("Returned from srun with exit code %s", ret)
+            logging.getLogger("slurminade").info(
+                "Returned from srun with exit code %s", ret
+            )
             return None
         jid = slurm.sbatch(command)
         self._all_job_ids.append(jid)
@@ -433,7 +435,11 @@ def srun(
         if conf is None:
             conf = {}
         conf = SlurmOptions(**conf)
-    command = command if isinstance(command, str) else " ".join((shlex.quote(c) for c in command))
+    command = (
+        command
+        if isinstance(command, str)
+        else " ".join(shlex.quote(c) for c in command)
+    )
     return get_dispatcher().srun(command, conf, simple_slurm_kwargs)
 
 
@@ -453,7 +459,11 @@ def sbatch(
         if conf is None:
             conf = {}
         conf = SlurmOptions(**conf)
-    command = command if isinstance(command, str) else " ".join((shlex.quote(c) for c in command))
+    command = (
+        command
+        if isinstance(command, str)
+        else " ".join(shlex.quote(c) for c in command)
+    )
     return get_dispatcher().sbatch(command, conf, simple_slurm_kwargs)
 
 
@@ -463,6 +473,3 @@ def join():
     :return: None
     """
     get_dispatcher().join()
-
-
-

src/slurminade/function.py

@@ -1,7 +1,7 @@
 import inspect
+import subprocess
 import typing
 from enum import Enum
-import subprocess
 
 from .dispatcher import FunctionCall, dispatch, get_dispatcher
 from .function_map import FunctionMap
@@ -191,11 +191,10 @@ def dec(func) -> SlurmFunction:
 
     return dec
 
-
 @slurmify()
 def exec(cmd: typing.Union[str, typing.List[str]]):
     """
     Execute a command.
     :param cmd: The command to be executed.
     """
-    subprocess.run(cmd, check=True)
+    subprocess.run(cmd, check=True)

tests/test_dispatch_guard.py

@@ -29,7 +29,7 @@ def test_dispatch_limit(self):
     def test_dispatch_limit_batch(self):
         slurminade.set_entry_point(__file__)
         set_dispatch_limit(2)
-        with slurminade.Batch(max_size=2):
+        with slurminade.JobBundling(max_size=2):
             for _ in range(4):
                 f.distribute()
         self.assertRaises(TooManyDispatchesError, f.distribute)