all cmd tests are working, except manyjobs

bin/seamless

@@ -20,9 +20,10 @@ import logging
 os.environ["__SEAMLESS_FRUGAL"] = "1"
 import seamless
 
+from seamless import Checksum, Buffer, CacheMissError
 from seamless.checksum.buffer_cache import buffer_cache
 from seamless.checksum.deserialize import deserialize_sync as deserialize
-from seamless import CacheMissError
+from seamless.util import unchecksum
 from seamless.cmd.message import set_verbosity, message as msg, message_and_exit as err
 from seamless.cmd import parsing
 from seamless.cmd.parsing import get_commands, guess_arguments
@@ -36,7 +37,6 @@ from seamless.cmd import interface
 from seamless.cmd.exceptions import SeamlessSystemExit
 from seamless.cmd.bytes2human import human2bytes
 from seamless.cmd.download import download
-from seamless import Checksum
 from seamless.Environment import Environment
 from seamless.checksum.json import json_dumps
 from seamless.workflow.core.direct.run import fingertip, extract_dunder, tf_get_buffer
@@ -375,7 +375,7 @@ if args.extra_inputs:
         ):
             err(f"--input argument '{inp}': file/directory does not exist")
 
-argtypesstr = json.dumps(argtypes_initial, sort_keys=False, indent=2)
+argtypesstr = json.dumps(unchecksum(argtypes_initial), sort_keys=False, indent=2)
 msg(1, "initial argtypes dict:\n" + argtypesstr)
 
 file_mapping_mode = args.file_mapping_mode
@@ -394,6 +394,7 @@ try:
 except ValueError as exc:
     err(*exc.args)
 
+argtypes = unchecksum(argtypes)
 argtypesstr = json.dumps(argtypes, sort_keys=False, indent=2)
 msg(1, "argtypes dict:\n" + argtypesstr)
 
@@ -734,6 +735,13 @@ else:
 transformation_checksum = Checksum(transformation_checksum).hex()
 msg(3, f"Transformation checksum: {transformation_checksum}")
 
+
+def file_write(handle, buf):
+    if isinstance(buf, Buffer):
+        buf = buf.value
+    handle.write(buf)
+
+
 if args.dry_run:
     if args.write_job is not None:
         os.mkdir(args.write_job)
@@ -751,14 +759,14 @@ if args.dry_run:
                 _f.write(transformation_checksum)
         else:
             with open(os.path.join(args.write_job, "transformation.json"), "wb") as _f:
-                _f.write(transformation_buffer)
+                file_write(_f, transformation_buffer)
             command = buffer_cache.buffer_cache[bytes.fromhex(code_checksum)]
             with open(os.path.join(args.write_job, "command.txt"), "wb") as _f:
-                _f.write(command)
+                file_write(_f, command)
             if env_checksum is not None:
                 env_buffer = buffer_cache.buffer_cache[bytes.fromhex(env_checksum)]
                 with open(os.path.join(args.write_job, "env.json"), "wb") as _f:
-                    _f.write(env_buffer)
+                    file_write(_f, env_buffer)
 
     if args.upload or args.write_job:
         print(transformation_checksum)

seamless/checksum/buffer_cache.py

@@ -189,6 +189,8 @@ def find_missing(self, checksum, buffer, persistent=False):
             if buffer is None:
                 buffer = self.buffer_cache.get(checksum)
             if buffer is not None:
+                if isinstance(buffer, Buffer):
+                    buffer = buffer.value
                 assert isinstance(buffer, bytes)
                 print_debug("Found missing buffer: {}".format(checksum.hex()))
                 if self.missing.pop(checksum):

seamless/checksum/database_client.py

@@ -87,6 +87,7 @@ def send_put_request(self, request: dict | bytes, *, raise_exception=True):
         if isinstance(request, bytes):
             rqbuf = request
         else:
+            request = unchecksum(request)
             rqbuf = json.dumps(request)
         with session.put(url, data=rqbuf) as response:
             if raise_exception and response.status_code != 200:
@@ -232,6 +233,7 @@ def send_get_request(self, request: dict | bytes):
         if isinstance(request, bytes):
             rqbuf = request
         else:
+            request = unchecksum(request)
             rqbuf = json.dumps(request)
         with session.get(url, data=rqbuf) as response:
             if response.status_code == 404:
@@ -262,6 +264,7 @@ async def send_get_request_async(self, request: dict | bytes):
         if isinstance(request, bytes):
             rqbuf = request
         else:
+            request = unchecksum(request)
             rqbuf = json.dumps(request)
         async with session_async.get(url, data=rqbuf) as response:
             if response.status == 404:

seamless/cmd/bash_transformation.py

@@ -184,17 +184,20 @@ def run_transformation(
     transformation_dict_py = unbashify(transformation_dict, {}, {})
     _, transformation_checksum_py = register_transformation_dict(transformation_dict_py)
     result_py = database.get_transformation_result(transformation_checksum_py)
-    if result_py is not None:
+    result_py = Checksum(result_py)
+    if result_py:
         if not undo and (not fingertip or can_read_buffer(result_py)):
             return Checksum(result_py)
     # /for caching
     _, transformation_checksum = register_transformation_dict(transformation_dict)
     if undo:
-        if result_py is None:
+        has_err = False
+        if not result_py:
             try:
                 result = transformation_cache.undo(transformation_checksum)
             except RuntimeError as exc:
                 result = "Cannot undo: " + " ".join(exc.args)
+                has_err = True
         else:
             result_checksum = result_py
             database.contest(transformation_checksum, result_checksum)
@@ -206,10 +209,10 @@ def run_transformation(
             else:
                 result = response
 
-        if isinstance(result, str):
+        if has_err:
             msg(0, result)
             return None
-        elif isinstance(result, bytes):
+        else:
             msg(
                 2,
                 f"Undo transformation {transformation_checksum.hex()} => {result.hex()}",

tests/cmd/cowsay-docker.sh

@@ -1,3 +1,3 @@
 seamless --docker cowsay --undo -c 'cowsay -t "Hello world!"' 
-seamless --docker cowsay -c 'cowsay -t Hello world!' 
+seamless --docker cowsay -c 'cowsay -t "Hello world!"' 
 seamless --docker cowsay --undo -c 'cowsay -t "Hello world!"' 

tests/cmd/fingertip-deepcell.sh

@@ -14,7 +14,7 @@ cs=11cd2207ae82a0ff9d4f90c6f469809444e32351cf38f227086d615e049b76b3
 echo 1
 cat random.txt
 rm -rf random.txt
-seamless-download -c $cs
+seamless-download $cs
 cat $cs
 echo ''
 seamless-delete-buffer ../dummy-bufferfolder/ random.txt.CHECKSUM 2>&1

tests/cmd/headify/headify-workflow.py

@@ -26,7 +26,7 @@
 
 seamless.delegate()
 
-from seamless import Transformer, Context, Cell
+from seamless.workflow import Transformer, Context, Cell
 
 try:
     Transformer.from_canonical_interface(tool="canonical-interface/headify")

tests/cmd/headify/test-run1000.out

@@ -1,17 +1,17 @@
-seamless.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
+seamless.workflow.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
 Output:
 Traceback (most recent call last):
-  File "/home/jovyan/seamless-scripts/run-transformation.py", line 152, in <module>
-    result = seamless.run_transformation(checksum.bytes(), fingertip=fingertip, tf_dunder=dunder, scratch=scratch, manager=manager)
-  File "/seamless/seamless/__init__.py", line 113, in run_transformation
-    return tf_cache.run_transformation(checksum, fingertip=fingertip, scratch=scratch, tf_dunder=tf_dunder, new_event_loop=new_event_loop, manager=manager)
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1270, in run_transformation
+  File "/home/jovyan/seamless-scripts/run-transformation.py", line 182, in <module>
+    result = seamless.run_transformation(
+  File "/seamless/seamless/direct/run_transformation.py", line 47, in run_transformation
+    return tf_cache.run_transformation(
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1464, in run_transformation
     asyncio.get_event_loop().run_until_complete(fut)
   File "/opt/conda/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
     return future.result()
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1215, in run_transformation_async
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1378, in run_transformation_async
     raise self.transformation_exceptions[tf_checksum]
-seamless.core.transformation.SeamlessTransformationError: 
+seamless.workflow.core.transformation.SeamlessTransformationError: 
 Bash transformer exception
 ==========================
 

tests/cmd/headify/test-run2.out

@@ -1,18 +1,18 @@
 seamless: Upload 1 files, total 36.00 B
-seamless.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
+seamless.workflow.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
 Output:
 Traceback (most recent call last):
-  File "/home/jovyan/seamless-scripts/run-transformation.py", line 152, in <module>
-    result = seamless.run_transformation(checksum.bytes(), fingertip=fingertip, tf_dunder=dunder, scratch=scratch, manager=manager)
-  File "/seamless/seamless/__init__.py", line 113, in run_transformation
-    return tf_cache.run_transformation(checksum, fingertip=fingertip, scratch=scratch, tf_dunder=tf_dunder, new_event_loop=new_event_loop, manager=manager)
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1270, in run_transformation
+  File "/home/jovyan/seamless-scripts/run-transformation.py", line 182, in <module>
+    result = seamless.run_transformation(
+  File "/seamless/seamless/direct/run_transformation.py", line 47, in run_transformation
+    return tf_cache.run_transformation(
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1464, in run_transformation
     asyncio.get_event_loop().run_until_complete(fut)
   File "/opt/conda/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
     return future.result()
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1215, in run_transformation_async
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1378, in run_transformation_async
     raise self.transformation_exceptions[tf_checksum]
-seamless.core.transformation.SeamlessTransformationError: 
+seamless.workflow.core.transformation.SeamlessTransformationError: 
 Bash transformer exception
 ==========================
 
@@ -46,20 +46,20 @@ text.txt
 cat: workfiles/text-head.txt: No such file or directory
 
 seamless: Upload 1 files, total 22.00 B
-seamless.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
+seamless.workflow.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
 Output:
 Traceback (most recent call last):
-  File "/home/jovyan/seamless-scripts/run-transformation.py", line 152, in <module>
-    result = seamless.run_transformation(checksum.bytes(), fingertip=fingertip, tf_dunder=dunder, scratch=scratch, manager=manager)
-  File "/seamless/seamless/__init__.py", line 113, in run_transformation
-    return tf_cache.run_transformation(checksum, fingertip=fingertip, scratch=scratch, tf_dunder=tf_dunder, new_event_loop=new_event_loop, manager=manager)
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1270, in run_transformation
+  File "/home/jovyan/seamless-scripts/run-transformation.py", line 182, in <module>
+    result = seamless.run_transformation(
+  File "/seamless/seamless/direct/run_transformation.py", line 47, in run_transformation
+    return tf_cache.run_transformation(
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1464, in run_transformation
     asyncio.get_event_loop().run_until_complete(fut)
   File "/opt/conda/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
     return future.result()
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1215, in run_transformation_async
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1378, in run_transformation_async
     raise self.transformation_exceptions[tf_checksum]
-seamless.core.transformation.SeamlessTransformationError: 
+seamless.workflow.core.transformation.SeamlessTransformationError: 
 Bash transformer exception
 ==========================
 

tests/cmd/headify/test-run3.out

@@ -1,4 +1,4 @@
-seamless: Upload 2 files, total 1.81 kB
+seamless: Upload 2 files, total 1.82 kB
 
 filelist.txt
 numbers.dat
@@ -7,20 +7,20 @@ text.txt
 
 cat: workfiles/text-head.txt: No such file or directory
 
-seamless.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
+seamless.workflow.core.transformation.SeamlessTransformationError: Error 400 from assistant:ERROR: Unknown error
 Output:
 Traceback (most recent call last):
-  File "/home/jovyan/seamless-scripts/run-transformation.py", line 152, in <module>
-    result = seamless.run_transformation(checksum.bytes(), fingertip=fingertip, tf_dunder=dunder, scratch=scratch, manager=manager)
-  File "/seamless/seamless/__init__.py", line 113, in run_transformation
-    return tf_cache.run_transformation(checksum, fingertip=fingertip, scratch=scratch, tf_dunder=tf_dunder, new_event_loop=new_event_loop, manager=manager)
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1270, in run_transformation
+  File "/home/jovyan/seamless-scripts/run-transformation.py", line 182, in <module>
+    result = seamless.run_transformation(
+  File "/seamless/seamless/direct/run_transformation.py", line 47, in run_transformation
+    return tf_cache.run_transformation(
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1464, in run_transformation
     asyncio.get_event_loop().run_until_complete(fut)
   File "/opt/conda/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
     return future.result()
-  File "/seamless/seamless/core/cache/transformation_cache.py", line 1215, in run_transformation_async
+  File "/seamless/seamless/workflow/core/cache/transformation_cache.py", line 1378, in run_transformation_async
     raise self.transformation_exceptions[tf_checksum]
-seamless.core.transformation.SeamlessTransformationError: 
+seamless.workflow.core.transformation.SeamlessTransformationError: 
 Bash transformer exception
 ==========================
 
@@ -40,15 +40,15 @@ partial-interface/headify workfiles/filelist.txt -n 3 --batch
 * Standard output
 *************************************************
 Traceback (most recent call last):
-  File "/tmp/seamless-bash-transformernchokduy/partial-interface/headify", line 58, in <module>
+  File "/tmp/seamless-bash-transformerewwisfvp/partial-interface/headify", line 58, in <module>
     headify(input_file, args.lines, output_file)
-  File "/tmp/seamless-bash-transformernchokduy/partial-interface/headify", line 39, in headify
+  File "/tmp/seamless-bash-transformerewwisfvp/partial-interface/headify", line 39, in headify
     with open(input_file, "r") as inpf:
 FileNotFoundError: [Errno 2] No such file or directory: 'workfiles/numbers.dat'
 
 *************************************************
 *************************************************
-Execution time: 0.1 seconds
+Execution time: 0.0 seconds
 
 
 buffer cache, 8 buffers undestroyed

tests/cmd/headify/test.out

@@ -8,21 +8,21 @@ Run 4 (seamless, inferior interface)...
 
 Run 5 (seamless, inferior interface, 2x5 sec sleep)...
 
-real	0m17.983s
-user	0m5.098s
-sys	0m0.317s
+real	0m14.213s
+user	0m3.539s
+sys	0m0.129s
 
 Run 5a (seamless, inferior interface, 2x5 sec sleep, repeat)...
 
-real	0m4.743s
-user	0m4.649s
-sys	0m0.286s
+real	0m3.185s
+user	0m3.338s
+sys	0m0.160s
 
 Run 6 (seamless, inferior interface, 2x5 sec sleep, rename)...
 
-real	0m16.988s
-user	0m5.030s
-sys	0m0.262s
+real	0m13.280s
+user	0m3.338s
+sys	0m0.186s
 
 Run 999 (seamless, inferior interface, multi-command)...
 
@@ -33,23 +33,23 @@ FAILS because canonical interface is for single-command only
 
 Run 8 (seamless, canonical interface, 2x5 sec sleep)...
 
-real	0m19.236s
-user	0m5.475s
-sys	0m0.292s
+real	0m16.559s
+user	0m4.369s
+sys	0m0.268s
 
 Run 8a (seamless, canonical interface, 2x5 sec sleep, repeat)...
 
-real	0m4.970s
-user	0m4.993s
-sys	0m0.266s
+real	0m3.943s
+user	0m3.889s
+sys	0m0.206s
 
 Run 9 (seamless, inferior interface, 2x5 sec sleep, rename)...
 Time should be around 5+2 secs instead of 10+2
 The first command of run() is now canonical and federates upon rename
 
-real	0m10.931s
-user	0m4.948s
-sys	0m0.323s
+real	0m8.125s
+user	0m3.393s
+sys	0m0.117s
 
 headify workflow version: should federate and return immediately
 ValueError: Multiple canonical commands available.
@@ -59,10 +59,9 @@ Please choose one and provide it as the "command" argument.
     headify --lines $lines infile
     headify infile -n $lines --sleep $sleep                         
 
-Waiting for background tasks
 Waiting for: Seamless transformer: .tf.tf
 Waiting for: Seamless transformer: .tf.tf
 Waiting for: Seamless transformer: .tf.tf
 Status: OK
 None
-7.4 seconds elapsed
+6.1 seconds elapsed

tests/cmd/multi-core-transformations.sh

@@ -4,7 +4,7 @@ random=$RANDOM
 seamless --direct-print --ncores 5 -c \
   "python multi-core-transformations.py 1 5 $random > tf" & 
 t1=$!
-sleep 0.5
+sleep 1
 random=$RANDOM
 seamless --direct-print --ncores 1 -c \
   "python multi-core-transformations.py 2 2 $random > tf2" &