diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0fc158d3..2dac1d86 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,7 @@ Changelog - Validate simulation state after runSerial|ParallelSolver, see https://github.com/hpsim/OBR/pull/168 - Improve obr submit, see https://github.com/hpsim/OBR/pull/170 - Fix broken test badge, see https://github.com/hpsim/OBR/pull/178 +- Fix logfile location for shell commands, see https://github.com/hpsim/OBR/pull/184 0.2.0 (2023-09-14) diff --git a/pyproject.toml b/pyproject.toml index 7f854bfa..42b9e4ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "obr" -version = "0.3.6" +version = "0.3.7" description = "A tool to create and run OpenFOAM parameter studies" authors = [ {name = "Gregor Olenik", email = "go@hpsim.de"}, diff --git a/src/obr/core/core.py b/src/obr/core/core.py index 582453f5..852d42f8 100644 --- a/src/obr/core/core.py +++ b/src/obr/core/core.py @@ -2,7 +2,6 @@ import os import subprocess import re -import hashlib import logging import json @@ -41,7 +40,7 @@ def key_to_path(sign_path: Union[str, Path]) -> str: return str(sign_path).replace(SIGNAC_PATH_TOKEN, PATH_TOKEN) -def logged_execute(cmd, path, doc): +def logged_execute(cmd, path, doc) -> Path: """execute cmd and logs success If cmd is a string, it will be interpreted as shell cmd @@ -51,7 +50,6 @@ def logged_execute(cmd, path, doc): path to log file """ - check_output(["mkdir", "-p", ".obr_store"], cwd=path) d = doc["history"] cmd_str = " ".join(cmd) cmd_str = path_to_key(cmd_str).split() # replace dots in cmd_str with _dot_'s @@ -60,7 +58,6 @@ def logged_execute(cmd, path, doc): else: flags = [] cmd_str = cmd_str[0] - log_path = None try: ret = check_output(cmd, cwd=path, stderr=subprocess.STDOUT).decode("utf-8") log = ret @@ -86,11 +83,16 @@ def logged_execute(cmd, path, doc): state = "failure" timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + log_path = None + + # Only write log files above a certain size + # otherwise the log is stored directly in the job doc if log and len(log) > 1000: - h = hashlib.new("md5") - h.update(log.encode()) - h.hexdigest() - fn = f"{cmd_str}_{timestamp}.log" + # the cmd_str might contain / for example if + # shell scripts are called. Hence we sanitize + # the script name + cmd_str_san = key_to_path(cmd_str.split("/")[-1]) + fn = f"{cmd_str_san}_{timestamp}.log" with open(path / fn, "w") as fh: fh.write(log) log = fn @@ -262,7 +264,8 @@ def find_tags(path: Path, tags: list, tag_mapping): yield f"{root}/{file}", campaign, tags -def execute(steps: list[str], job) -> bool: +def execute_shell(steps: list[str], job) -> bool: + """execute a list of shell commands on a job""" path = Path(job.path) / "case" if not steps: return False @@ -278,8 +281,6 @@ def execute(steps: list[str], job) -> bool: continue steps_filt.append(step) - # steps_filt = map(lambda x: " ".join(x.split()), steps_filt) - for step in steps_filt: if not step: continue diff --git a/src/obr/signac_wrapper/operations.py b/src/obr/signac_wrapper/operations.py index e8ad5e38..98638b62 100644 --- a/src/obr/signac_wrapper/operations.py +++ b/src/obr/signac_wrapper/operations.py @@ -13,7 +13,7 @@ from datetime import datetime from .labels import owns_mesh, final, finished -from ..core.core import execute +from ..core.core import execute_shell from obr.OpenFOAM.case import OpenFOAMCase from obr.core.queries import filter_jobs, query_impl, Query @@ -355,7 +355,7 @@ def shell(job: Job, args={}): steps = [f"{k} {v}".replace("_dot_", ".") for k, v in args.items()] else: steps = [args] - execute(steps, job) + execute_shell(steps, job) @generate