From dda509fc18f40efaf9558c2973449d3857827f5b Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Thu, 8 Feb 2024 21:34:21 +0100 Subject: [PATCH 1/6] sanitize log file name Additionally: - rename execute command - remove unneded md5sum calculation --- src/obr/core/core.py | 24 +++++++++++++----------- src/obr/signac_wrapper/operations.py | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/obr/core/core.py b/src/obr/core/core.py index 582453f5..9bfd61d6 100644 --- a/src/obr/core/core.py +++ b/src/obr/core/core.py @@ -41,7 +41,7 @@ def key_to_path(sign_path: Union[str, Path]) -> str: return str(sign_path).replace(SIGNAC_PATH_TOKEN, PATH_TOKEN) -def logged_execute(cmd, path, doc): +def logged_execute(cmd, path, doc) -> Path: """execute cmd and logs success If cmd is a string, it will be interpreted as shell cmd @@ -51,7 +51,6 @@ def logged_execute(cmd, path, doc): path to log file """ - check_output(["mkdir", "-p", ".obr_store"], cwd=path) d = doc["history"] cmd_str = " ".join(cmd) cmd_str = path_to_key(cmd_str).split() # replace dots in cmd_str with _dot_'s @@ -60,7 +59,6 @@ def logged_execute(cmd, path, doc): else: flags = [] cmd_str = cmd_str[0] - log_path = None try: ret = check_output(cmd, cwd=path, stderr=subprocess.STDOUT).decode("utf-8") log = ret @@ -86,11 +84,16 @@ def logged_execute(cmd, path, doc): state = "failure" timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + log_path = None + + # Only write log files above a certain size + # otherwise the log is stored directly in the job doc if log and len(log) > 1000: - h = hashlib.new("md5") - h.update(log.encode()) - h.hexdigest() - fn = f"{cmd_str}_{timestamp}.log" + # the cmd_str might contain / for example if + # shell scripts are called. Hence we sanitize + # the script name + cmd_str_san = cmd_str.split("/")[-1] + fn = f"{cmd_str_san}_{timestamp}.log" with open(path / fn, "w") as fh: fh.write(log) log = fn @@ -262,7 +265,8 @@ def find_tags(path: Path, tags: list, tag_mapping): yield f"{root}/{file}", campaign, tags -def execute(steps: list[str], job) -> bool: +def execute_shell(steps: list[str], job) -> bool: + """execute a list of shell commands on a job""" path = Path(job.path) / "case" if not steps: return False @@ -278,13 +282,11 @@ def execute(steps: list[str], job) -> bool: continue steps_filt.append(step) - # steps_filt = map(lambda x: " ".join(x.split()), steps_filt) - for step in steps_filt: if not step: continue step = parse_variables(step) - logged_execute(step.split(), path, job.doc) + log_file = logged_execute(step.split(), path, job.doc) return True diff --git a/src/obr/signac_wrapper/operations.py b/src/obr/signac_wrapper/operations.py index e8ad5e38..98638b62 100644 --- a/src/obr/signac_wrapper/operations.py +++ b/src/obr/signac_wrapper/operations.py @@ -13,7 +13,7 @@ from datetime import datetime from .labels import owns_mesh, final, finished -from ..core.core import execute +from ..core.core import execute_shell from obr.OpenFOAM.case import OpenFOAMCase from obr.core.queries import filter_jobs, query_impl, Query @@ -355,7 +355,7 @@ def shell(job: Job, args={}): steps = [f"{k} {v}".replace("_dot_", ".") for k, v in args.items()] else: steps = [args] - execute(steps, job) + execute_shell(steps, job) @generate From d5f8ba4543313bf999cdd47baaaa991518131ac4 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Thu, 8 Feb 2024 21:49:51 +0100 Subject: [PATCH 2/6] fixup, remove unneeded import --- src/obr/core/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/obr/core/core.py b/src/obr/core/core.py index 9bfd61d6..0ab15152 100644 --- a/src/obr/core/core.py +++ b/src/obr/core/core.py @@ -2,7 +2,6 @@ import os import subprocess import re -import hashlib import logging import json From cbe66033c786d7e55874792fda05458afa4aeba8 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Thu, 8 Feb 2024 21:50:18 +0100 Subject: [PATCH 3/6] update changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0fc158d3..2dac1d86 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,7 @@ Changelog - Validate simulation state after runSerial|ParallelSolver, see https://github.com/hpsim/OBR/pull/168 - Improve obr submit, see https://github.com/hpsim/OBR/pull/170 - Fix broken test badge, see https://github.com/hpsim/OBR/pull/178 +- Fix logfile location for shell commands, see https://github.com/hpsim/OBR/pull/184 0.2.0 (2023-09-14) From fcc29676c4bd09fc0c91d292209836f1ea23f854 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Thu, 8 Feb 2024 21:51:33 +0100 Subject: [PATCH 4/6] version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7f854bfa..42b9e4ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "obr" -version = "0.3.6" +version = "0.3.7" description = "A tool to create and run OpenFOAM parameter studies" authors = [ {name = "Gregor Olenik", email = "go@hpsim.de"}, From 0dcea25980534c1d395f97135b15516f85d38dff Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Thu, 8 Feb 2024 21:52:15 +0100 Subject: [PATCH 5/6] remove unneeded variable --- src/obr/core/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/obr/core/core.py b/src/obr/core/core.py index 0ab15152..281d1e9d 100644 --- a/src/obr/core/core.py +++ b/src/obr/core/core.py @@ -285,7 +285,7 @@ def execute_shell(steps: list[str], job) -> bool: if not step: continue step = parse_variables(step) - log_file = logged_execute(step.split(), path, job.doc) + logged_execute(step.split(), path, job.doc) return True From e84fe02cd857b4cfdb7cda54b83f0b337720fca2 Mon Sep 17 00:00:00 2001 From: Gregor Olenik Date: Fri, 9 Feb 2024 07:06:45 +0100 Subject: [PATCH 6/6] get rid of _dot_ in log name --- src/obr/core/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/obr/core/core.py b/src/obr/core/core.py index 281d1e9d..852d42f8 100644 --- a/src/obr/core/core.py +++ b/src/obr/core/core.py @@ -91,7 +91,7 @@ def logged_execute(cmd, path, doc) -> Path: # the cmd_str might contain / for example if # shell scripts are called. Hence we sanitize # the script name - cmd_str_san = cmd_str.split("/")[-1] + cmd_str_san = key_to_path(cmd_str.split("/")[-1]) fn = f"{cmd_str_san}_{timestamp}.log" with open(path / fn, "w") as fh: fh.write(log)