From 50425dd536d2263045cd79cbfb66389150d657ba Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 18 Oct 2020 17:41:37 -1000 Subject: [PATCH 1/8] prototype --- WDL/_grammar.py | 8 ++-- WDL/_parser.py | 5 ++- WDL/runtime/task.py | 6 ++- WDL/runtime/task_container.py | 70 +++++++++++++++++++++++++++++++---- stubs/docker/__init__.py | 18 ++++++++- tests/test_7runner.py | 29 +++++++++++++++ 6 files changed, 122 insertions(+), 14 deletions(-) diff --git a/WDL/_grammar.py b/WDL/_grammar.py index fc96b707..0b1bca3a 100644 --- a/WDL/_grammar.py +++ b/WDL/_grammar.py @@ -323,7 +323,7 @@ // WDL task commands: with {} and <<< >>> command and ${} and ~{} placeholder styles placeholder: expr -?command: command1 | command2 +?command: "command" (command1 | command2) // meta/parameter_meta sections (effectively JSON) meta_object: "{" [meta_kv (","? meta_kv)*] "}" @@ -335,7 +335,7 @@ // task runtime section (key-expression pairs) runtime_section: "runtime" "{" [runtime_kv (","? runtime_kv)*] "}" -runtime_kv: CNAME ":" expr +runtime_kv: CNAME ":" (expr | command2) /////////////////////////////////////////////////////////////////////////////////////////////////// // decl @@ -451,11 +451,11 @@ COMMAND1_CHAR: /[^~$}]/ | /\$[^{$~]/ | /~[^{$~]/ COMMAND1_FRAGMENT: COMMAND1_CHAR+ -command1: "command" "{" (COMMAND1_FRAGMENT? /\$/* /\~/* _EITHER_DELIM placeholder "}")* COMMAND1_FRAGMENT? /\$/* /\~/* "}" -> command +command1: "{" (COMMAND1_FRAGMENT? /\$/* /\~/* _EITHER_DELIM placeholder "}")* COMMAND1_FRAGMENT? /\$/* /\~/* "}" -> command COMMAND2_CHAR: /[^~>]/ | /~[^{~]/ | />[^>]/ | />>[^>]/ COMMAND2_FRAGMENT: COMMAND2_CHAR+ -command2: "command" "<<<" (COMMAND2_FRAGMENT? /\~/? "~{" placeholder "}")* COMMAND2_FRAGMENT? /\~/* ">>>" -> command +command2: "<<<" (COMMAND2_FRAGMENT? /\~/? "~{" placeholder "}")* COMMAND2_FRAGMENT? /\~/* ">>>" -> command CNAME: /[a-zA-Z][a-zA-Z0-9_]*/ diff --git a/WDL/_parser.py b/WDL/_parser.py index df270dc3..cab2902b 100644 --- a/WDL/_parser.py +++ b/WDL/_parser.py @@ -340,7 +340,10 @@ def meta_section(self, items, meta): return d def runtime_kv(self, items, meta): - return (items[0].value, items[1]) + expr = items[1] + if isinstance(expr, dict) and "command" in expr: # command2 (e.g. inlineDockerfile) + expr = expr["command"] + return (items[0].value, expr) def runtime_section(self, items, meta): d = dict() diff --git a/WDL/runtime/task.py b/WDL/runtime/task.py index 451bd8a8..4ee036d3 100644 --- a/WDL/runtime/task.py +++ b/WDL/runtime/task.py @@ -419,7 +419,11 @@ def _eval_task_runtime( logger.debug(_("runtime values", **dict((key, str(v)) for key, v in runtime_values.items()))) ans = {} - if "docker" in runtime_values: + if "inlineDockerfile" in runtime_values: + ans["inlineDockerfile"] = _util.strip_leading_whitespace( + runtime_values["inlineDockerfile"].coerce(Type.String()).value + )[1].strip() + elif "docker" in runtime_values: ans["docker"] = runtime_values["docker"].coerce(Type.String()).value host_limits = container.__class__.detect_resource_limits(cfg, logger) diff --git a/WDL/runtime/task_container.py b/WDL/runtime/task_container.py index 72eee933..f1c6d93d 100644 --- a/WDL/runtime/task_container.py +++ b/WDL/runtime/task_container.py @@ -16,6 +16,7 @@ import stat from typing import Callable, Iterable, List, Set, Tuple, Type, Any, Dict, Optional from abc import ABC, abstractmethod +from io import BytesIO import docker from .. import Error from .._util import TerminationSignalFlag, path_really_within, chmod_R_plus, PygtailLogger @@ -486,17 +487,22 @@ def _run(self, logger: logging.Logger, terminating: Callable[[], bool], command: outfile.write(command) # prepare docker configuration - image_tag = self.runtime_values.get("docker", "ubuntu:18.04") + resources, user, groups = self.misc_config(logger) + mounts = self.prepare_mounts(logger) + + # connect to dockerd + client = docker.from_env(version="auto", timeout=900) + + # figure desired image, building inlineDockerfile if called for + if "inlineDockerfile" in self.runtime_values: + image_tag = self.build_inline_dockerfile(logger.getChild("inlineDockerfile"), client) + else: + image_tag = self.runtime_values.get("docker", "ubuntu:18.04") if ":" not in image_tag: # seems we need to do this explicitly under some configurations -- issue #232 image_tag += ":latest" logger.info(_("docker image", tag=image_tag)) - mounts = self.prepare_mounts(logger) - - # connect to dockerd - client = docker.from_env(version="auto", timeout=900) - resources, user, groups = self.misc_config(logger, client) svc = None exit_code = None try: @@ -677,7 +683,7 @@ def escape(s): return mounts def misc_config( - self, logger: logging.Logger, client: docker.DockerClient + self, logger: logging.Logger ) -> Tuple[Optional[Dict[str, str]], Optional[str], List[str]]: resources = {} cpu = self.runtime_values.get("cpu", 0) @@ -854,3 +860,53 @@ def unique_service_name(self, run_id: str) -> str: junk = base64.b32encode(junk).decode().lower() assert len(junk) == 24 return f"wdl-{run_id[:34]}-{junk}" # 4 + 34 + 1 + 24 = 63 + + def build_inline_dockerfile( + self, + logger: logging.Logger, + client: docker.DockerClient, + tries: Optional[int] = None, + ) -> str: + dockerfile_utf8 = self.runtime_values["inlineDockerfile"].encode("utf8") + dockerfile_sha256 = hashlib.sha256(dockerfile_utf8).hexdigest() + # TODO: check maximum tag length + tag = "miniwdl_inline_" + if "-" in self.run_id: + tag += self.run_id.split("-")[1].lower() + else: + tag += self.run_id.lower() + tag += ":" + dockerfile_sha256 + build_logfile = os.path.join(self.host_dir, "inlineDockerfile.log.txt") + + def write_log(stream: Iterable[Dict[str, str]]): + # tee the log messages to logger.verbose and build_logfile + with open(build_logfile, "w") as outfile: + for d in stream: + if "stream" in d: + msg = d["stream"].strip() + if msg: + logger.verbose(msg) + print(msg, file=outfile) + + logger.info(_("starting docker build", tag=tag)) + logger.debug(_("Dockerfile", txt=self.runtime_values["inlineDockerfile"])) + try: + image, build_log = client.images.build(fileobj=BytesIO(dockerfile_utf8), tag=tag) + except docker.errors.BuildError as exn: + if isinstance(tries, int): + tries -= 1 + else: + tries = self.runtime_values.get("maxRetries", 0) + if tries > 0: + logger.error( + _("failed docker build will be retried", tries_remaining=tries, msg=exn.msg) + ) + return self.build_inline_dockerfile(logger, client, tries=tries) + else: + write_log(exn.build_log) + logger.error(_("docker build failed", msg=exn.msg, log=build_logfile)) + raise Error.RuntimeError("inlineDockerfile build failed") + + write_log(build_log) + logger.notice(_("docker build", tag=tag, log=build_logfile)) # pyre-ignore + return tag diff --git a/stubs/docker/__init__.py b/stubs/docker/__init__.py index 9d3aa129..3563a0f1 100644 --- a/stubs/docker/__init__.py +++ b/stubs/docker/__init__.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, List, Iterable +from typing import Dict, Any, List, Iterable, Tuple class Container: @property @@ -22,6 +22,13 @@ class Containers: def run(self, image_tag: str, **kwargs) -> Container: ... +class Images: + def build(self, **kwargs) -> Tuple[Image, Iterable[Dict[str,str]]]: + ... + +class Image: + ... + class Node: attrs: Dict[str,Any] @@ -80,11 +87,20 @@ class Mount: def __init__(self, *args, **kwargs): ... +class errors: + class BuildError(Exception): + msg : str + build_log : Iterable[Dict[str,str]] + class DockerClient: @property def containers(self) -> Containers: ... + @property + def images(self) -> Images: + ... + def close(self) -> None: ... diff --git a/tests/test_7runner.py b/tests/test_7runner.py index 9d6b88c6..a3010e91 100644 --- a/tests/test_7runner.py +++ b/tests/test_7runner.py @@ -673,3 +673,32 @@ def test_weird_filenames(self): euid = os.geteuid() for fn in outp["files_out"]: assert os.stat(fn).st_uid == euid + + +class TestInlineDockerfile(RunnerTestCase): + def test1(self): + wdl = """ + version development + task t { + input { + Array[String]+ apt_pkgs + } + command <<< + set -euxo pipefail + apt list --installed | tr '/' $'\t' | sort > installed.txt + sort "~{write_lines(apt_pkgs)}" > expected.txt + join -j 1 -v 2 installed.txt expected.txt > missing.txt + if [ -s missing.txt ]; then + >&2 cat missing.txt + exit 1 + fi + >>> + runtime { + inlineDockerfile: <<< + FROM ubuntu:20.04 + RUN apt-get -qq update && apt-get install -y ~{sep(' ', apt_pkgs)} + >>> + } + } + """ + self._run(wdl, {"apt_pkgs": ["samtools", "tabix"]}) From a981f2aa10380a5ffed12616f8829b8ba029efc0 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 18 Oct 2020 21:28:08 -1000 Subject: [PATCH 2/8] polish --- WDL/runtime/task_container.py | 28 +++++++++++++++++----------- stubs/docker/__init__.py | 3 ++- tests/test_7runner.py | 4 ++++ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/WDL/runtime/task_container.py b/WDL/runtime/task_container.py index f1c6d93d..def66cac 100644 --- a/WDL/runtime/task_container.py +++ b/WDL/runtime/task_container.py @@ -868,14 +868,6 @@ def build_inline_dockerfile( tries: Optional[int] = None, ) -> str: dockerfile_utf8 = self.runtime_values["inlineDockerfile"].encode("utf8") - dockerfile_sha256 = hashlib.sha256(dockerfile_utf8).hexdigest() - # TODO: check maximum tag length - tag = "miniwdl_inline_" - if "-" in self.run_id: - tag += self.run_id.split("-")[1].lower() - else: - tag += self.run_id.lower() - tag += ":" + dockerfile_sha256 build_logfile = os.path.join(self.host_dir, "inlineDockerfile.log.txt") def write_log(stream: Iterable[Dict[str, str]]): @@ -883,16 +875,30 @@ def write_log(stream: Iterable[Dict[str, str]]): with open(build_logfile, "w") as outfile: for d in stream: if "stream" in d: - msg = d["stream"].strip() + msg = d["stream"].rstrip() if msg: logger.verbose(msg) print(msg, file=outfile) + # formulate image tag + dockerfile_digest = hashlib.sha256(dockerfile_utf8).digest() + dockerfile_digest = base64.b32encode(dockerfile_digest[:15]).decode().lower() + tag_part1 = "miniwdl_auto_" + tag_part3 = ":" + dockerfile_digest + tag_part2 = self.run_id.lower() + if "-" in tag_part2: + tag_part2 = tag_part2.split("-")[1] + maxtag2 = 64 - len(tag_part1) - len(tag_part3) + assert maxtag2 > 0 + tag = tag_part1 + tag_part2 + tag_part3 + + # run docker build logger.info(_("starting docker build", tag=tag)) logger.debug(_("Dockerfile", txt=self.runtime_values["inlineDockerfile"])) try: image, build_log = client.images.build(fileobj=BytesIO(dockerfile_utf8), tag=tag) except docker.errors.BuildError as exn: + # potentially retry, if task has runtime.maxRetries if isinstance(tries, int): tries -= 1 else: @@ -905,8 +911,8 @@ def write_log(stream: Iterable[Dict[str, str]]): else: write_log(exn.build_log) logger.error(_("docker build failed", msg=exn.msg, log=build_logfile)) - raise Error.RuntimeError("inlineDockerfile build failed") + raise exn write_log(build_log) - logger.notice(_("docker build", tag=tag, log=build_logfile)) # pyre-ignore + logger.notice(_("docker build", tag=image.tags[0], log=build_logfile)) # pyre-ignore return tag diff --git a/stubs/docker/__init__.py b/stubs/docker/__init__.py index 3563a0f1..9ef0d274 100644 --- a/stubs/docker/__init__.py +++ b/stubs/docker/__init__.py @@ -27,7 +27,8 @@ def build(self, **kwargs) -> Tuple[Image, Iterable[Dict[str,str]]]: ... class Image: - ... + id: str + tags: List[str] class Node: attrs: Dict[str,Any] diff --git a/tests/test_7runner.py b/tests/test_7runner.py index a3010e91..8abef748 100644 --- a/tests/test_7runner.py +++ b/tests/test_7runner.py @@ -698,7 +698,11 @@ def test1(self): FROM ubuntu:20.04 RUN apt-get -qq update && apt-get install -y ~{sep(' ', apt_pkgs)} >>> + maxRetries: 1 } } """ self._run(wdl, {"apt_pkgs": ["samtools", "tabix"]}) + + with self.assertRaises(docker.errors.BuildError): + self._run(wdl, {"apt_pkgs": ["bogusfake123"]}) From 6763bff3115883a7ea9e60dbbae0fa7334f7d347 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 18 Oct 2020 22:41:30 -1000 Subject: [PATCH 3/8] fix --- tests/test_7runner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_7runner.py b/tests/test_7runner.py index 8abef748..bb62c3b0 100644 --- a/tests/test_7runner.py +++ b/tests/test_7runner.py @@ -704,5 +704,4 @@ def test1(self): """ self._run(wdl, {"apt_pkgs": ["samtools", "tabix"]}) - with self.assertRaises(docker.errors.BuildError): - self._run(wdl, {"apt_pkgs": ["bogusfake123"]}) + self._run(wdl, {"apt_pkgs": ["bogusfake123"]}, expected_exception=docker.errors.BuildError) From 1b03a075f954036810444532b97236182be7a589 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Sun, 18 Oct 2020 23:32:27 -1000 Subject: [PATCH 4/8] fix --- tests/test_7runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_7runner.py b/tests/test_7runner.py index bb62c3b0..33a3158d 100644 --- a/tests/test_7runner.py +++ b/tests/test_7runner.py @@ -679,6 +679,9 @@ class TestInlineDockerfile(RunnerTestCase): def test1(self): wdl = """ version development + workflow w { + call t + } task t { input { Array[String]+ apt_pkgs @@ -702,6 +705,6 @@ def test1(self): } } """ - self._run(wdl, {"apt_pkgs": ["samtools", "tabix"]}) + self._run(wdl, {"t.apt_pkgs": ["samtools", "tabix"]}) - self._run(wdl, {"apt_pkgs": ["bogusfake123"]}, expected_exception=docker.errors.BuildError) + self._run(wdl, {"t.apt_pkgs": ["bogusfake123"]}, expected_exception=docker.errors.BuildError) From a52ef9b1d2882ecd7e076f7eb9c1c2d8416c3d4f Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Mon, 19 Oct 2020 11:18:31 -1000 Subject: [PATCH 5/8] wip --- WDL/_grammar.py | 2 +- WDL/_parser.py | 5 +--- WDL/runtime/task.py | 12 +++++--- WDL/runtime/task_container.py | 52 +++++++++++++++++++---------------- tests/test_7runner.py | 8 +++--- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/WDL/_grammar.py b/WDL/_grammar.py index 0b1bca3a..dc19987c 100644 --- a/WDL/_grammar.py +++ b/WDL/_grammar.py @@ -335,7 +335,7 @@ // task runtime section (key-expression pairs) runtime_section: "runtime" "{" [runtime_kv (","? runtime_kv)*] "}" -runtime_kv: CNAME ":" (expr | command2) +runtime_kv: CNAME ":" expr /////////////////////////////////////////////////////////////////////////////////////////////////// // decl diff --git a/WDL/_parser.py b/WDL/_parser.py index cab2902b..df270dc3 100644 --- a/WDL/_parser.py +++ b/WDL/_parser.py @@ -340,10 +340,7 @@ def meta_section(self, items, meta): return d def runtime_kv(self, items, meta): - expr = items[1] - if isinstance(expr, dict) and "command" in expr: # command2 (e.g. inlineDockerfile) - expr = expr["command"] - return (items[0].value, expr) + return (items[0].value, items[1]) def runtime_section(self, items, meta): d = dict() diff --git a/WDL/runtime/task.py b/WDL/runtime/task.py index 4ee036d3..aca45346 100644 --- a/WDL/runtime/task.py +++ b/WDL/runtime/task.py @@ -14,6 +14,7 @@ import re from typing import Tuple, List, Dict, Optional, Callable, Iterable, Set, Any, Union from contextlib import ExitStack +from docker.errors import BuildError as DockerBuildError from .. import Error, Type, Env, Value, StdLib, Tree, Expr, _util from .._util import ( @@ -420,9 +421,12 @@ def _eval_task_runtime( ans = {} if "inlineDockerfile" in runtime_values: - ans["inlineDockerfile"] = _util.strip_leading_whitespace( - runtime_values["inlineDockerfile"].coerce(Type.String()).value - )[1].strip() + # join Array[String] + dockerfile = runtime_values["inlineDockerfile"] + if not isinstance(dockerfile, Value.Array): + dockerfile = Value.Array(dockerfile.type, [dockerfile]) + dockerfile = "\n".join(elt.coerce(Type.String()).value for elt in dockerfile.value) + ans["inlineDockerfile"] = dockerfile elif "docker" in runtime_values: ans["docker"] = runtime_values["docker"].coerce(Type.String()).value @@ -533,7 +537,7 @@ def _try_task( ) ) interruptions += 1 - elif not isinstance(exn, Terminated) and retries < max_retries: + elif not isinstance(exn, (Terminated, DockerBuildError)) and retries < max_retries: logger.error( _( "failed task will be retried", diff --git a/WDL/runtime/task_container.py b/WDL/runtime/task_container.py index def66cac..1572b6ea 100644 --- a/WDL/runtime/task_container.py +++ b/WDL/runtime/task_container.py @@ -493,19 +493,21 @@ def _run(self, logger: logging.Logger, terminating: Callable[[], bool], command: # connect to dockerd client = docker.from_env(version="auto", timeout=900) - # figure desired image, building inlineDockerfile if called for - if "inlineDockerfile" in self.runtime_values: - image_tag = self.build_inline_dockerfile(logger.getChild("inlineDockerfile"), client) - else: - image_tag = self.runtime_values.get("docker", "ubuntu:18.04") - if ":" not in image_tag: - # seems we need to do this explicitly under some configurations -- issue #232 - image_tag += ":latest" - logger.info(_("docker image", tag=image_tag)) - svc = None exit_code = None try: + # figure desired image, building inlineDockerfile if necessary + if "inlineDockerfile" in self.runtime_values: + image_tag = self.build_inline_dockerfile( + logger.getChild("inlineDockerfile"), client + ) + else: + image_tag = self.runtime_values.get("docker", "ubuntu:18.04") + if ":" not in image_tag: + # seems we need to do this explicitly under some configurations -- issue #232 + image_tag += ":latest" + logger.info(_("docker image", tag=image_tag)) + # run container as a transient docker swarm service, letting docker handle the resource # scheduling (waiting until requested # of CPUs are available). kwargs = { @@ -861,13 +863,28 @@ def unique_service_name(self, run_id: str) -> str: assert len(junk) == 24 return f"wdl-{run_id[:34]}-{junk}" # 4 + 34 + 1 + 24 = 63 + _build_inline_dockerfile_lock: threading.Lock = threading.Lock() + def build_inline_dockerfile( self, logger: logging.Logger, client: docker.DockerClient, tries: Optional[int] = None, ) -> str: + # formulate image tag using digest of dockerfile text dockerfile_utf8 = self.runtime_values["inlineDockerfile"].encode("utf8") + dockerfile_digest = hashlib.sha256(dockerfile_utf8).digest() + dockerfile_digest = base64.b32encode(dockerfile_digest[:15]).decode().lower() + tag_part1 = "miniwdl_auto_" + tag_part3 = ":" + dockerfile_digest + tag_part2 = self.run_id.lower() + if "-" in tag_part2: + tag_part2 = tag_part2.split("-")[1] + maxtag2 = 64 - len(tag_part1) - len(tag_part3) + assert maxtag2 > 0 + tag = tag_part1 + tag_part2 + tag_part3 + + # prepare to tee docker build log to logger.verbose and a file build_logfile = os.path.join(self.host_dir, "inlineDockerfile.log.txt") def write_log(stream: Iterable[Dict[str, str]]): @@ -880,23 +897,12 @@ def write_log(stream: Iterable[Dict[str, str]]): logger.verbose(msg) print(msg, file=outfile) - # formulate image tag - dockerfile_digest = hashlib.sha256(dockerfile_utf8).digest() - dockerfile_digest = base64.b32encode(dockerfile_digest[:15]).decode().lower() - tag_part1 = "miniwdl_auto_" - tag_part3 = ":" + dockerfile_digest - tag_part2 = self.run_id.lower() - if "-" in tag_part2: - tag_part2 = tag_part2.split("-")[1] - maxtag2 = 64 - len(tag_part1) - len(tag_part3) - assert maxtag2 > 0 - tag = tag_part1 + tag_part2 + tag_part3 - # run docker build logger.info(_("starting docker build", tag=tag)) logger.debug(_("Dockerfile", txt=self.runtime_values["inlineDockerfile"])) try: - image, build_log = client.images.build(fileobj=BytesIO(dockerfile_utf8), tag=tag) + with SwarmContainer._build_inline_dockerfile_lock: # one build at a time + image, build_log = client.images.build(fileobj=BytesIO(dockerfile_utf8), tag=tag) except docker.errors.BuildError as exn: # potentially retry, if task has runtime.maxRetries if isinstance(tries, int): diff --git a/tests/test_7runner.py b/tests/test_7runner.py index 33a3158d..10718fa9 100644 --- a/tests/test_7runner.py +++ b/tests/test_7runner.py @@ -697,10 +697,10 @@ def test1(self): fi >>> runtime { - inlineDockerfile: <<< - FROM ubuntu:20.04 - RUN apt-get -qq update && apt-get install -y ~{sep(' ', apt_pkgs)} - >>> + inlineDockerfile: [ + "FROM ubuntu:20.04", + "RUN apt-get -qq update && apt-get install -y ${sep(' ', apt_pkgs)}" + ] maxRetries: 1 } } From 5c98f5fc2f3f7a24c0352a417d5f2c2e66bde2da Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Mon, 19 Oct 2020 11:21:58 -1000 Subject: [PATCH 6/8] fix --- WDL/runtime/task_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WDL/runtime/task_container.py b/WDL/runtime/task_container.py index 1572b6ea..678fb605 100644 --- a/WDL/runtime/task_container.py +++ b/WDL/runtime/task_container.py @@ -898,10 +898,10 @@ def write_log(stream: Iterable[Dict[str, str]]): print(msg, file=outfile) # run docker build - logger.info(_("starting docker build", tag=tag)) logger.debug(_("Dockerfile", txt=self.runtime_values["inlineDockerfile"])) try: with SwarmContainer._build_inline_dockerfile_lock: # one build at a time + logger.info(_("starting docker build", tag=tag)) image, build_log = client.images.build(fileobj=BytesIO(dockerfile_utf8), tag=tag) except docker.errors.BuildError as exn: # potentially retry, if task has runtime.maxRetries From 2351d611cdc1d5f6aabbfac4d88d4f17f159d031 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Mon, 19 Oct 2020 11:22:23 -1000 Subject: [PATCH 7/8] fix --- WDL/runtime/task_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WDL/runtime/task_container.py b/WDL/runtime/task_container.py index 678fb605..9587e1d3 100644 --- a/WDL/runtime/task_container.py +++ b/WDL/runtime/task_container.py @@ -898,10 +898,10 @@ def write_log(stream: Iterable[Dict[str, str]]): print(msg, file=outfile) # run docker build - logger.debug(_("Dockerfile", txt=self.runtime_values["inlineDockerfile"])) try: with SwarmContainer._build_inline_dockerfile_lock: # one build at a time logger.info(_("starting docker build", tag=tag)) + logger.debug(_("Dockerfile", txt=self.runtime_values["inlineDockerfile"])) image, build_log = client.images.build(fileobj=BytesIO(dockerfile_utf8), tag=tag) except docker.errors.BuildError as exn: # potentially retry, if task has runtime.maxRetries From 152b369b42be9549a74ca2bee84513adf9b0c349 Mon Sep 17 00:00:00 2001 From: Mike Lin Date: Thu, 31 Dec 2020 22:50:29 -1000 Subject: [PATCH 8/8] short circuit --- WDL/runtime/task_container.py | 8 +++++++- tests/test_7runner.py | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/WDL/runtime/task_container.py b/WDL/runtime/task_container.py index e01a8336..a71324be 100644 --- a/WDL/runtime/task_container.py +++ b/WDL/runtime/task_container.py @@ -920,7 +920,13 @@ def build_inline_dockerfile( assert maxtag2 > 0 tag = tag_part1 + tag_part2 + tag_part3 - # TODO: short-circuit if image with this digest tag already exists + # short-circuit if digest-tagged image already exists + try: + existing = client.images.get(tag) + logger.notice(_("docker build cached", tag=tag, id=existing.id)) # pyre-ignore + return tag + except docker.errors.ImageNotFound: + pass # prepare to tee docker build log to logger.verbose and a file build_logfile = os.path.join(self.host_dir, "inlineDockerfile.log.txt") diff --git a/tests/test_7runner.py b/tests/test_7runner.py index 1a978377..2ee1607d 100644 --- a/tests/test_7runner.py +++ b/tests/test_7runner.py @@ -5,6 +5,7 @@ import os import shutil import json +import time import docker from testfixtures import log_capture from .context import WDL @@ -769,7 +770,8 @@ def test_weird_filenames(self): class TestInlineDockerfile(RunnerTestCase): - def test1(self): + @log_capture() + def test1(self, capture): wdl = """ version development workflow w { @@ -778,6 +780,7 @@ def test1(self): task t { input { Array[String]+ apt_pkgs + Float timestamp } command <<< set -euxo pipefail @@ -792,14 +795,19 @@ def test1(self): runtime { inlineDockerfile: [ "FROM ubuntu:20.04", - "RUN apt-get -qq update && apt-get install -y ${sep(' ', apt_pkgs)}" + "RUN apt-get -qq update && apt-get install -y ${sep(' ', apt_pkgs)}", + "RUN touch ${timestamp}" ] maxRetries: 1 } } """ - self._run(wdl, {"t.apt_pkgs": ["samtools", "tabix"]}) - self._run(wdl, {"t.apt_pkgs": ["bogusfake123"]}, expected_exception=docker.errors.BuildError) + t = time.time() # to ensure the image is built anew on every test run + self._run(wdl, {"t.apt_pkgs": ["samtools", "tabix"], "t.timestamp": t}) + self._run(wdl, {"t.apt_pkgs": ["samtools", "tabix"], "t.timestamp": t}) + logs = [str(record.msg) for record in capture.records if str(record.msg).startswith("docker build cached")] + self.assertEqual(len(logs), 1) + self._run(wdl, {"t.apt_pkgs": ["bogusfake123"], "t.timestamp": t}, expected_exception=docker.errors.BuildError) class TestAbbreviatedCallInput(RunnerTestCase):