From 2dd5ec1213e4f59ecb53f6a7695532a78425c472 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 8 Oct 2018 09:42:32 -0600 Subject: [PATCH 1/3] Initial changes to dockerutils to support env expansion and also pre/post build scripts --- .gitignore | 1 + README.md | 8 ++++++-- scripts/build-image | 35 +++++++++++++++++++++++++++++++++-- scripts/publish-image | 2 ++ scripts/run-image | 2 ++ 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index fc1f062..19ef7ae 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ build activate .coverage .pytest_cache +.venv diff --git a/README.md b/README.md index fc80bec..cfcec59 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ docker directory tree. Each of these sections may contain one of the following: * `pull_FROM_on_force` - defaults to False, if True, add --pull to build command when force building image (or base image) * `image_repo` - the repository to publish the image to * `publication_tag` - the tag for publication (full image name + tag) +* `pre_build_script` - A shell command or script to run before a docker build is issued +* `post_build_script` - A shell command or script to run after a docker build has been compeleted (successfully) ### Synthetic Images Additionally, "synthetic" images can be specified by adding a `run-image` section with a `synthetic_images` definition @@ -102,14 +104,16 @@ tag=experiment.2017.12.16 ... ``` ### Volume Replacement Variables -The volume specification may contain replacement variable designations of the form `{var}`. The supported variables -include: + +The volume specification may contain either environment variables (`$name` and `${name}` formats) as well as specific +variable replacement designations of the form `{var}`. The supported variables include: * `project_root` - will be replaced with the root directory name of the project * `user` - will be replaced with the user name of the user running the command * `project` - replace with project namge ### Image Push Replacement Variables + * `account` - AWS account designation * `region` - AWS region * `image` - Image name diff --git a/scripts/build-image b/scripts/build-image index a409986..80ac02a 100755 --- a/scripts/build-image +++ b/scripts/build-image @@ -16,7 +16,31 @@ def is_multistage(mode): return 'as builder' in open(f'docker/{mode}/Dockerfile').read() -def build(image, image_name, image_tag, pull=False): +def run_pre_script(script: str, config: dict) -> int: + if config.get('run_as_root', False): + script = f'sudo {script}' + + print(f'Running pre-build-script: "{script}"') + return subprocess.call(shlex.split(script), cwd=os.getcwd()) + + +def run_post_script(script: str, config: dict) -> int: + if config.get('run_as_root', False): + script = f'sudo {script}' + + print(f'Running post-build-script: "{script}"') + return subprocess.call(shlex.split(script), cwd=os.getcwd()) + + +def build(image, image_name, image_tag, config={}, pull=False): + pre_script = config.get('pre_build_script', None) + post_script = config.get('post_build_script', None) + + if pre_script: + rc = run_pre_script(pre_script, config=config) + if rc != 0: + return rc + rc = 0 pull_base = '' if pull: @@ -30,6 +54,13 @@ def build(image, image_name, image_tag, pull=False): if not rc: rc = image_operation(f'docker build {pull_base} --compress -t {image_name}:{image_tag} ' f'-f docker/{image}/Dockerfile .') + + if rc != 0: + return rc + + if post_script: + rc = run_post_script(post_script, config=config) + return rc @@ -96,7 +127,7 @@ if __name__ == '__main__': if 'pull_FROM_on_force' in config[image]: pull_FROM_on_force = config[image]['pull_FROM_on_force'] - rc = fn(image, image_name, image_tag, args.pull_base or (args.force_build_base and pull_FROM_on_force)) + rc = fn(image, image_name, image_tag, config=config.get(image, {}), pull=args.pull_base or (args.force_build_base and pull_FROM_on_force)) # because an image may not be present on the clean, ignore a non-zero return code if rc and not args.image == 'clean': sys.exit(rc) diff --git a/scripts/publish-image b/scripts/publish-image index 3dd51af..6f62b01 100755 --- a/scripts/publish-image +++ b/scripts/publish-image @@ -87,6 +87,7 @@ if __name__ == '__main__': tag=image_tag, user=getpass.getuser() ) + publication_tag = os.path.expandvars(publication_tag) ecr_client = boto3.client('ecr') auth_data = ecr_client.get_authorization_token() user, password = base64.b64decode(auth_data['authorizationData'][0]['authorizationToken']).decode().split( @@ -103,6 +104,7 @@ if __name__ == '__main__': tag=image_tag, user=getpass.getuser() ) + publication_tag = os.path.expandvars(publication_tag) # TODO: should we pick up user and password for docker.com? Maybe via credstash? rc = docker_login_dockerhub() diff --git a/scripts/run-image b/scripts/run-image index e166477..1886aa0 100755 --- a/scripts/run-image +++ b/scripts/run-image @@ -45,6 +45,8 @@ def run(mode, image_name, image_tag, **kwargs): user=user, project=os.path.split(get_root_dir())[1] ) + volumes = os.path.expandvars(volumes) + if kwargs['network']: kwargs['network'] = f"--network {kwargs['network']}" From 720f7d40b2121c137d38a758d116b9ef30466d9f Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 8 Oct 2018 10:12:54 -0600 Subject: [PATCH 2/3] Fixed a couple of little items in the build-image command --- scripts/build-image | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/build-image b/scripts/build-image index 80ac02a..d2d7dbb 100755 --- a/scripts/build-image +++ b/scripts/build-image @@ -39,6 +39,7 @@ def build(image, image_name, image_tag, config={}, pull=False): if pre_script: rc = run_pre_script(pre_script, config=config) if rc != 0: + print(f'pre-build-script failed: {rc}') return rc rc = 0 @@ -56,6 +57,7 @@ def build(image, image_name, image_tag, config={}, pull=False): f'-f docker/{image}/Dockerfile .') if rc != 0: + print(f'docker build failed: {rc}') return rc if post_script: @@ -123,11 +125,13 @@ if __name__ == '__main__': image_tag = args.image_tag pull_FROM_on_force = False + image_config = {} if image in config.sections(): - if 'pull_FROM_on_force' in config[image]: + image_config = config[image] + if 'pull_FROM_on_force' in image_config: pull_FROM_on_force = config[image]['pull_FROM_on_force'] - rc = fn(image, image_name, image_tag, config=config.get(image, {}), pull=args.pull_base or (args.force_build_base and pull_FROM_on_force)) + rc = fn(image, image_name, image_tag, config=image_config, pull=args.pull_base or (args.force_build_base and pull_FROM_on_force)) # because an image may not be present on the clean, ignore a non-zero return code if rc and not args.image == 'clean': sys.exit(rc) From 0939cc833e40cfaf43ca7230d7eda34238cae751 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 8 Oct 2018 12:10:21 -0600 Subject: [PATCH 3/3] Added the changes requested on the PR --- README.md | 3 +++ scripts/build-image | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cfcec59..a7d8fc1 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,9 @@ variable replacement designations of the form `{var}`. The supported variables ### Image Push Replacement Variables +The `publication_tag` may contain either environment variables (`$name` and `${name}` formats) as well as specific +variable replacement designations of the form `{var}`. The supported variables include: + * `account` - AWS account designation * `region` - AWS region * `image` - Image name diff --git a/scripts/build-image b/scripts/build-image index d2d7dbb..b39773a 100755 --- a/scripts/build-image +++ b/scripts/build-image @@ -17,17 +17,11 @@ def is_multistage(mode): def run_pre_script(script: str, config: dict) -> int: - if config.get('run_as_root', False): - script = f'sudo {script}' - print(f'Running pre-build-script: "{script}"') return subprocess.call(shlex.split(script), cwd=os.getcwd()) def run_post_script(script: str, config: dict) -> int: - if config.get('run_as_root', False): - script = f'sudo {script}' - print(f'Running post-build-script: "{script}"') return subprocess.call(shlex.split(script), cwd=os.getcwd())