From 12ed48994a8bed980019b6ab4b442cfc542a9fb7 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 8 Oct 2018 12:13:04 -0600 Subject: [PATCH] Support env expansion and pre/post build scripts (#17) * Initial changes to dockerutils to support env expansion and also pre/post build scripts * Fixed a couple of little items in the build-image command * Added the changes requested on the PR --- .gitignore | 1 + README.md | 11 +++++++++-- scripts/build-image | 35 ++++++++++++++++++++++++++++++++--- scripts/publish-image | 2 ++ scripts/run-image | 2 ++ 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index fc1f062..19ef7ae 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ build activate .coverage .pytest_cache +.venv diff --git a/README.md b/README.md index fc80bec..a7d8fc1 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ docker directory tree. Each of these sections may contain one of the following: * `pull_FROM_on_force` - defaults to False, if True, add --pull to build command when force building image (or base image) * `image_repo` - the repository to publish the image to * `publication_tag` - the tag for publication (full image name + tag) +* `pre_build_script` - A shell command or script to run before a docker build is issued +* `post_build_script` - A shell command or script to run after a docker build has been compeleted (successfully) ### Synthetic Images Additionally, "synthetic" images can be specified by adding a `run-image` section with a `synthetic_images` definition @@ -102,14 +104,19 @@ tag=experiment.2017.12.16 ... ``` ### Volume Replacement Variables -The volume specification may contain replacement variable designations of the form `{var}`. The supported variables -include: + +The volume specification may contain either environment variables (`$name` and `${name}` formats) as well as specific +variable replacement designations of the form `{var}`. The supported variables include: * `project_root` - will be replaced with the root directory name of the project * `user` - will be replaced with the user name of the user running the command * `project` - replace with project namge ### Image Push Replacement Variables + +The `publication_tag` may contain either environment variables (`$name` and `${name}` formats) as well as specific +variable replacement designations of the form `{var}`. The supported variables include: + * `account` - AWS account designation * `region` - AWS region * `image` - Image name diff --git a/scripts/build-image b/scripts/build-image index a409986..b39773a 100755 --- a/scripts/build-image +++ b/scripts/build-image @@ -16,7 +16,26 @@ def is_multistage(mode): return 'as builder' in open(f'docker/{mode}/Dockerfile').read() -def build(image, image_name, image_tag, pull=False): +def run_pre_script(script: str, config: dict) -> int: + print(f'Running pre-build-script: "{script}"') + return subprocess.call(shlex.split(script), cwd=os.getcwd()) + + +def run_post_script(script: str, config: dict) -> int: + print(f'Running post-build-script: "{script}"') + return subprocess.call(shlex.split(script), cwd=os.getcwd()) + + +def build(image, image_name, image_tag, config={}, pull=False): + pre_script = config.get('pre_build_script', None) + post_script = config.get('post_build_script', None) + + if pre_script: + rc = run_pre_script(pre_script, config=config) + if rc != 0: + print(f'pre-build-script failed: {rc}') + return rc + rc = 0 pull_base = '' if pull: @@ -30,6 +49,14 @@ def build(image, image_name, image_tag, pull=False): if not rc: rc = image_operation(f'docker build {pull_base} --compress -t {image_name}:{image_tag} ' f'-f docker/{image}/Dockerfile .') + + if rc != 0: + print(f'docker build failed: {rc}') + return rc + + if post_script: + rc = run_post_script(post_script, config=config) + return rc @@ -92,11 +119,13 @@ if __name__ == '__main__': image_tag = args.image_tag pull_FROM_on_force = False + image_config = {} if image in config.sections(): - if 'pull_FROM_on_force' in config[image]: + image_config = config[image] + if 'pull_FROM_on_force' in image_config: pull_FROM_on_force = config[image]['pull_FROM_on_force'] - rc = fn(image, image_name, image_tag, args.pull_base or (args.force_build_base and pull_FROM_on_force)) + rc = fn(image, image_name, image_tag, config=image_config, pull=args.pull_base or (args.force_build_base and pull_FROM_on_force)) # because an image may not be present on the clean, ignore a non-zero return code if rc and not args.image == 'clean': sys.exit(rc) diff --git a/scripts/publish-image b/scripts/publish-image index 3dd51af..6f62b01 100755 --- a/scripts/publish-image +++ b/scripts/publish-image @@ -87,6 +87,7 @@ if __name__ == '__main__': tag=image_tag, user=getpass.getuser() ) + publication_tag = os.path.expandvars(publication_tag) ecr_client = boto3.client('ecr') auth_data = ecr_client.get_authorization_token() user, password = base64.b64decode(auth_data['authorizationData'][0]['authorizationToken']).decode().split( @@ -103,6 +104,7 @@ if __name__ == '__main__': tag=image_tag, user=getpass.getuser() ) + publication_tag = os.path.expandvars(publication_tag) # TODO: should we pick up user and password for docker.com? Maybe via credstash? rc = docker_login_dockerhub() diff --git a/scripts/run-image b/scripts/run-image index e166477..1886aa0 100755 --- a/scripts/run-image +++ b/scripts/run-image @@ -45,6 +45,8 @@ def run(mode, image_name, image_tag, **kwargs): user=user, project=os.path.split(get_root_dir())[1] ) + volumes = os.path.expandvars(volumes) + if kwargs['network']: kwargs['network'] = f"--network {kwargs['network']}"