diff --git a/scripts/run-image b/scripts/run-image index 08409d6..f9c9cbd 100755 --- a/scripts/run-image +++ b/scripts/run-image @@ -11,6 +11,8 @@ import os import shlex import subprocess +from string import Template + from dockerutils import * if sys.version_info < (3, 0): from ConfigParser import ConfigParser @@ -36,6 +38,24 @@ def fetch_env_variables(config, image, args_env=None): return ' '.join(['-e {key}={value}'.format(key=key, value=value) for key, value in env_vars.items()]) +def populate_aws_env_variables(): + AWS_ENVS = [ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_DEFAULT_REGION', + 'AWS_REGION', + 'AWS_SESSION_TOKEN', + 'AWS_SECURITY_TOKEN' + ] + + variables = {} + for test_env in AWS_ENVS: + if os.getenv(test_env): + variables[test_env] = "${" + test_env + "}" + + return ' '.join(['-e {key}={value}'.format(key=k, value=v) for k,v in variables.items()]) + + def run(mode, image_name, image_tag, **kwargs): user = getpass.getuser() volumes = kwargs['volumes'].format( @@ -64,7 +84,11 @@ def run(mode, image_name, image_tag, **kwargs): init=kwargs['init']) print('\n\n============================================================================') print('{cmd}\n\n'.format(cmd=cmd)) - return subprocess.call(shlex.split(cmd), cwd=os.getcwd()) + + # Since we are using secure env values I don't want those to print in the above commmand, but + # they need to be expanded for the subprocess.call + expanded_cmd = Template(cmd).substitute(os.environ) + return subprocess.call(shlex.split(expanded_cmd), cwd=os.getcwd()) if __name__ == '__main__': @@ -101,7 +125,7 @@ if __name__ == '__main__': init = '--init' run_config = { - 'environment': fetch_env_variables(config, args.image, args.env), + 'environment': fetch_env_variables(config, args.image, args.env) + populate_aws_env_variables(), 'keep_container': args.keep or '--rm', 'interactive': '-d' if args.keep else '-it', 'gpu': gpu, diff --git a/scripts/run-notebook b/scripts/run-notebook index 0eaaf8c..c006cfa 100755 --- a/scripts/run-notebook +++ b/scripts/run-notebook @@ -4,6 +4,7 @@ DOCKER_DS_DONT_PULL=${DOCKER_DS_DONT_PULL:-pull} DOCKER_DS_DIFFS=${DOCKER_DS_DIFFS:-no_diffs} DOCKER_ENV="" +NOTEBOOK_IMAGE=resero/docker-ds . dock-sync @@ -28,6 +29,14 @@ if [ "$DOCKER_DS_DIFFS" != "no_diffs" ]; then DOCKER_ENV="-e DOCKER_DS_DIFFS=1" fi +AWS_ENV_NAMES=('AWS_ACCESS_KEY_ID' 'AWS_SECRET_ACCESS_KEY' 'AWS_DEFAULT_REGION' 'AWS_REGION' 'AWS_SESSION_TOKEN' 'AWS_SECURITY_TOKEN') +AWS_ENV='' +for aws_env in ${AWS_ENV_NAMES[@]}; do + if [ ! -z "${!aws_env}" ]; then + AWS_ENV+=" -e $aws_env=\${$aws_env}" + fi +done + if [ -d "./docker/notebook" ]; then build-image -f notebook if [ -z "$DOCKER_HOST" ] @@ -41,7 +50,7 @@ if [ -d "./docker/notebook" ]; then else # Script to run resero-labs/docker-ds notebok against the current directory if [ "$DOCKER_DS_DONT_PULL" = "pull" ]; then - docker pull resero-labs/docker-ds:$DOCKER_TAG + docker pull $NOTEBOOK_IMAGE:$DOCKER_TAG fi date_stamp=$(date "+%Y_%m_%d_%H.%M.%S") @@ -63,13 +72,14 @@ else volume_mounts="--mount type=bind,source=$(pwd),target=/home/jovyan/project" fi fi - echo 'docker run '$GPU_OPTS' --init --name '$USER'_notebook_'$date_stamp' '$DOCKER_ENV' -e NOTEBOOK_MODE=lab --rm -it '$volume_mounts' -p 8888:8888 resero-labs/docker-ds:'$DOCKER_TAG - docker run $GPU_OPTS --init --name $USER"_notebook_"$date_stamp $DOCKER_ENV -e NOTEBOOK_MODE=lab --rm -it $volume_mounts -p 8888:8888 resero-labs/docker-ds:$DOCKER_TAG + cmd='docker run '$GPU_OPTS' --init --name '$USER'_notebook_'$date_stamp' '$DOCKER_ENV' -e NOTEBOOK_MODE=lab '$AWS_ENV' --rm -it '$volume_mounts' -p 8888:8888 '$NOTEBOOK_IMAGE':'$DOCKER_TAG + echo $cmd + eval $cmd else sync-up - echo 'docker run '$GPU_OPTS' --init --name '$USER'_notebook_'$date_stamp' '$DOCKER_ENV' -e NOTEBOOK_MODE=lab --rm -it --mount type=bind,source=/data/workspaces/'$USER'/code/'${PWD##*/}',target=/home/jovyan/project -v /data:/data -p 8888:8888 resero-labs/docker-ds:'$DOCKER_TAG - docker run --init $GPU_OPTS --name $USER"_notebook_"$date_stamp $DOCKER_ENV -e NOTEBOOK_MODE=lab --rm -it --mount type=bind,source="/data/workspaces/"$USER"/code/"${PWD##*/}",target=/home/jovyan/project" -v /data:/data -p 8888:8888 resero-labs/docker-ds:$DOCKER_TAG + cmd='docker run '$GPU_OPTS' --init --name '$USER'_notebook_'$date_stamp' '$DOCKER_ENV' -e NOTEBOOK_MODE=lab '$AWS_ENV' --rm -it --mount type=bind,source=/data/workspaces/'$USER'/code/'${PWD##*/}',target=/home/jovyan/project -v /data:/data -p 8888:8888 '$NOTEBOOK_IMAGE':'$DOCKER_TAG + echo $cmd + eval $cmd sync-down fi - -fi \ No newline at end of file +fi