Merge branch 'master' into update-deps

mozilla · Dec 6, 2018 · ecd8c0d · ecd8c0d
2 parents 9679e42 + a60e025
commit ecd8c0d
Show file tree

Hide file tree

Showing 14 changed files with 494 additions and 404 deletions.
diff --git a/lib/please_cli/please_cli/__init__.py b/lib/please_cli/please_cli/__init__.py
@@ -106,7 +106,8 @@ def cmd_tools(ctx):
 
 
 cmd.add_command(cmd_tools, "tools")
-cmd_tools.add_command(please_cli.base_image.cmd, "base-image")
+cmd_tools.add_command(please_cli.base_image.build, "build-base-image")
+cmd_tools.add_command(please_cli.base_image.push, "push-base-image")
 cmd_tools.add_command(please_cli.build.cmd, "build")
 cmd_tools.add_command(please_cli.build.cmd_docker, "docker")
 cmd_tools.add_command(please_cli.shell.cmd_docker_shell, "docker-shell")

diff --git a/lib/please_cli/please_cli/base_image.py b/lib/please_cli/please_cli/base_image.py
@@ -32,18 +32,6 @@
     default='docker',
     help='Path to docker command (default: docker).',
     )
-@click.option(
-    '--docker-username',
-    required=False,
-    default=None,
-    help='https://hub.docker.com username',
-    )
-@click.option(
-    '--docker-password',
-    required=False,
-    default=None,
-    help='https://hub.docker.com password',
-    )
 @click.option(
     '--docker-repo',
     required=True,
@@ -70,40 +58,28 @@
     default=[],
     help='Public key for nix cache',
     )
-@click.option(
-    '--dont-push',
-    is_flag=True,
-    help='Push to docker registry',
-    )
 @cli_common.cli.taskcluster_options
-def cmd(docker_username,
-        docker_password,
-        docker,
-        docker_repo,
-        docker_tag,
-        nix_cache_public_keys,
-        nix_cache_public_urls,
+def build(docker,
+          docker_repo,
+          docker_tag,
+          nix_cache_public_keys,
+          nix_cache_public_urls,
+          taskcluster_secret,
+          taskcluster_client_id,
+          taskcluster_access_token,
+          ):
+
+    secrets = cli_common.taskcluster.get_secrets(
         taskcluster_secret,
-        taskcluster_client_id,
-        taskcluster_access_token,
-        dont_push,
-        ):
-
-    secrets = cli_common.taskcluster.get_secrets(taskcluster_secret,
-                                                 None,
-                                                 required=[
-                                                     'DOCKER_USERNAME',
-                                                     'DOCKER_PASSWORD',
-                                                     'NIX_CACHE_PUBLIC_KEYS',
-                                                     'NIX_CACHE_PUBLIC_URLS',
-                                                 ],
-                                                 taskcluster_client_id=taskcluster_client_id,
-                                                 taskcluster_access_token=taskcluster_access_token,
-                                                 )
-
+        None,
+        required=[
+            'NIX_CACHE_PUBLIC_KEYS',
+            'NIX_CACHE_PUBLIC_URLS',
+        ],
+        taskcluster_client_id=taskcluster_client_id,
+        taskcluster_access_token=taskcluster_access_token,
+    )
 
-    docker_username = docker_username or secrets['DOCKER_USERNAME']
-    docker_password = docker_password or secrets['DOCKER_PASSWORD']
     nix_cache_public_keys = nix_cache_public_keys or secrets['NIX_CACHE_PUBLIC_KEYS']
     nix_cache_public_urls = nix_cache_public_urls or secrets['NIX_CACHE_PUBLIC_URLS']
 
@@ -130,18 +106,13 @@ def cmd(docker_username,
        'sha256' not in nix_json:
         raise click.ClickException('`nix/nix.json` is not of correct format.')
 
-
     try:
-        click.echo(' => Creating Dockerfile ... ', nl=False)
-        with click_spinner.spinner():
-            shutil.copyfile(docker_file, temp_docker_file)
-        please_cli.utils.check_result(0, '')
+        shutil.copyfile(docker_file, temp_docker_file)
 
         click.echo(' => Building base docker image ... ', nl=False)
         with click_spinner.spinner():
             result, output, error = cli_common.command.run(
                 [
-                    # 'sudo',
                     docker,
                     'build',
                     '--no-cache',
@@ -155,8 +126,7 @@ def cmd(docker_username,
                     '--build-arg', 'NIX_SHA256=' + nix_json['sha256'],
                     '--build-arg', 'NIX_CACHE_PUBLIC_KEYS=' + ' '.join(nix_cache_public_keys),
                     '--build-arg', 'NIX_CACHE_PUBLIC_URLS=' + ' '.join(nix_cache_public_urls),
-                    '-t',
-                    f'{docker_repo}:{docker_tag}',
+                    '-t', f'{docker_repo}:{docker_tag}',
                     please_cli.config.ROOT_DIR,
                 ],
                 stream=True,
@@ -165,42 +135,74 @@ def cmd(docker_username,
             )
         please_cli.utils.check_result(result, output)
 
-        if not dont_push:
-
-            click.echo(' => Logging into hub.docker.com ... ', nl=False)
-            with click_spinner.spinner():
-                result, output, error = cli_common.command.run(
-                    [
-                        # 'sudo',
-                        docker,
-                        'login',
-                        '--username', docker_username,
-                        '--password', docker_password,
-                    ],
-                    stream=True,
-                    stderr=subprocess.STDOUT,
-                    log_command=False,
-                )
-            please_cli.utils.check_result(result, output)
-
-            click.echo(' => Pushing base docker image ... ', nl=False)
-            with click_spinner.spinner():
-                result, output, error = cli_common.command.run(
-                    [
-                        # 'sudo',
-                        docker,
-                        'push',
-                        '{}:{}'.format(docker_repo, docker_tag),
-                    ],
-                    stream=True,
-                    stderr=subprocess.STDOUT,
-                )
-            please_cli.utils.check_result(result, output)
-
     finally:
         if os.path.exists(temp_docker_file):
             os.unlink(temp_docker_file)
 
 
-if __name__ == "__main__":
-    cmd()
+@click.command(
+    cls=please_cli.utils.ClickCustomCommand,
+    short_help="Push base docker image.",
+    epilog="Happy hacking!",
+    )
+@click.option(
+    '--docker-registry',
+    required=True,
+    help='Docker registry.',
+    default=please_cli.config.DOCKER_BASE_REGISTRY,
+    )
+@click.option(
+    '--docker-repo',
+    required=True,
+    help='Docker repository.',
+    default=please_cli.config.DOCKER_BASE_REPO,
+    )
+@click.option(
+    '--docker-username',
+    help='Docker username.',
+    )
+@click.option(
+    '--docker-password',
+    help='Docker password.',
+    )
+@click.option(
+    '--docker-tag',
+    required=True,
+    help='Tag of base image.',
+    default=please_cli.config.DOCKER_BASE_TAG,
+    )
+@cli_common.cli.taskcluster_options
+def push(docker_registry,
+         docker_repo,
+         docker_username,
+         docker_password,
+         docker_tag,
+         taskcluster_secret,
+         taskcluster_client_id,
+         taskcluster_access_token,
+         ):
+
+    secrets = cli_common.taskcluster.get_secrets(
+        taskcluster_secret,
+        None,
+        required=[
+            'DOCKER_USERNAME',
+            'DOCKER_PASSWORD',
+        ],
+        taskcluster_client_id=taskcluster_client_id,
+        taskcluster_access_token=taskcluster_access_token,
+        )
+
+    docker_username = docker_username or secrets['DOCKER_USERNAME']
+    docker_password = docker_password or secrets['DOCKER_PASSWORD']
+    image_reference = f'docker-daemon:{docker_repo}:{docker_tag}'
+    click.echo(' => Pushing the image to the registry ... ', nl=False)
+    with click_spinner.spinner():
+        please_cli.utils.push_docker_image(
+            registry=docker_registry,
+            username=docker_username,
+            password=docker_password,
+            image=image_reference,
+            repo=docker_repo,
+            tag=docker_tag,
+        )
diff --git a/lib/please_cli/please_cli/config.py b/lib/please_cli/please_cli/config.py
@@ -40,6 +40,7 @@
 CHANNELS = ['master', 'testing', 'staging', 'production']
 DEPLOY_CHANNELS = ['testing', 'staging', 'production']
 
+DOCKER_BASE_REGISTRY = 'index.docker.io'
 DOCKER_BASE_REPO = 'mozillareleng/services'
 DOCKER_BASE_TAG = 'base-' + VERSION
 
@@ -893,6 +894,29 @@
                     },
                 },
             },
+            {
+                'target': 'DOCKERHUB',
+                'options': {
+                    'testing': {
+                        'enable': True,
+                        'nix_path_attribute': 'worker_dockerflow',
+                        'docker_registry': 'index.docker.io',
+                        'docker_repo': 'mozilla/shipitbackend',
+                    },
+                    'staging': {
+                        'enable': True,
+                        'nix_path_attribute': 'worker_dockerflow',
+                        'docker_registry': 'index.docker.io',
+                        'docker_repo': 'mozilla/shipitbackend',
+                    },
+                    'production': {
+                        'enable': True,
+                        'nix_path_attribute': 'worker_dockerflow',
+                        'docker_registry': 'index.docker.io',
+                        'docker_repo': 'mozilla/shipitbackend',
+                    },
+                },
+            },
         ],
     },
     'shipit/frontend': {

diff --git a/lib/please_cli/please_cli/deploy.py b/lib/please_cli/please_cli/deploy.py
@@ -314,7 +314,7 @@ def cmd_HEROKU(ctx,
                 registry=docker_registry,
                 username=heroku_username,
                 password=heroku_api_token,
-                image=project_path,
+                image=f'docker-archive://{project_path}',
                 repo=repo,
                 tag=tag,
                 interactive=interactive,
@@ -479,7 +479,7 @@ def cmd_TASKCLUSTER_HOOK(ctx,
                     registry=docker_registry,
                     username=docker_username,
                     password=docker_password,
-                    image=image,
+                    image=f'docker-archive://{image}',
                     repo=docker_repo,
                     tag=image_tag,
                     interactive=interactive,
@@ -638,7 +638,7 @@ def cmd_DOCKERHUB(ctx,
                     registry=docker_registry,
                     username=docker_username,
                     password=docker_password,
-                    image=project_path,
+                    image=f'docker-archive://{project_path}',
                     repo=docker_repo,
                     tag=tag,
                     interactive=interactive,

diff --git a/lib/please_cli/please_cli/utils.py b/lib/please_cli/please_cli/utils.py
@@ -163,11 +163,7 @@ def push_docker_image(registry, username, password, image, repo, tag,
                       interactive=False):
     dest = f'docker://{registry}/{repo}:{tag}'
     with authfile(registry, username, password) as auth_file:
-        command = ['skopeo', 'copy',
-                   '--authfile', auth_file,
-                   'docker-archive://{}'.format(image),
-                   dest
-                  ]
+        command = ['skopeo', 'copy', '--authfile', auth_file, image, dest]
         result, output, error = cli_common.command.run(
             command,
             stream=True,