Support buildkit build time secrets

compose/config/config_schema_v3.7.json

@@ -88,7 +88,8 @@
                 "cache_from": {"$ref": "#/definitions/list_of_strings"},
                 "network": {"type": "string"},
                 "target": {"type": "string"},
-                "shm_size": {"type": ["integer", "string"]}
+                "shm_size": {"type": ["integer", "string"]},
+                "secrets": {"$ref": "#/definitions/build_secrets"}
               },
               "additionalProperties": false
             }
@@ -563,6 +564,24 @@
       ]
     },
 
+    "build_secrets": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {
+        "oneOf": [
+          {"type": "string"},
+          {
+            "type": "object",
+            "required": ["id"],
+            "properties": {
+              "id": {"type": "string"},
+              "src": {"type": "string"}
+            }
+          }
+        ]
+      }
+    },
+
     "list_of_strings": {
       "type": "array",
       "items": {"type": "string"},

compose/service.py

@@ -1080,7 +1080,9 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a
                 'Impossible to perform platform-targeted builds for API version < 1.35'
             )
 
-        builder = self.client if not cli else _CLIBuilder(progress)
+        builder = self.client if not cli else _CLIBuilder(progress,
+                                                          self.options.get('environment'),
+                                                          build_opts.get('secrets'))
         build_output = builder.build(
             path=path,
             tag=self.image_name,
@@ -1717,8 +1719,12 @@ def rewrite_build_path(path):
 
 
 class _CLIBuilder(object):
-    def __init__(self, progress):
+    def __init__(self, progress, environment=None, secrets=None):
         self._progress = progress
+        if environment is None:
+            environment = {}
+        self._environment = environment
+        self._secrets = secrets
 
     def build(self, path, tag=None, quiet=False, fileobj=None,
               nocache=False, rm=False, timeout=None,
@@ -1785,6 +1791,11 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
             dockerfile = os.path.join(path, dockerfile)
         iidfile = tempfile.mktemp()
 
+        sub_env = merge_environment(
+            os.environ.copy(),
+            self._environment
+        )
+
         command_builder = _CommandBuilder()
         command_builder.add_params("--build-arg", buildargs)
         command_builder.add_list("--cache-from", cache_from)
@@ -1797,11 +1808,17 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
         command_builder.add_arg("--tag", tag)
         command_builder.add_arg("--target", target)
         command_builder.add_arg("--iidfile", iidfile)
+
+        if "DOCKER_BUILDKIT" in sub_env and self._secrets is not None:
+            buildkit_secrets = self.get_buildkit_mounts(mount_spec=self._secrets)
+            if buildkit_secrets is not None:
+                command_builder.add_list('--secret', buildkit_secrets)
+
         args = command_builder.build([path])
 
         magic_word = "Successfully built "
         appear = False
-        with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p:
+        with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, env=sub_env) as p:
             while True:
                 line = p.stdout.readline()
                 if not line:
@@ -1825,6 +1842,16 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
         if not appear:
             yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)})
 
+    def get_buildkit_mounts(self, mount_spec=None):
+        if mount_spec is None:
+            return None
+        mounts = []
+        for spec in mount_spec:
+            if isinstance(spec, six.string_types):
+                spec = {'id': spec}
+            mounts.append(','.join(["{}={}".format(k, v) for (k, v) in spec.items()]))
+        return mounts
+
 
 class _CommandBuilder(object):
     def __init__(self):

tests/acceptance/cli_test.py

@@ -877,6 +877,28 @@ def test_build_override_dir(self):
 
         assert 'Successfully built' in result.stdout
 
+    @mock.patch.dict(os.environ)
+    def test_build_with_secrets(self):
+        os.environ["COMPOSE_DOCKER_CLI_BUILD"] = "1"
+        os.environ["DOCKER_BUILDKIT"] = "1"
+        self.base_dir = 'tests/fixtures/build-secrets'
+        build_result = self.dispatch(['build', '--build-arg', 'CACHEBUST=1'])
+        assert 'Successfully built' in build_result.stdout
+        run_result = self.dispatch(['run', 'foo'])
+        assert 'secret 1' in run_result.stdout
+        assert 'secret 3' in run_result.stdout
+
+    @mock.patch.dict(os.environ)
+    def test_build_with_secrets_substitution(self):
+        os.environ["COMPOSE_DOCKER_CLI_BUILD"] = "1"
+        os.environ["DOCKER_BUILDKIT"] = "1"
+        os.environ["FOO_SECRET"] = "secret_2"
+        self.base_dir = 'tests/fixtures/build-secrets'
+        build_result = self.dispatch(['build', '--build-arg', 'CACHEBUST=2'])
+        assert 'Successfully built' in build_result.stdout
+        run_result = self.dispatch(['run', 'foo'])
+        assert 'secret 2' in run_result.stdout
+
     def test_build_override_dir_invalid_path(self):
         config_path = os.path.abspath('tests/fixtures/build-path-override-dir/docker-compose.yml')
         result = self.dispatch([

tests/fixtures/build-secrets/Dockerfile

@@ -0,0 +1,9 @@
+# syntax=docker/dockerfile:1.0.0-experimental
+FROM busybox
+
+# https://github.com/moby/moby/issues/1996#issuecomment-550020843
+ARG CACHEBUST
+
+RUN --mount=type=secret,target=/secret,required cp secret out
+RUN --mount=type=secret,target=/secret_3,required cat secret_3 >> out
+CMD cat out

tests/fixtures/build-secrets/docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3.7'
+services:
+  foo:
+    build:
+      context: .
+      secrets:
+         -
+            id: secret
+            src: "${FOO_SECRET:-secret_1}"
+         - secret_3

tests/fixtures/build-secrets/secret_1

@@ -0,0 +1 @@
+secret 1

tests/fixtures/build-secrets/secret_2

@@ -0,0 +1 @@
+secret 2

tests/fixtures/build-secrets/secret_3

@@ -0,0 +1 @@
+secret 3

tests/integration/service_test.py

@@ -7,6 +7,7 @@
 import tempfile
 from distutils.spawn import find_executable
 from os import path
+from textwrap import dedent
 
 import pytest
 from docker.errors import APIError
@@ -973,12 +974,49 @@ def test_build_cli(self):
         self.addCleanup(shutil.rmtree, base_dir)
 
         with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
-            f.write("FROM busybox\n")
+            # f.write("FROM busybox\n")
+            f.write(dedent("""\
+                           # syntax=docker/dockerfile:1.0.0-experimental
+                           FROM busybox
+                           RUN --mount=type=secret,target=/secret_1 true
+                           """))
 
         service = self.create_service('web',
                                       build={'context': base_dir},
                                       environment={
-                                          'COMPOSE_DOCKER_CLI_BUILD': '1',
+                                          'DOCKER_BUILDKIT': '1',
+                                      })
+        service.build(cli=True)
+        self.addCleanup(self.client.remove_image, service.image_name)
+        assert self.client.inspect_image('composetest_web')
+
+    def test_build_cli_with_secrets(self):
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+        secret_path_1 = os.path.join(base_dir, 'secret_1')
+        secret_path_2 = os.path.join(base_dir, 'secret_2')
+
+        with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
+            # f.write("FROM busybox\n")
+            f.write(dedent("""\
+                           # syntax=docker/dockerfile:1.0.0-experimental
+                           FROM busybox
+                           RUN --mount=type=secret,target=/secret_1,required cat /secret_1
+                           RUN --mount=type=secret,target=/secret_2,required cat /secret_2
+                           """))
+        with open(secret_path_1, 'w') as f:
+            f.write("secret 1\n")
+        with open(secret_path_2, 'w') as f:
+            f.write("secret 2\n")
+
+        service = self.create_service('web',
+                                      build={
+                                          'context': base_dir,
+                                          'secrets': [
+                                              {'id': 'secret_1', 'src': secret_path_1},
+                                              {'id': 'secret_2', 'src': secret_path_2}]
+                                      },
+                                      environment={
                                           'DOCKER_BUILDKIT': '1',
                                       })
         service.build(cli=True)
@@ -992,12 +1030,7 @@ def test_up_build_cli(self):
         with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
             f.write("FROM busybox\n")
 
-        web = self.create_service('web',
-                                  build={'context': base_dir},
-                                  environment={
-                                      'COMPOSE_DOCKER_CLI_BUILD': '1',
-                                      'DOCKER_BUILDKIT': '1',
-                                  })
+        web = self.create_service('web', build={'context': base_dir})
         project = Project('composetest', [web], self.client)
         project.up(do_build=BuildAction.force)