adds build time secret support

This is done using the build section in the yaml which may look like a strange decision given build secrets are almost by definition not in a well known location. It's intended that this feature be used with env substitution.

Signed-off-by: Ollie Edwards <oliver.s.edwards@gmail.com>

compose/config/config_schema_v3.7.json

@@ -88,7 +88,8 @@
                 "cache_from": {"$ref": "#/definitions/list_of_strings"},
                 "network": {"type": "string"},
                 "target": {"type": "string"},
-                "shm_size": {"type": ["integer", "string"]}
+                "shm_size": {"type": ["integer", "string"]},
+                "secrets": {"$ref": "#/definitions/list_of_strings"}
               },
               "additionalProperties": false
             }

compose/service.py

@@ -1080,7 +1080,9 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a
                 'Impossible to perform platform-targeted builds for API version < 1.35'
             )
 
-        builder = self.client if not cli else _CLIBuilder(progress, self.options.get('environment'))
+        builder = self.client if not cli else _CLIBuilder(progress,
+                                                          self.options.get('environment'),
+                                                          build_opts.get('secrets'))
         build_output = builder.build(
             path=path,
             tag=self.image_name,
@@ -1717,11 +1719,12 @@ def rewrite_build_path(path):
 
 
 class _CLIBuilder(object):
-    def __init__(self, progress, environment=None):
+    def __init__(self, progress, environment=None, secrets=None):
         self._progress = progress
         if environment is None:
             environment = {}
         self._environment = environment
+        self._secrets = secrets
 
     def build(self, path, tag=None, quiet=False, fileobj=None,
               nocache=False, rm=False, timeout=None,
@@ -1800,6 +1803,8 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
         command_builder.add_arg("--tag", tag)
         command_builder.add_arg("--target", target)
         command_builder.add_arg("--iidfile", iidfile)
+        if self._secrets is not None:
+            command_builder.add_list('--secret', self._secrets)
         args = command_builder.build([path])
 
         magic_word = "Successfully built "

tests/acceptance/cli_test.py

@@ -877,6 +877,27 @@ def test_build_override_dir(self):
 
         assert 'Successfully built' in result.stdout
 
+    @mock.patch.dict(os.environ)
+    def test_build_with_secrets(self):
+        os.environ["COMPOSE_DOCKER_CLI_BUILD"] = "1"
+        os.environ["DOCKER_BUILDKIT"] = "1"
+        self.base_dir = 'tests/fixtures/build-secrets'
+        build_result = self.dispatch(['build', '--build-arg', 'CACHEBUST=1'])
+        assert 'Successfully built' in build_result.stdout
+        run_result = self.dispatch(['run', 'foo'])
+        assert 'secret 1' in run_result.stdout
+
+    @mock.patch.dict(os.environ)
+    def test_build_with_secrets_substitution(self):
+        os.environ["COMPOSE_DOCKER_CLI_BUILD"] = "1"
+        os.environ["DOCKER_BUILDKIT"] = "1"
+        os.environ["FOO_SECRET"] = "secret_2"
+        self.base_dir = 'tests/fixtures/build-secrets'
+        build_result = self.dispatch(['build', '--build-arg', 'CACHEBUST=2'])
+        assert 'Successfully built' in build_result.stdout
+        run_result = self.dispatch(['run', 'foo'])
+        assert 'secret 2' in run_result.stdout
+
     def test_build_override_dir_invalid_path(self):
         config_path = os.path.abspath('tests/fixtures/build-path-override-dir/docker-compose.yml')
         result = self.dispatch([

tests/fixtures/build-secrets/Dockerfile

@@ -0,0 +1,8 @@
+# syntax=docker/dockerfile:1.0.0-experimental
+FROM busybox
+
+# https://github.com/moby/moby/issues/1996#issuecomment-550020843
+ARG CACHEBUST
+
+RUN --mount=type=secret,target=/secret,required cp secret out
+CMD cat out

tests/fixtures/build-secrets/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3.7'
+services:
+  foo:
+    build:
+      context: .
+      secrets:
+        - "id=secret,src=${FOO_SECRET:-secret_1}"

tests/fixtures/build-secrets/secret_1

@@ -0,0 +1 @@
+secret 1

tests/fixtures/build-secrets/secret_2

@@ -0,0 +1 @@
+secret 2

tests/integration/service_test.py

@@ -990,6 +990,39 @@ def test_build_cli(self):
         self.addCleanup(self.client.remove_image, service.image_name)
         assert self.client.inspect_image('composetest_web')
 
+    def test_build_cli_with_secrets(self):
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+        secret_path_1 = os.path.join(base_dir, 'secret_1')
+        secret_path_2 = os.path.join(base_dir, 'secret_2')
+
+        with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
+            # f.write("FROM busybox\n")
+            f.write(dedent("""\
+                           # syntax=docker/dockerfile:1.0.0-experimental
+                           FROM busybox
+                           RUN --mount=type=secret,target=/secret_1,required cat /secret_1
+                           RUN --mount=type=secret,target=/secret_2,required cat /secret_2
+                           """))
+        with open(secret_path_1, 'w') as f:
+            f.write("secret 1\n")
+        with open(secret_path_2, 'w') as f:
+            f.write("secret 2\n")
+
+        service = self.create_service('web',
+                                      build={
+                                          'context': base_dir,
+                                          'secrets': [
+                                              'id=secret_1,src={}'.format(secret_path_1),
+                                              'id=secret_2,src={}'.format(secret_path_2)]
+                                      },
+                                      environment={
+                                          'DOCKER_BUILDKIT': '1',
+                                      })
+        service.build(cli=True)
+        self.addCleanup(self.client.remove_image, service.image_name)
+        assert self.client.inspect_image('composetest_web')
+
     def test_up_build_cli(self):
         base_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, base_dir)