Add abstraction for buildkit CLI args

Rather than putting buildkit strings directly in compose file, use a mapping abstraction. The format is slightly odd looking string or dict as the only truly required mount arg is id but we may also want to provide a whole dict of options including source and any other flags that buildkit supports for a mount spec.

Signed-off-by: Ollie Edwards <oliver.s.edwards@gmail.com>

compose/config/config_schema_v3.7.json

@@ -89,7 +89,7 @@
                 "network": {"type": "string"},
                 "target": {"type": "string"},
                 "shm_size": {"type": ["integer", "string"]},
-                "secrets": {"$ref": "#/definitions/list_of_strings"}
+                "secrets": {"$ref": "#/definitions/list_of_string_or_dict"}
               },
               "additionalProperties": false
             }
@@ -564,23 +564,38 @@
       ]
     },
 
+    "string_or_dict": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/dict"}
+      ]
+    },
+
+    "list_of_string_or_dict": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {"$ref": "#/definitions/string_or_dict"}
+    },
+
     "list_of_strings": {
       "type": "array",
       "items": {"type": "string"},
       "uniqueItems": true
     },
 
+    "dict": {
+      "type": "object",
+      "patternProperties": {
+        ".+": {
+          "type": ["string", "number", "null"]
+        }
+      },
+      "additionalProperties": false
+    },
+
     "list_or_dict": {
       "oneOf": [
-        {
-          "type": "object",
-          "patternProperties": {
-            ".+": {
-              "type": ["string", "number", "null"]
-            }
-          },
-          "additionalProperties": false
-        },
+        {"$ref": "#/definitions/dict"},
         {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
       ]
     },

compose/service.py

@@ -1791,6 +1791,11 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
             dockerfile = os.path.join(path, dockerfile)
         iidfile = tempfile.mktemp()
 
+        sub_env = merge_environment(
+            os.environ.copy(),
+            self._environment
+        )
+
         command_builder = _CommandBuilder()
         command_builder.add_params("--build-arg", buildargs)
         command_builder.add_list("--cache-from", cache_from)
@@ -1803,16 +1808,16 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
         command_builder.add_arg("--tag", tag)
         command_builder.add_arg("--target", target)
         command_builder.add_arg("--iidfile", iidfile)
-        if self._secrets is not None:
-            command_builder.add_list('--secret', self._secrets)
+
+        if "DOCKER_BUILDKIT" in sub_env and self._secrets is not None:
+            buildkit_secrets = self.get_buildkit_mounts(mount_spec=self._secrets)
+            if buildkit_secrets is not None:
+                command_builder.add_list('--secret', buildkit_secrets)
+
         args = command_builder.build([path])
 
         magic_word = "Successfully built "
         appear = False
-        sub_env = merge_environment(
-            os.environ.copy(),
-            self._environment
-        )
         with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, env=sub_env) as p:
             while True:
                 line = p.stdout.readline()
@@ -1837,6 +1842,16 @@ def build(self, path, tag=None, quiet=False, fileobj=None,
         if not appear:
             yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)})
 
+    def get_buildkit_mounts(self, mount_spec=None):
+        if mount_spec is None:
+            return None
+        mounts = []
+        for spec in mount_spec:
+            if isinstance(spec, six.string_types):
+                spec = {'id': spec}
+            mounts.append(','.join(["{}={}".format(k, v) for (k, v) in spec.items()]))
+        return mounts
+
 
 class _CommandBuilder(object):
     def __init__(self):

tests/acceptance/cli_test.py

@@ -886,6 +886,7 @@ def test_build_with_secrets(self):
         assert 'Successfully built' in build_result.stdout
         run_result = self.dispatch(['run', 'foo'])
         assert 'secret 1' in run_result.stdout
+        assert 'secret 3' in run_result.stdout
 
     @mock.patch.dict(os.environ)
     def test_build_with_secrets_substitution(self):

tests/fixtures/build-secrets/Dockerfile

@@ -5,4 +5,5 @@ FROM busybox
 ARG CACHEBUST
 
 RUN --mount=type=secret,target=/secret,required cp secret out
+RUN --mount=type=secret,target=/secret_3,required cat secret_3 >> out
 CMD cat out

tests/fixtures/build-secrets/docker-compose.yml

@@ -4,4 +4,7 @@ services:
     build:
       context: .
       secrets:
-        - "id=secret,src=${FOO_SECRET:-secret_1}"
+         -
+            id: secret
+            src: "${FOO_SECRET:-secret_1}"
+         - secret_3

tests/fixtures/build-secrets/secret_3

@@ -0,0 +1 @@
+secret 3

tests/integration/service_test.py

@@ -1013,8 +1013,8 @@ def test_build_cli_with_secrets(self):
                                       build={
                                           'context': base_dir,
                                           'secrets': [
-                                              'id=secret_1,src={}'.format(secret_path_1),
-                                              'id=secret_2,src={}'.format(secret_path_2)]
+                                              {'id': 'secret_1', 'src': secret_path_1},
+                                              {'id': 'secret_2', 'src': secret_path_2}]
                                       },
                                       environment={
                                           'DOCKER_BUILDKIT': '1',