From 89e6d53a20a22d7fea42d98cc0f188e0d65538c9 Mon Sep 17 00:00:00 2001 From: Ollie Edwards Date: Fri, 22 Nov 2019 12:23:03 +0000 Subject: [PATCH] Add abstraction for buildkit CLI args Rather than putting buildkit strings directly in compose file, use a mapping abstraction. The format is slightly odd looking string or dict as the only truly required mount arg is id but we may also want to provide a whole dict of options including source and any other flags that buildkit supports for a mount spec. Signed-off-by: Ollie Edwards --- compose/config/config_schema_v3.7.json | 35 +++++++++++++------ compose/service.py | 27 ++++++++++---- tests/acceptance/cli_test.py | 1 + tests/fixtures/build-secrets/Dockerfile | 1 + .../fixtures/build-secrets/docker-compose.yml | 5 ++- tests/fixtures/build-secrets/secret_3 | 1 + tests/integration/service_test.py | 4 +-- 7 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 tests/fixtures/build-secrets/secret_3 diff --git a/compose/config/config_schema_v3.7.json b/compose/config/config_schema_v3.7.json index dbdac57c3ed..b2665596ea2 100644 --- a/compose/config/config_schema_v3.7.json +++ b/compose/config/config_schema_v3.7.json @@ -89,7 +89,7 @@ "network": {"type": "string"}, "target": {"type": "string"}, "shm_size": {"type": ["integer", "string"]}, - "secrets": {"$ref": "#/definitions/list_of_strings"} + "secrets": {"$ref": "#/definitions/list_of_string_or_dict"} }, "additionalProperties": false } @@ -564,23 +564,38 @@ ] }, + "string_or_dict": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/dict"} + ] + }, + + "list_of_string_or_dict": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/string_or_dict"} + }, + "list_of_strings": { "type": "array", "items": {"type": "string"}, "uniqueItems": true }, + "dict": { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + "list_or_dict": { "oneOf": [ - { - "type": "object", - "patternProperties": { - ".+": { - "type": ["string", "number", "null"] - } - }, - "additionalProperties": false - }, + {"$ref": "#/definitions/dict"}, {"type": "array", "items": {"type": "string"}, "uniqueItems": true} ] }, diff --git a/compose/service.py b/compose/service.py index 949da3ed6ef..c01571c15a2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1791,6 +1791,11 @@ def build(self, path, tag=None, quiet=False, fileobj=None, dockerfile = os.path.join(path, dockerfile) iidfile = tempfile.mktemp() + sub_env = merge_environment( + os.environ.copy(), + self._environment + ) + command_builder = _CommandBuilder() command_builder.add_params("--build-arg", buildargs) command_builder.add_list("--cache-from", cache_from) @@ -1803,16 +1808,16 @@ def build(self, path, tag=None, quiet=False, fileobj=None, command_builder.add_arg("--tag", tag) command_builder.add_arg("--target", target) command_builder.add_arg("--iidfile", iidfile) - if self._secrets is not None: - command_builder.add_list('--secret', self._secrets) + + if "DOCKER_BUILDKIT" in sub_env and self._secrets is not None: + buildkit_secrets = self.get_buildkit_mounts(mount_spec=self._secrets) + if buildkit_secrets is not None: + command_builder.add_list('--secret', buildkit_secrets) + args = command_builder.build([path]) magic_word = "Successfully built " appear = False - sub_env = merge_environment( - os.environ.copy(), - self._environment - ) with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, env=sub_env) as p: while True: line = p.stdout.readline() @@ -1837,6 +1842,16 @@ def build(self, path, tag=None, quiet=False, fileobj=None, if not appear: yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) + def get_buildkit_mounts(self, mount_spec=None): + if mount_spec is None: + return None + mounts = [] + for spec in mount_spec: + if isinstance(spec, six.string_types): + spec = {'id': spec} + mounts.append(','.join(["{}={}".format(k, v) for (k, v) in spec.items()])) + return mounts + class _CommandBuilder(object): def __init__(self): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8d613725d09..c7f91c3dcd0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -886,6 +886,7 @@ def test_build_with_secrets(self): assert 'Successfully built' in build_result.stdout run_result = self.dispatch(['run', 'foo']) assert 'secret 1' in run_result.stdout + assert 'secret 3' in run_result.stdout @mock.patch.dict(os.environ) def test_build_with_secrets_substitution(self): diff --git a/tests/fixtures/build-secrets/Dockerfile b/tests/fixtures/build-secrets/Dockerfile index 6998735ec78..db5ce8f04f0 100644 --- a/tests/fixtures/build-secrets/Dockerfile +++ b/tests/fixtures/build-secrets/Dockerfile @@ -5,4 +5,5 @@ FROM busybox ARG CACHEBUST RUN --mount=type=secret,target=/secret,required cp secret out +RUN --mount=type=secret,target=/secret_3,required cat secret_3 >> out CMD cat out diff --git a/tests/fixtures/build-secrets/docker-compose.yml b/tests/fixtures/build-secrets/docker-compose.yml index 29696a7b676..56e884c0629 100644 --- a/tests/fixtures/build-secrets/docker-compose.yml +++ b/tests/fixtures/build-secrets/docker-compose.yml @@ -4,4 +4,7 @@ services: build: context: . secrets: - - "id=secret,src=${FOO_SECRET:-secret_1}" + - + id: secret + src: "${FOO_SECRET:-secret_1}" + - secret_3 diff --git a/tests/fixtures/build-secrets/secret_3 b/tests/fixtures/build-secrets/secret_3 new file mode 100644 index 00000000000..6e9349d38e6 --- /dev/null +++ b/tests/fixtures/build-secrets/secret_3 @@ -0,0 +1 @@ +secret 3 diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 039cb424c1a..289a591fa10 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1013,8 +1013,8 @@ def test_build_cli_with_secrets(self): build={ 'context': base_dir, 'secrets': [ - 'id=secret_1,src={}'.format(secret_path_1), - 'id=secret_2,src={}'.format(secret_path_2)] + {'id': 'secret_1', 'src': secret_path_1}, + {'id': 'secret_2', 'src': secret_path_2}] }, environment={ 'DOCKER_BUILDKIT': '1',