diff --git a/compose/config/config_schema_v3.7.json b/compose/config/config_schema_v3.7.json index cd7882f5b24..ff4e130cf36 100644 --- a/compose/config/config_schema_v3.7.json +++ b/compose/config/config_schema_v3.7.json @@ -88,7 +88,8 @@ "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, "target": {"type": "string"}, - "shm_size": {"type": ["integer", "string"]} + "shm_size": {"type": ["integer", "string"]}, + "secrets": {"$ref": "#/definitions/build_secrets"} }, "additionalProperties": false } @@ -563,6 +564,24 @@ ] }, + "build_secrets": { + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["id"], + "properties": { + "id": {"type": "string"}, + "src": {"type": "string"} + } + } + ] + } + }, + "list_of_strings": { "type": "array", "items": {"type": "string"}, diff --git a/compose/service.py b/compose/service.py index d329be97992..c01571c15a2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1080,7 +1080,9 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a 'Impossible to perform platform-targeted builds for API version < 1.35' ) - builder = self.client if not cli else _CLIBuilder(progress) + builder = self.client if not cli else _CLIBuilder(progress, + self.options.get('environment'), + build_opts.get('secrets')) build_output = builder.build( path=path, tag=self.image_name, @@ -1717,8 +1719,12 @@ def rewrite_build_path(path): class _CLIBuilder(object): - def __init__(self, progress): + def __init__(self, progress, environment=None, secrets=None): self._progress = progress + if environment is None: + environment = {} + self._environment = environment + self._secrets = secrets def build(self, path, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, timeout=None, @@ -1785,6 +1791,11 @@ def build(self, path, tag=None, quiet=False, fileobj=None, dockerfile = os.path.join(path, dockerfile) iidfile = tempfile.mktemp() + sub_env = merge_environment( + os.environ.copy(), + self._environment + ) + command_builder = _CommandBuilder() command_builder.add_params("--build-arg", buildargs) command_builder.add_list("--cache-from", cache_from) @@ -1797,11 +1808,17 @@ def build(self, path, tag=None, quiet=False, fileobj=None, command_builder.add_arg("--tag", tag) command_builder.add_arg("--target", target) command_builder.add_arg("--iidfile", iidfile) + + if "DOCKER_BUILDKIT" in sub_env and self._secrets is not None: + buildkit_secrets = self.get_buildkit_mounts(mount_spec=self._secrets) + if buildkit_secrets is not None: + command_builder.add_list('--secret', buildkit_secrets) + args = command_builder.build([path]) magic_word = "Successfully built " appear = False - with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: + with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, env=sub_env) as p: while True: line = p.stdout.readline() if not line: @@ -1825,6 +1842,16 @@ def build(self, path, tag=None, quiet=False, fileobj=None, if not appear: yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) + def get_buildkit_mounts(self, mount_spec=None): + if mount_spec is None: + return None + mounts = [] + for spec in mount_spec: + if isinstance(spec, six.string_types): + spec = {'id': spec} + mounts.append(','.join(["{}={}".format(k, v) for (k, v) in spec.items()])) + return mounts + class _CommandBuilder(object): def __init__(self): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a03d56567c2..c7f91c3dcd0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -877,6 +877,28 @@ def test_build_override_dir(self): assert 'Successfully built' in result.stdout + @mock.patch.dict(os.environ) + def test_build_with_secrets(self): + os.environ["COMPOSE_DOCKER_CLI_BUILD"] = "1" + os.environ["DOCKER_BUILDKIT"] = "1" + self.base_dir = 'tests/fixtures/build-secrets' + build_result = self.dispatch(['build', '--build-arg', 'CACHEBUST=1']) + assert 'Successfully built' in build_result.stdout + run_result = self.dispatch(['run', 'foo']) + assert 'secret 1' in run_result.stdout + assert 'secret 3' in run_result.stdout + + @mock.patch.dict(os.environ) + def test_build_with_secrets_substitution(self): + os.environ["COMPOSE_DOCKER_CLI_BUILD"] = "1" + os.environ["DOCKER_BUILDKIT"] = "1" + os.environ["FOO_SECRET"] = "secret_2" + self.base_dir = 'tests/fixtures/build-secrets' + build_result = self.dispatch(['build', '--build-arg', 'CACHEBUST=2']) + assert 'Successfully built' in build_result.stdout + run_result = self.dispatch(['run', 'foo']) + assert 'secret 2' in run_result.stdout + def test_build_override_dir_invalid_path(self): config_path = os.path.abspath('tests/fixtures/build-path-override-dir/docker-compose.yml') result = self.dispatch([ diff --git a/tests/fixtures/build-secrets/Dockerfile b/tests/fixtures/build-secrets/Dockerfile new file mode 100644 index 00000000000..db5ce8f04f0 --- /dev/null +++ b/tests/fixtures/build-secrets/Dockerfile @@ -0,0 +1,9 @@ +# syntax=docker/dockerfile:1.0.0-experimental +FROM busybox + +# https://github.com/moby/moby/issues/1996#issuecomment-550020843 +ARG CACHEBUST + +RUN --mount=type=secret,target=/secret,required cp secret out +RUN --mount=type=secret,target=/secret_3,required cat secret_3 >> out +CMD cat out diff --git a/tests/fixtures/build-secrets/docker-compose.yml b/tests/fixtures/build-secrets/docker-compose.yml new file mode 100644 index 00000000000..56e884c0629 --- /dev/null +++ b/tests/fixtures/build-secrets/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.7' +services: + foo: + build: + context: . + secrets: + - + id: secret + src: "${FOO_SECRET:-secret_1}" + - secret_3 diff --git a/tests/fixtures/build-secrets/secret_1 b/tests/fixtures/build-secrets/secret_1 new file mode 100644 index 00000000000..a64bbd647ac --- /dev/null +++ b/tests/fixtures/build-secrets/secret_1 @@ -0,0 +1 @@ +secret 1 diff --git a/tests/fixtures/build-secrets/secret_2 b/tests/fixtures/build-secrets/secret_2 new file mode 100644 index 00000000000..b71ba7d0f9c --- /dev/null +++ b/tests/fixtures/build-secrets/secret_2 @@ -0,0 +1 @@ +secret 2 diff --git a/tests/fixtures/build-secrets/secret_3 b/tests/fixtures/build-secrets/secret_3 new file mode 100644 index 00000000000..6e9349d38e6 --- /dev/null +++ b/tests/fixtures/build-secrets/secret_3 @@ -0,0 +1 @@ +secret 3 diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index c50aab08bb2..289a591fa10 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -7,6 +7,7 @@ import tempfile from distutils.spawn import find_executable from os import path +from textwrap import dedent import pytest from docker.errors import APIError @@ -973,12 +974,49 @@ def test_build_cli(self): self.addCleanup(shutil.rmtree, base_dir) with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: - f.write("FROM busybox\n") + # f.write("FROM busybox\n") + f.write(dedent("""\ + # syntax=docker/dockerfile:1.0.0-experimental + FROM busybox + RUN --mount=type=secret,target=/secret_1 true + """)) service = self.create_service('web', build={'context': base_dir}, environment={ - 'COMPOSE_DOCKER_CLI_BUILD': '1', + 'DOCKER_BUILDKIT': '1', + }) + service.build(cli=True) + self.addCleanup(self.client.remove_image, service.image_name) + assert self.client.inspect_image('composetest_web') + + def test_build_cli_with_secrets(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + secret_path_1 = os.path.join(base_dir, 'secret_1') + secret_path_2 = os.path.join(base_dir, 'secret_2') + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + # f.write("FROM busybox\n") + f.write(dedent("""\ + # syntax=docker/dockerfile:1.0.0-experimental + FROM busybox + RUN --mount=type=secret,target=/secret_1,required cat /secret_1 + RUN --mount=type=secret,target=/secret_2,required cat /secret_2 + """)) + with open(secret_path_1, 'w') as f: + f.write("secret 1\n") + with open(secret_path_2, 'w') as f: + f.write("secret 2\n") + + service = self.create_service('web', + build={ + 'context': base_dir, + 'secrets': [ + {'id': 'secret_1', 'src': secret_path_1}, + {'id': 'secret_2', 'src': secret_path_2}] + }, + environment={ 'DOCKER_BUILDKIT': '1', }) service.build(cli=True) @@ -992,12 +1030,7 @@ def test_up_build_cli(self): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") - web = self.create_service('web', - build={'context': base_dir}, - environment={ - 'COMPOSE_DOCKER_CLI_BUILD': '1', - 'DOCKER_BUILDKIT': '1', - }) + web = self.create_service('web', build={'context': base_dir}) project = Project('composetest', [web], self.client) project.up(do_build=BuildAction.force)