From a67a2f70e6e97439f692a0798f0522d86138a13d Mon Sep 17 00:00:00 2001 From: MarkW Date: Wed, 26 Sep 2018 16:50:11 +0100 Subject: [PATCH] CLOUD-2069 Adding support for multiple compose files. (#332) Support for multiple compose files --- README.md | 16 +++- lib/puppet/provider/docker_compose/ruby.rb | 98 +++++++++++----------- lib/puppet/type/docker_compose.rb | 16 ++-- spec/acceptance/compose_spec.rb | 13 +-- spec/acceptance/compose_v3_spec.rb | 66 ++++++++++----- spec/spec_helper_acceptance.rb | 22 ++++- 6 files changed, 147 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index dca90a02..34907240 100755 --- a/README.md +++ b/README.md @@ -554,7 +554,8 @@ compose_test: Specify the `file` resource to add a Compose file to the machine you have Puppet running on. To define a `docker_compose` resource pointing to the Compose file, add the following code to the manifest file: ```puppet -docker_compose { '/tmp/docker-compose.yml': +docker_compose { 'test': + compose_files => ['/tmp/docker-compose.yml'], ensure => present, } ``` @@ -564,7 +565,8 @@ Puppet automatically runs Compose, because the relevant Compose services aren't In the example below, Puppet runs Compose when the number of containers specified for a service don't match the scale values. ```puppet -docker_compose { '/tmp/docker-compose.yml': +docker_compose { 'test': + compose_files => ['/tmp/docker-compose.yml'], ensure => present, scale => { 'compose_test' => 2, @@ -575,6 +577,16 @@ docker_compose { '/tmp/docker-compose.yml': Give options to the ```docker-compose up``` command, such as ```--remove-orphans```, by using the ```up_args``` option. +To supply multiple overide compose files add the following to the manifest file: + +```puppet +docker_compose {'test': + compose_files => ['master-docker-compose.yml', 'override-compose.yml], +} +``` + +Please note you should supply your master docker-compose file as the first element in the array. As per docker multi compose file support compose files will be merged in the order they are specified in the array. + If you are using a v3.2 compose file or above on a Docker Swarm cluster, use the `docker::stack` class. Include the file resource before you run the stack command. To deploy the stack, add the following code to the manifest file: diff --git a/lib/puppet/provider/docker_compose/ruby.rb b/lib/puppet/provider/docker_compose/ruby.rb index f7ef9a74..e4c9837e 100644 --- a/lib/puppet/provider/docker_compose/ruby.rb +++ b/lib/puppet/provider/docker_compose/ruby.rb @@ -6,45 +6,47 @@ commands docker: 'docker' def exists? - Puppet.info("Checking for compose project #{project}") - compose_file = YAML.safe_load(File.read(name)) + Puppet.info("Checking for compose project #{name}") compose_services = {} - containers = docker([ - 'ps', - '--format', - "{{.Label \"com.docker.compose.service\"}}-{{.Image}}", - '--filter', - "label=com.docker.compose.project=#{project}", - ]).split("\n") - case compose_file['version'] - when %r{^2(\.[0-3])?$}, %r{^3(\.[0-6])?$} - compose_services = compose_file['services'] - # in compose v1 "version" parameter is not specified - when nil - compose_services = compose_file - else - raise(Puppet::Error, "Unsupported docker compose file syntax version \"#{compose_file['version']}\"!") - end + resource[:compose_files].each do |file| + compose_file = YAML.safe_load(File.read(file)) + containers = docker([ + 'ps', + '--format', + "{{.Label \"com.docker.compose.service\"}}-{{.Image}}", + '--filter', + "label=com.docker.compose.project=#{name}", + ]).split("\n") + case compose_file['version'] + when %r{^2(\.[0-3])?$}, %r{^3(\.[0-6])?$} + compose_services = compose_services.merge(compose_file['services']) + # in compose v1 "version" parameter is not specified + when nil + compose_services = compose_services.merge(compose_file) + else + raise(Puppet::Error, "Unsupported docker compose file syntax version \"#{compose_file['version']}\"!") + end - if compose_services.count != containers.count - return false - end + if compose_services.count != containers.count + return false + end - counts = Hash[*compose_services.each.map { |key, array| - image = (array['image']) ? array['image'] : get_image(key, compose_services) - Puppet.info("Checking for compose service #{key} #{image}") - ["#{key}-#{image}", containers.count("#{key}-#{image}")] - }.flatten] + counts = Hash[*compose_services.each.map { |key, array| + image = (array['image']) ? array['image'] : get_image(key, compose_services) + Puppet.info("Checking for compose service #{key} #{image}") + ["#{key}-#{image}", containers.count("#{key}-#{image}")] + }.flatten] - # No containers found for the project - if counts.empty? || - # Containers described in the compose file are not running - counts.any? { |_k, v| v.zero? } || - # The scaling factors in the resource do not match the number of running containers - resource[:scale] && counts.merge(resource[:scale]) != counts - false - else - true + # No containers found for the project + if counts.empty? || + # Containers described in the compose file are not running + counts.any? { |_k, v| v.zero? } || + # The scaling factors in the resource do not match the number of running containers + resource[:scale] && counts.merge(resource[:scale]) != counts + false + else + true + end end end @@ -61,37 +63,37 @@ def get_image(service_name, compose_services) end def create - Puppet.info("Running compose project #{project}") - args = ['-f', name, 'up', '-d', '--remove-orphans'].insert(2, resource[:options]).insert(5, resource[:up_args]).compact + Puppet.info("Running compose project #{name}") + args = [compose_files, '-p', name, 'up', '-d', '--remove-orphans'].insert(2, resource[:options]).insert(5, resource[:up_args]).compact dockercompose(args) return unless resource[:scale] instructions = resource[:scale].map { |k, v| "#{k}=#{v}" } Puppet.info("Scaling compose project #{project}: #{instructions.join(' ')}") - args = ['-f', name, 'scale'].insert(2, resource[:options]).compact + instructions + args = [compose_files, '-p', name, 'scale'].insert(2, resource[:options]).compact + instructions dockercompose(args) end def destroy - Puppet.info("Removing all containers for compose project #{project}") - kill_args = ['-f', name, 'kill'].insert(2, resource[:options]).compact + Puppet.info("Removing all containers for compose project #{name}") + kill_args = [compose_files, '-p', name, 'kill'].insert(2, resource[:options]).compact dockercompose(kill_args) - rm_args = ['-f', name, 'rm', '--force', '-v'].insert(2, resource[:options]).compact + rm_args = [compose_files, '-p', name, 'rm', '--force', '-v'].insert(2, resource[:options]).compact dockercompose(rm_args) end def restart return unless exists? - Puppet.info("Rebuilding and Restarting all containers for compose project #{project}") - kill_args = ['-f', name, 'kill'].insert(2, resource[:options]).compact + Puppet.info("Rebuilding and Restarting all containers for compose project #{name}") + kill_args = [compose_files, '-p', name, 'kill'].insert(2, resource[:options]).compact dockercompose(kill_args) - build_args = ['-f', name, 'build'].insert(2, resource[:options]).compact + build_args = [compose_files, '-p', name, 'build'].insert(2, resource[:options]).compact dockercompose(build_args) create end - private - - def project - File.basename(File.dirname(name)).downcase.gsub(%r{[^0-9a-z ]}i, '') + def compose_files + resource[:compose_files].map { |x| ['-f', x] }.flatten end + + private end diff --git a/lib/puppet/type/docker_compose.rb b/lib/puppet/type/docker_compose.rb index 7f720e72..47628ca2 100644 --- a/lib/puppet/type/docker_compose.rb +++ b/lib/puppet/type/docker_compose.rb @@ -7,10 +7,6 @@ def refresh provider.restart end - newparam(:name) do - desc 'Docker compose file path.' - end - newparam(:scale) do desc 'A hash of compose services and number of containers.' validate do |value| @@ -38,7 +34,15 @@ def refresh end end - autorequire(:file) do - self[:name] + newparam(:compose_files, :array_matching => :all) do + desc 'An array of Docker Compose Files paths.' + validate do |value| + raise _('compose files should be an array') unless value.is_a? Array + end + end + + newparam(:name) do + isnamevar + desc 'The name of the project' end end diff --git a/spec/acceptance/compose_spec.rb b/spec/acceptance/compose_spec.rb index 80fe50a0..07813712 100644 --- a/spec/acceptance/compose_spec.rb +++ b/spec/acceptance/compose_spec.rb @@ -25,7 +25,8 @@ class { 'docker::compose': } context 'Creating compose projects' do before(:all) do @install = <<-code -docker_compose { '/tmp/docker-compose.yml': +docker_compose { 'test3': + compose_files => ['#{tmp_path}/docker-compose.yml'], ensure => present, } code @@ -36,7 +37,7 @@ class { 'docker::compose': } apply_manifest(@install, :catch_changes=>true) end - describe command("docker inspect tmp_compose_test_1"), :sudo => true do + describe command("docker inspect test3_compose_test_1"), :sudo => true do its(:exit_status) { should eq 0 } end end @@ -44,13 +45,15 @@ class { 'docker::compose': } context 'Destroying compose projects' do before(:all) do install = <<-code -docker_compose { '/tmp/docker-compose.yml': +docker_compose { 'test4': + compose_files => ['#{tmp_path}/docker-compose.yml'], ensure => present, } code apply_manifest(install, :catch_failures=>true) @uninstall = <<-code -docker_compose { '/tmp/docker-compose.yml': +docker_compose { 'test4': + compose_files => ['#{tmp_path}/docker-compose.yml'], ensure => absent, } code @@ -61,7 +64,7 @@ class { 'docker::compose': } apply_manifest(@uninstall, :catch_changes=>true) end - describe command("docker inspect tmp_compose_test_1"), :sudo => true do + describe command("docker inspect test4_compose_test_1"), :sudo => true do its(:exit_status) { should eq 1 } end end diff --git a/spec/acceptance/compose_v3_spec.rb b/spec/acceptance/compose_v3_spec.rb index 81caf397..6adf6c70 100644 --- a/spec/acceptance/compose_v3_spec.rb +++ b/spec/acceptance/compose_v3_spec.rb @@ -5,11 +5,13 @@ file_extension = '.exe' docker_args = 'docker_ee => true' tmp_path = 'C:/cygwin64/tmp' + test_container = 'nanoserver-sac2016' else install_dir = '/usr/local/bin' file_extension = '' docker_args = '' tmp_path = '/tmp' + test_container = 'debian' end describe 'docker compose' do @@ -29,10 +31,10 @@ class { 'docker::compose': it 'should have docker compose installed' do shell('docker-compose --help', :acceptable_exit_codes => [0]) end - before(:all) do @install = <<-code -docker_compose { '#{tmp_path}/docker-compose-v3.yml': +docker_compose { 'web': + compose_files => ['#{tmp_path}/docker-compose-v3.yml'], ensure => present, } code @@ -44,32 +46,52 @@ class { 'docker::compose': end it 'should find a docker container' do - shell('docker inspect tmp_compose_test_1', :acceptable_exit_codes => [0]) + shell('docker inspect web_compose_test_1', :acceptable_exit_codes => [0]) end end - context 'Destroying compose v3 projects' do + context 'creating compose projects with multi compose files' do before(:all) do - install = <<-code -docker_compose { '#{tmp_path}/docker-compose-v3.yml': + @install = <<-pp1 +docker_compose { 'web1': + compose_files => ['#{tmp_path}/docker-compose-v3.yml', '#{tmp_path}/docker-compose-override-v3.yml'], ensure => present, } - code - apply_manifest(install, :catch_failures=>true) - @uninstall = <<-code -docker_compose { '#{tmp_path}/docker-compose-v3.yml': + pp1 + + apply_manifest(@install, :catch_failures=>true) + end + + it "should find container with #{test_container} tag" do + shell("docker inspect web1_compose_test_1 | grep #{test_container}", :acceptable_exit_codes => [0]) + end + end + + context 'Destroying project with multiple compose files' do + before(:all) do + @install = <<-pp1 +docker_compose { 'web1': + compose_files => ['#{tmp_path}/docker-compose-v3.yml', '#{tmp_path}/docker-compose-override-v3.yml'], + ensure => present, +} + pp1 + + @destroy = <<-pp2 +docker_compose { 'web1': + compose_files => ['#{tmp_path}/docker-compose-v3.yml', '#{tmp_path}/docker-compose-override-v3.yml'], ensure => absent, } - code - apply_manifest(@uninstall, :catch_failures=>true) + pp2 + apply_manifest(@install, :catch_failures=>true) + apply_manifest(@destroy, :catch_failures=>true) end it 'should be idempotent' do - apply_manifest(@uninstall, :catch_changes=>true) + apply_manifest(@destroy, :catch_changes=>true) end it 'should not find a docker container' do - shell('docker inspect tmp_compose_test_1', :acceptable_exit_codes => [1]) + shell('docker inspect web1_compose_test_1', :acceptable_exit_codes => [1]) end end @@ -77,9 +99,9 @@ class { 'docker::compose': before(:all) do @version = '1.21.2' @pp = <<-code - class { 'docker::compose': - version => '#{@version}', - } +class { 'docker::compose': + version => '#{@version}', +} code apply_manifest(@pp, :catch_failures=>true) end @@ -99,10 +121,10 @@ class { 'docker::compose': before(:all) do @version = '1.21.2' @pp = <<-code - class { 'docker::compose': - ensure => absent, - version => '#{@version}', - } +class { 'docker::compose': + ensure => absent, + version => '#{@version}', +} code apply_manifest(@pp, :catch_failures=>true) end @@ -124,4 +146,4 @@ class { 'docker::compose': } apply_manifest(install_code, :catch_failures=>true) end end -end + end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 26ad97ad..d77427ba 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -104,18 +104,36 @@ def retry_on_error_matching(max_retry_count = 3, retry_wait_interval_secs = 5, e command: /bin/sh -c "while true; do echo hello world; sleep 1; done" EOS docker_compose_content_v3 = <<-EOS -version: "3" +version: "3.4" services: compose_test: image: ubuntu:14.04 command: /bin/sh -c "while true; do echo hello world; sleep 1; done" EOS + docker_compose_override_v3 = <<-EOS +version: "3.4" +services: + compose_test: + image: debian:jessie + command: /bin/sh -c "while true; do echo hello world; sleep 1; done" + EOS docker_compose_content_v3_windows = <<-EOS version: "3" services: compose_test: image: hello-world:nanoserver command: cmd.exe /C "ping /t 8.8.8.8" +networks: + default: + external: + name: nat + EOS + docker_compose_override_v3_windows = <<-EOS +version: "3" +services: + compose_test: + image: hello-world:nanoserver-sac2016 + command: cmd.exe /C "ping /t 8.8.8.8" networks: default: external: @@ -125,8 +143,10 @@ def retry_on_error_matching(max_retry_count = 3, retry_wait_interval_secs = 5, e create_remote_file(host, "/tmp/docker-compose-v2.yml", docker_compose_content_v2) if fact_on(host, 'osfamily') == 'windows' create_remote_file(host, "/tmp/docker-compose-v3.yml", docker_compose_content_v3_windows) + create_remote_file(host, "/tmp/docker-compose-override-v3.yml", docker_compose_override_v3_windows) else create_remote_file(host, "/tmp/docker-compose-v3.yml", docker_compose_content_v3) + create_remote_file(host, "/tmp/docker-compose-override-v3.yml", docker_compose_override_v3) end if fact_on(host, 'osfamily') == 'windows'