From 8cb22724f05eb0528fa8957fd41c867c4060133a Mon Sep 17 00:00:00 2001 From: Andreas Zuber Date: Mon, 13 May 2019 13:35:29 +0200 Subject: [PATCH 1/3] make it possible to configure the tasks in the rakefile --- lib/ra10ke.rb | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/ra10ke.rb b/lib/ra10ke.rb index 5adabff..a1274d6 100644 --- a/lib/ra10ke.rb +++ b/lib/ra10ke.rb @@ -7,7 +7,17 @@ module Ra10ke class RakeTask < ::Rake::TaskLib + attr_accessor :basedir, :moduledir, :puppetfile_path, :puppetfile_name, :force + def initialize(*args) + @basedir = Dir.pwd + @moduledir = nil + @puppetfile_path = nil + @puppetfile_name = nil + @force = nil + + yield(self) if block_given? + namespace :r10k do desc "Print outdated forge modules" task :dependencies do @@ -15,7 +25,7 @@ def initialize(*args) require 'puppet_forge' PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}" - puppetfile = R10K::Puppetfile.new(Dir.pwd) + puppetfile = get_puppetfile puppetfile.load! PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/ @@ -81,9 +91,9 @@ def initialize(*args) require 'r10k/action/puppetfile/check' puppetfile = R10K::Action::Puppetfile::Check.new({ - :root => Dir.pwd, - :moduledir => nil, - :puppetfile => nil + :root => @basedir, + :moduledir => @moduledir, + :puppetfile => @puppetfile_path }, '') unless puppetfile.call @@ -92,22 +102,10 @@ def initialize(*args) end desc "Install modules specified in Puppetfile" - task :install, [:path] do |_, args| + task :install do require 'r10k/puppetfile' - require 'pathname' - - if !args.has_key?(:path) - raise "task requires 'path' argument" - end - modpath = Pathname.new args[:path] - if !modpath.absolute? - modpath = Pathname.new(Dir.pwd) + modpath - end - - puppetfile_dir = Pathname.new Dir.pwd - modules_dir = File.join(puppetfile_dir, modpath.relative_path_from(puppetfile_dir)) - puppetfile = R10K::Puppetfile.new(puppetfile_dir, modules_dir) + puppetfile = get_puppetfile puppetfile.load! puts "Processing Puppetfile for fixtures" @@ -132,7 +130,9 @@ def initialize(*args) end end + + def get_puppetfile + R10K::Puppetfile.new(@basedir, @moduledir, @puppetfile_path, @puppetfile_name, @force) + end end end - -Ra10ke::RakeTask.new From b990f319a87f4a7ee9a6107c5e09b075954d9181 Mon Sep 17 00:00:00 2001 From: Andreas Zuber Date: Wed, 15 May 2019 14:48:07 +0200 Subject: [PATCH 2/3] move solve_dependencies to a define method The solve_dependencie task was moved to a define method which is then called from the main class. This way it can use the settings defined in the main class. --- lib/ra10ke.rb | 4 + lib/ra10ke/solve.rb | 374 ++++++++++++++++++++++---------------------- 2 files changed, 189 insertions(+), 189 deletions(-) diff --git a/lib/ra10ke.rb b/lib/ra10ke.rb index a1274d6..00c9a95 100644 --- a/lib/ra10ke.rb +++ b/lib/ra10ke.rb @@ -7,6 +7,8 @@ module Ra10ke class RakeTask < ::Rake::TaskLib + include Ra10ke::Solve + attr_accessor :basedir, :moduledir, :puppetfile_path, :puppetfile_name, :force def initialize(*args) @@ -19,6 +21,8 @@ def initialize(*args) yield(self) if block_given? namespace :r10k do + define_task_solve_dependencies(*args) + desc "Print outdated forge modules" task :dependencies do require 'r10k/puppetfile' diff --git a/lib/ra10ke/solve.rb b/lib/ra10ke/solve.rb index ab8c957..5e23166 100644 --- a/lib/ra10ke/solve.rb +++ b/lib/ra10ke/solve.rb @@ -11,218 +11,214 @@ FETCH_LIMIT = 3 module Ra10ke::Solve - class RakeTask < ::Rake::TaskLib - def initialize(*_args) - namespace :r10k do - desc 'Find missing or outdated module dependencies' - task :solve_dependencies, [:allow_major_bump] do |_t, args| - require 'r10k/puppetfile' - require 'r10k/module/git' - require 'r10k/module/metadata_file' - require 'puppet_forge' + def define_task_solve_dependencies(*_args) + desc 'Find missing or outdated module dependencies' + task :solve_dependencies, [:allow_major_bump] do |_t, args| + require 'r10k/puppetfile' + require 'r10k/module/git' + require 'r10k/module/metadata_file' + require 'puppet_forge' - allow_major_bump = false - allow_major_bump = true if args[:allow_major_bump] + allow_major_bump = false + allow_major_bump = true if args[:allow_major_bump] - # Same as in the dependencies task, but oh well. - PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}" - puppetfile = R10K::Puppetfile.new(Dir.pwd) - puppetfile.load! - PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/ + # Same as in the dependencies task, but oh well. + PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}" + puppetfile = get_puppetfile + puppetfile.load! + PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/ - # ignore file allows for "don't tell me about this" - ignore_modules = [] - if File.exist?('.r10kignore') - ignore_modules = File.readlines('.r10kignore').each(&:chomp!) - end - # Actual new logic begins here: - cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache')) - # Metadata cache, since the Forge is slow: - @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache') - # The graph of available module versions - @graph = Solve::Graph.new - # Set of modules that we have already added to the graph - @processed_modules = Set.new - # The set of "demands" we make of the solver. Will be a list of module names - # Could also demand certain version constraints to hold, but the code does not do it - # Can be either "module-name" or ["module-name", "version-constraint"] - @demands = Set.new - # List of modules we have in the Puppetfile, as [name, version] pairs - @current_modules = [] - - puppetfile.modules.each do |puppet_module| - next if ignore_modules.include? puppet_module.title - if puppet_module.class == R10K::Module::Forge - module_name = puppet_module.title.tr('/', '-') - installed_version = puppet_module.expected_version - puts "Processing Forge module #{module_name}-#{installed_version}" - @current_modules << [module_name, installed_version] - @graph.artifact(module_name, installed_version) - constraint = '>=0.0.0' - unless allow_major_bump - ver = Semverse::Version.new installed_version - if ver.major.zero? - constraint = "~>#{installed_version}" - else - nver = Semverse::Version.new([ver.major + 1, 0, 0]) - constraint = "<#{nver}" - end - end - puts "...Adding a demand: #{module_name} #{constraint}" + # ignore file allows for "don't tell me about this" + ignore_modules = [] + if File.exist?('.r10kignore') + ignore_modules = File.readlines('.r10kignore').each(&:chomp!) + end + # Actual new logic begins here: + cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache')) + # Metadata cache, since the Forge is slow: + @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache') + # The graph of available module versions + @graph = Solve::Graph.new + # Set of modules that we have already added to the graph + @processed_modules = Set.new + # The set of "demands" we make of the solver. Will be a list of module names + # Could also demand certain version constraints to hold, but the code does not do it + # Can be either "module-name" or ["module-name", "version-constraint"] + @demands = Set.new + # List of modules we have in the Puppetfile, as [name, version] pairs + @current_modules = [] - @demands.add([module_name, constraint]) - puts '...Fetching latest release version information' - forge_rel = PuppetForge::Module.find(module_name).current_release - mod = @graph.artifact(module_name, forge_rel.version) - puts '...Adding its requirements to the graph' - meta = get_release_metadata(module_name, forge_rel) - add_reqs_to_graph(mod, meta) + puppetfile.modules.each do |puppet_module| + next if ignore_modules.include? puppet_module.title + if puppet_module.class == R10K::Module::Forge + module_name = puppet_module.title.tr('/', '-') + installed_version = puppet_module.expected_version + puts "Processing Forge module #{module_name}-#{installed_version}" + @current_modules << [module_name, installed_version] + @graph.artifact(module_name, installed_version) + constraint = '>=0.0.0' + unless allow_major_bump + ver = Semverse::Version.new installed_version + if ver.major.zero? + constraint = "~>#{installed_version}" + else + nver = Semverse::Version.new([ver.major + 1, 0, 0]) + constraint = "<#{nver}" end - - next unless puppet_module.class == R10K::Module::Git - # This downloads the git module to modules/modulename - meta = fetch_git_metadata(puppet_module) - version = get_key_or_sym(meta, :version) - module_name = puppet_module.title.tr('/', '-') - @current_modules << [module_name, version] - # We should add git modules with exact versions, or the system might recommend updating to a - # Forge version. - puts "Adding git module #{module_name} to the list of required modules with exact version: #{version}" - @demands.add([module_name, version]) - mod = @graph.artifact(module_name, version) - puts "...Adding requirements for git module #{module_name}-#{version}" - add_reqs_to_graph(mod, meta) end - puts - puts 'Resolving dependencies...' - if allow_major_bump - puts 'WARNING: Potentially breaking updates are allowed for this resolution' - end - result = Solve.it!(@graph, @demands, sorted: true) - puts - print_module_diff(@current_modules, result) + puts "...Adding a demand: #{module_name} #{constraint}" + + @demands.add([module_name, constraint]) + puts '...Fetching latest release version information' + forge_rel = PuppetForge::Module.find(module_name).current_release + mod = @graph.artifact(module_name, forge_rel.version) + puts '...Adding its requirements to the graph' + meta = get_release_metadata(module_name, forge_rel) + add_reqs_to_graph(mod, meta) end + + next unless puppet_module.class == R10K::Module::Git + # This downloads the git module to modules/modulename + meta = fetch_git_metadata(puppet_module) + version = get_key_or_sym(meta, :version) + module_name = puppet_module.title.tr('/', '-') + @current_modules << [module_name, version] + # We should add git modules with exact versions, or the system might recommend updating to a + # Forge version. + puts "Adding git module #{module_name} to the list of required modules with exact version: #{version}" + @demands.add([module_name, version]) + mod = @graph.artifact(module_name, version) + puts "...Adding requirements for git module #{module_name}-#{version}" + add_reqs_to_graph(mod, meta) + end + puts + puts 'Resolving dependencies...' + if allow_major_bump + puts 'WARNING: Potentially breaking updates are allowed for this resolution' end + result = Solve.it!(@graph, @demands, sorted: true) + puts + print_module_diff(@current_modules, result) end + end - def get_release_metadata(name, release) - meta = nil - @metadata_cache.transaction do - meta = @metadata_cache["#{name}-#{release.version}"] - unless meta - meta = release.metadata - @metadata_cache["#{name}-#{release.version}"] = meta - end + private + + def get_release_metadata(name, release) + meta = nil + @metadata_cache.transaction do + meta = @metadata_cache["#{name}-#{release.version}"] + unless meta + meta = release.metadata + @metadata_cache["#{name}-#{release.version}"] = meta end - meta end + meta + end - def fetch_git_metadata(puppet_module) - # No caching here. I don't think it's really possible to do in a sane way. - puts "Fetching git module #{puppet_module.title}, saving to modules/" - puppet_module.sync - metadata_path = Pathname.new(puppet_module.full_path) + 'metadata.json' - unless metadata_path.exist? - puts 'WARNING: metadata.json does not exist, assuming version 0.0.0 and no dependencies' - return { - version: '0.0.0', - name: puppet_module.title, - dependencies: [] - } - end - metadata = R10K::Module::MetadataFile.new(metadata_path) - metadata = metadata.read - { - version: metadata.version, - name: metadata.name, - dependencies: metadata.dependencies + def fetch_git_metadata(puppet_module) + # No caching here. I don't think it's really possible to do in a sane way. + puts "Fetching git module #{puppet_module.title}, saving to modules/" + puppet_module.sync + metadata_path = Pathname.new(puppet_module.full_path) + 'metadata.json' + unless metadata_path.exist? + puts 'WARNING: metadata.json does not exist, assuming version 0.0.0 and no dependencies' + return { + version: '0.0.0', + name: puppet_module.title, + dependencies: [] } end + metadata = R10K::Module::MetadataFile.new(metadata_path) + metadata = metadata.read + { + version: metadata.version, + name: metadata.name, + dependencies: metadata.dependencies + } + end - # Is there a better way? :( - def get_key_or_sym(hash, k) - hash.fetch(k.to_sym, hash.fetch(k.to_s, nil)) - end + # Is there a better way? :( + def get_key_or_sym(hash, k) + hash.fetch(k.to_sym, hash.fetch(k.to_s, nil)) + end - # At least puppet-extlib has malformed metadata - def get_version_req(dep) - req = get_key_or_sym(dep, :version_requirement) - req = get_key_or_sym(dep, :version_range) unless req - req - end + # At least puppet-extlib has malformed metadata + def get_version_req(dep) + req = get_key_or_sym(dep, :version_requirement) + req = get_key_or_sym(dep, :version_range) unless req + req + end - def print_module_diff(current, resolution) - current.sort! - resolution.sort! - outdated = [] - missing = [] - resolution.each do |mod| - cur_mod, cur_version = current.shift - mod, version = mod - if (cur_mod == mod) && cur_version && (cur_version != version) - outdated << [mod, cur_version, version] - elsif cur_mod != mod - missing << [mod, version] - current.unshift [cur_mod, cur_version] - end - end - missing.each do |m| - puts format('MISSING: %-25s %s', *m) - end - outdated.each do |o| - puts format('OUTDATED: %-25s %s -> %s', *o) + def print_module_diff(current, resolution) + current.sort! + resolution.sort! + outdated = [] + missing = [] + resolution.each do |mod| + cur_mod, cur_version = current.shift + mod, version = mod + if (cur_mod == mod) && cur_version && (cur_version != version) + outdated << [mod, cur_version, version] + elsif cur_mod != mod + missing << [mod, version] + current.unshift [cur_mod, cur_version] end end + missing.each do |m| + puts format('MISSING: %-25s %s', *m) + end + outdated.each do |o| + puts format('OUTDATED: %-25s %s -> %s', *o) + end + end - def add_reqs_to_graph(artifact, metadata, no_demands = nil) - deps = get_key_or_sym(metadata, :dependencies) - my_name = get_key_or_sym(metadata, :name) - deps.each do |dep| - name = get_key_or_sym(dep, :name).tr('/', '-') - # Add dependency to the global set of modules we want, so that we can - # actually ask the solver for the versioned thing - @demands.add(name) unless no_demands - ver = get_version_req(dep) - unless ver - # no version specified, so anything will do - ver = '>=0.0.0' - end - ver.split(/(?=[<])/).each do |bound| - bound.strip! - v = begin - Semverse::Constraint.new(bound) - rescue - nil - end - if v - artifact.depends(name, v.to_s) - else - puts "WARNING: Invalid version constraint: #{bound}" - end - end - # Find the dependency in the forge, unless it's already been processed - # and add its releases to the global graph - next unless @processed_modules.add?(name) - puts "Fetching module info for #{name}" - mod = begin - PuppetForge::Module.find(name) - rescue - # It's probably a git module - nil - end - next unless mod # Git module, or non-forge dependency. Skip to next for now. - # Fetching metadata for all releases takes ages (which is weird, since it's mostly static info) - mod.releases.take(FETCH_LIMIT).each do |rel| - meta = get_release_metadata(name, rel) - rel_artifact = @graph.artifact(name, rel.version) - puts "...Recursively adding requirements for dependency #{name} version #{rel.version}" - # We don't want to add the requirements to the list of demands for all versions, - # but we need them in the graph to be able to solve dependencies - add_reqs_to_graph(rel_artifact, meta, :no_demands) + def add_reqs_to_graph(artifact, metadata, no_demands = nil) + deps = get_key_or_sym(metadata, :dependencies) + my_name = get_key_or_sym(metadata, :name) + deps.each do |dep| + name = get_key_or_sym(dep, :name).tr('/', '-') + # Add dependency to the global set of modules we want, so that we can + # actually ask the solver for the versioned thing + @demands.add(name) unless no_demands + ver = get_version_req(dep) + unless ver + # no version specified, so anything will do + ver = '>=0.0.0' + end + ver.split(/(?=[<])/).each do |bound| + bound.strip! + v = begin + Semverse::Constraint.new(bound) + rescue + nil + end + if v + artifact.depends(name, v.to_s) + else + puts "WARNING: Invalid version constraint: #{bound}" end end + # Find the dependency in the forge, unless it's already been processed + # and add its releases to the global graph + next unless @processed_modules.add?(name) + puts "Fetching module info for #{name}" + mod = begin + PuppetForge::Module.find(name) + rescue + # It's probably a git module + nil + end + next unless mod # Git module, or non-forge dependency. Skip to next for now. + # Fetching metadata for all releases takes ages (which is weird, since it's mostly static info) + mod.releases.take(FETCH_LIMIT).each do |rel| + meta = get_release_metadata(name, rel) + rel_artifact = @graph.artifact(name, rel.version) + puts "...Recursively adding requirements for dependency #{name} version #{rel.version}" + # We don't want to add the requirements to the list of demands for all versions, + # but we need them in the graph to be able to solve dependencies + add_reqs_to_graph(rel_artifact, meta, :no_demands) + end end end end - -Ra10ke::Solve::RakeTask.new From 3980243332c4843bb01e9796fd7dcf003fd20c25 Mon Sep 17 00:00:00 2001 From: Andreas Zuber Date: Wed, 15 May 2019 15:08:30 +0200 Subject: [PATCH 3/3] Add settings doc in Readme --- README.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c296a4d..dc78f4c 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,34 @@ Add the following line to your `Gemfile`: gem 'ra10ke' ``` -Add the following line in your `Rakefile`: +Add the following lines in your `Rakefile`: ```ruby require 'ra10ke' +Ra10ke::RakeTask.new ``` +## Configuration + +You can configure the tasks in a block: + +```ruby +Ra10ke::RakeTask.new do |t| + t.basedir = File.join(Dir.pwd, 'some_dir') + t.moduledir = File.join(Dir.pwd, 'some_dir/strange_module_dir') +end +``` + +Available settings are: + +| Setting | Documentation | +|-----------------|-----------------------------------------------------------------------------------------------| +| basedir | Base directory with the Puppetfile and modules directory (Default: Same directory as Rakefile)| +| moduledir | Directory to install the modules in (Default: 'modules' in basedir) | +| puppetfile_path | Directroy where the Puppetfile is (Default: basedir) | +| puppetfile_name | The Puppetfile name (Default: basedir/Puppetfile) | +| force | Overwrite locally changed files on install (Default: false) | + ## Rake tasks ### r10k:syntax