diff --git a/README.md b/README.md index da4e0ef..a640c05 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,18 @@ Ignoring specific modules: Under specific conditions you may not wish to report on specific modules being out of date, to ignore a module create `.r10kignore` file in the same directory as your Puppetfile. +### r10k:solve_dependencies + +Reads the Puppetfile in the current directory and uses the ruby 'solve' library to find +missing and outdated dependencies based on their metadata. + +The solver does not allow major version bumps according to SemVer by default. To allow +major upgrades, call the rake task with any parameter. + +The rake task will download git modules into the modules/ directory to access their metadata.json. +It will also cache forge metadata in ̃$XDG_CACHE_DIR/ra10ke.metadata_cache in order to make subsequent +runs faster. + #### Limitations * It works only with modules from the [Forge](https://forge.puppetlabs.com), and Git. diff --git a/lib/ra10ke.rb b/lib/ra10ke.rb index 5e174d2..41dfcb1 100644 --- a/lib/ra10ke.rb +++ b/lib/ra10ke.rb @@ -1,6 +1,7 @@ require 'rake' require 'rake/tasklib' require 'ra10ke/version' +require 'ra10ke/solve' require 'git' module Ra10ke diff --git a/lib/ra10ke/solve.rb b/lib/ra10ke/solve.rb new file mode 100644 index 0000000..207cf66 --- /dev/null +++ b/lib/ra10ke/solve.rb @@ -0,0 +1,227 @@ +require 'rake' +require 'rake/tasklib' +require 'ra10ke/version' +require 'git' +require 'set' +require 'solve' +require 'yaml/store' +require 'semverse/version' + +# How many versions to fetch from the Forge, at most +FETCH_LIMIT = 3 + +module Ra10ke::Solve + class RakeTask < ::Rake::TaskLib + def initialize(*_args) + namespace :r10k do + desc 'Find missing or outdated module dependencies' + task :solve_dependencies, [:allow_major_bump] do |_t, args| + require 'r10k/puppetfile' + require 'r10k/module/git' + require 'r10k/module/metadata_file' + require 'puppet_forge' + + allow_major_bump = false + allow_major_bump = true if args[:allow_major_bump] + + # Same as in the dependencies task, but oh well. + PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}" + puppetfile = R10K::Puppetfile.new('.') + puppetfile.load! + + # ignore file allows for "don't tell me about this" + ignore_modules = [] + if File.exist?('.r10kignore') + ignore_modules = File.readlines('.r10kignore').each(&:chomp!) + end + # Actual new logic begins here: + cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache')) + # Metadata cache, since the Forge is slow: + @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache') + # The graph of available module versions + @graph = Solve::Graph.new + # Set of modules that we have already added to the graph + @processed_modules = Set.new + # The set of "demands" we make of the solver. Will be a list of module names + # Could also demand certain version constraints to hold, but the code does not do it + # Can be either "module-name" or ["module-name", "version-constraint"] + @demands = Set.new + # List of modules we have in the Puppetfile, as [name, version] pairs + @current_modules = [] + + puppetfile.modules.each do |puppet_module| + next if ignore_modules.include? puppet_module.title + if puppet_module.class == R10K::Module::Forge + module_name = puppet_module.title.tr('/', '-') + installed_version = puppet_module.expected_version + puts "Processing Forge module #{module_name}-#{installed_version}" + @current_modules << [module_name, installed_version] + @graph.artifact(module_name, installed_version) + constraint = '>=0.0.0' + unless allow_major_bump + ver = Semverse::Version.new installed_version + if ver.major.zero? + constraint = "~>#{installed_version}" + else + nver = Semverse::Version.new([ver.major + 1, 0, 0]) + constraint = "<#{nver}" + end + end + puts "...Adding a demand: #{module_name} #{constraint}" + + @demands.add([module_name, constraint]) + puts '...Fetching latest release version information' + forge_rel = PuppetForge::Module.find(module_name).current_release + mod = @graph.artifact(module_name, forge_rel.version) + puts '...Adding its requirements to the graph' + meta = get_release_metadata(module_name, forge_rel) + add_reqs_to_graph(mod, meta) + end + + next unless puppet_module.class == R10K::Module::Git + # This downloads the git module to modules/modulename + meta = fetch_git_metadata(puppet_module) + version = get_key_or_sym(meta, :version) + module_name = puppet_module.title.tr('/', '-') + @current_modules << [module_name, version] + # We should add git modules with exact versions, or the system might recommend updating to a + # Forge version. + puts "Adding git module #{module_name} to the list of required modules with exact version: #{version}" + @demands.add([module_name, version]) + mod = @graph.artifact(module_name, version) + puts "...Adding requirements for git module #{module_name}-#{version}" + add_reqs_to_graph(mod, meta) + end + puts + puts 'Resolving dependencies...' + if allow_major_bump + puts 'WARNING: Potentially breaking updates are allowed for this resolution' + end + result = Solve.it!(@graph, @demands, sorted: true) + puts + print_module_diff(@current_modules, result) + end + end + end + + def get_release_metadata(name, release) + meta = nil + @metadata_cache.transaction do + meta = @metadata_cache["#{name}-#{release.version}"] + unless meta + meta = release.metadata + @metadata_cache["#{name}-#{release.version}"] = meta + end + end + meta + end + + def fetch_git_metadata(puppet_module) + # No caching here. I don't think it's really possible to do in a sane way. + puts "Fetching git module #{puppet_module.title}, saving to modules/" + puppet_module.sync + metadata_path = Pathname.new(puppet_module.full_path) + 'metadata.json' + unless metadata_path.exist? + puts 'WARNING: metadata.json does not exist, assuming version 0.0.0 and no dependencies' + return { + version: '0.0.0', + name: puppet_module.title, + dependencies: [] + } + end + metadata = R10K::Module::MetadataFile.new(metadata_path) + metadata = metadata.read + { + version: metadata.version, + name: metadata.name, + dependencies: metadata.dependencies + } + end + + # Is there a better way? :( + def get_key_or_sym(hash, k) + hash.fetch(k.to_sym, hash.fetch(k.to_s, nil)) + end + + # At least puppet-extlib has malformed metadata + def get_version_req(dep) + req = get_key_or_sym(dep, :version_requirement) + req = get_key_or_sym(dep, :version_range) unless req + req + end + + def print_module_diff(current, resolution) + current.sort! + resolution.sort! + outdated = [] + missing = [] + resolution.each do |mod| + cur_mod, cur_version = current.shift + mod, version = mod + if (cur_mod == mod) && cur_version && (cur_version != version) + outdated << [mod, cur_version, version] + elsif cur_mod != mod + missing << [mod, version] + current.unshift [cur_mod, cur_version] + end + end + missing.each do |m| + puts format('MISSING: %-25s %s', *m) + end + outdated.each do |o| + puts format('OUTDATED: %-25s %s -> %s', *o) + end + end + + def add_reqs_to_graph(artifact, metadata, no_demands = nil) + deps = get_key_or_sym(metadata, :dependencies) + my_name = get_key_or_sym(metadata, :name) + deps.each do |dep| + name = get_key_or_sym(dep, :name).tr('/', '-') + # Add dependency to the global set of modules we want, so that we can + # actually ask the solver for the versioned thing + @demands.add(name) unless no_demands + ver = get_version_req(dep) + unless ver + # no version specified, so anything will do + ver = '>=0.0.0' + end + ver.split(/(?=[<])/).each do |bound| + bound.strip! + v = begin + Semverse::Constraint.new(bound) + rescue + nil + end + if v + artifact.depends(name, v.to_s) + else + puts "WARNING: Invalid version constraint: #{bound}" + end + end + # Find the dependency in the forge, unless it's already been processed + # and add its releases to the global graph + next unless @processed_modules.add?(name) + puts "Fetching module info for #{name}" + mod = begin + PuppetForge::Module.find(name) + rescue + # It's probably a git module + nil + end + next unless mod # Git module, or non-forge dependency. Skip to next for now. + # Fetching metadata for all releases takes ages (which is weird, since it's mostly static info) + mod.releases.take(FETCH_LIMIT).each do |rel| + meta = get_release_metadata(name, rel) + rel_artifact = @graph.artifact(name, rel.version) + puts "...Recursively adding requirements for dependency #{name} version #{rel.version}" + # We don't want to add the requirements to the list of demands for all versions, + # but we need them in the graph to be able to solve dependencies + add_reqs_to_graph(rel_artifact, meta, :no_demands) + end + end + end + end +end + +Ra10ke::Solve::RakeTask.new diff --git a/ra10ke.gemspec b/ra10ke.gemspec index 6324121..fd8b6ee 100644 --- a/ra10ke.gemspec +++ b/ra10ke.gemspec @@ -19,4 +19,5 @@ Gem::Specification.new do |spec| spec.add_dependency "puppet_forge" spec.add_dependency "r10k" spec.add_dependency "git" + spec.add_dependency "solve" end