diff --git a/docs/sources/go.md b/docs/sources/go.md index ba1909c1..cac75593 100644 --- a/docs/sources/go.md +++ b/docs/sources/go.md @@ -23,3 +23,20 @@ go: The setting supports absolute, relative and expandable (e.g. "~") paths. Relative paths are considered relative to the repository root. Non-empty `GOPATH` configuration settings will override the `GOPATH` environment variable while enumerating `go` dependencies. The `GOPATH` environment variable is restored once dependencies have been enumerated. + +#### Versioning + +The go source supports multiple versioning strategies to determine if cached dependency metadata is stale. A version strategy is chosen based on the availability of go module information along with the current app configuration. + +1. Go Module version - This strategy uses the version of the go module. + - :exclamation: This strategy will always be used if go module information is available because the version comes from an externally provided identifier. Locating the version of the source package used via this identifier will be easier than other strategies. +2. Git commit SHA - This strategy uses the latest Git commit SHA available for the package's import path directory as the version. This is the default strategy used if a go module version isn't available and the setting is not configured. + - :warning: The latest Git commit won't capture any changes that are committed alongside a cached file update. Make sure to update cached files after all other changes are committed. + + ```yaml + version_strategy: git # or leave this key unset + ``` +3. Contents hash - This strategy uses a hash of the files in the package's import path directory as the version. + ```yaml + version_strategy: contents + ``` diff --git a/docs/sources/manifests.md b/docs/sources/manifests.md index 5ff4d2e7..c4d96249 100644 --- a/docs/sources/manifests.md +++ b/docs/sources/manifests.md @@ -145,3 +145,18 @@ manifest: licenses: package: path/to/LICENSE ``` + +### License content versioning + +The manifest source supports multiple versioning strategies to determine if cached dependency metadata is stale. A version strategy is chosen based on the current app configuration. + +1. Git commit SHA - This strategy uses the latest Git commit SHA available for the package's import path directory as the version. This is the default strategy used if not otherwise configured. + - :warning: The latest Git commit won't capture any changes that are committed alongside a cached file update. Make sure to update cached files after all other changes are committed. + + ```yaml + version_strategy: git # or leave this key unset + ``` +2. Contents hash - This strategy uses a hash of the files in the package's import path directory as the version. + ```yaml + version_strategy: contents + ``` diff --git a/lib/licensed/sources/go.rb b/lib/licensed/sources/go.rb index 71895526..922280bd 100644 --- a/lib/licensed/sources/go.rb +++ b/lib/licensed/sources/go.rb @@ -1,10 +1,13 @@ # frozen_string_literal: true require "json" require "pathname" +require "licensed/sources/helpers/content_versioning" module Licensed module Sources class Go < Source + include Licensed::Sources::ContentVersioning + def enabled? Licensed::Shell.tool_available?("go") && go_source? end @@ -102,7 +105,19 @@ def package_version(package) # find most recent git SHA for a package, or nil if SHA is # not available Dir.chdir package_directory do - Licensed::Git.version(".") + contents_version *contents_version_arguments + end + end + + # Determines the arguments to pass to contents_version based on which + # version strategy is selected + # + # Returns an array of arguments to pass to contents version + def contents_version_arguments + if version_strategy == Licensed::Sources::ContentVersioning::GIT + ["."] + else + Dir["*"] end end diff --git a/lib/licensed/sources/helpers/content_versioning.rb b/lib/licensed/sources/helpers/content_versioning.rb new file mode 100644 index 00000000..742a7397 --- /dev/null +++ b/lib/licensed/sources/helpers/content_versioning.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "ruby-xxhash" + +module Licensed + module Sources + module ContentVersioning + GIT = "git".freeze + CONTENTS = "contents".freeze + + # Find the version for a list of paths using the version strategy + # specified for the source from the configuration + # + # paths - list of paths to find version + # + # Returns a version identifier for the given files + def contents_version(*paths) + case version_strategy + when CONTENTS + contents_hash(paths) + when GIT + git_version(paths) + end + end + + # Returns the version strategy configured for the source + def version_strategy + # default to git for backwards compatible behavior + @version_strategy ||= begin + case config.fetch("version_strategy", nil) + when CONTENTS + CONTENTS + when GIT + GIT + else + Licensed::Git.available? ? GIT : CONTENTS + end + end + end + + # Find the version for a list of paths using Git commit information + # + # paths - list of paths to find version + # + # Returns the most recent git SHA from the given paths + def git_version(paths) + return if paths.nil? + + paths.map { |path| Licensed::Git.version(path) } + .reject { |sha| sha.to_s.empty? } + .max_by { |sha| Licensed::Git.commit_date(sha) } + end + + # Find the version for a list of paths using their file contents + # + # paths - list of paths to find version + # + # Returns a hash of the path contents as an identifier for the group + def contents_hash(paths) + return if paths.nil? + + paths = paths.compact.select { |path| File.file?(path) } + return if paths.empty? + + paths.sort + .reduce(Digest::XXHash64.new, :file) + .digest + .to_s(16) # convert to hex + end + end + end +end diff --git a/lib/licensed/sources/manifest.rb b/lib/licensed/sources/manifest.rb index 498e0ef4..7e0901ab 100644 --- a/lib/licensed/sources/manifest.rb +++ b/lib/licensed/sources/manifest.rb @@ -1,9 +1,12 @@ # frozen_string_literal: true require "pathname/common_prefix" +require "licensed/sources/helpers/content_versioning" module Licensed module Sources class Manifest < Source + include Licensed::Sources::ContentVersioning + def enabled? File.exist?(manifest_path) || generate_manifest? end @@ -12,7 +15,7 @@ def enumerate_dependencies packages.map do |package_name, sources| Licensed::Sources::Manifest::Dependency.new( name: package_name, - version: package_version(sources), + version: contents_version(*sources), path: configured_license_path(package_name) || sources_license_path(sources), sources: sources, metadata: { @@ -23,15 +26,6 @@ def enumerate_dependencies end end - # Returns the latest git SHA available from `sources` - def package_version(sources) - return if sources.nil? || sources.empty? - - sources.map { |s| Licensed::Git.version(s) } - .compact - .max_by { |sha| Licensed::Git.commit_date(sha) } - end - # Returns the license path for a package specified in the configuration. def configured_license_path(package_name) license_path = @config.dig("manifest", "licenses", package_name) diff --git a/licensed.gemspec b/licensed.gemspec index e47a7695..d019c64b 100644 --- a/licensed.gemspec +++ b/licensed.gemspec @@ -28,6 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "pathname-common_prefix", "~> 0.0.1" spec.add_dependency "tomlrb", "~> 1.2" spec.add_dependency "bundler", ">= 1.10" + spec.add_dependency "ruby-xxHash", "~> 0.4" spec.add_development_dependency "rake", "~> 10.0" spec.add_development_dependency "minitest", "~> 5.8" diff --git a/test/fixtures/manifest/manifest.json b/test/fixtures/manifest/manifest.json index 89921a16..89ac84f4 100644 --- a/test/fixtures/manifest/manifest.json +++ b/test/fixtures/manifest/manifest.json @@ -7,5 +7,6 @@ "test/fixtures/manifest/multiple_license_headers/source.c": "bsd3_multi_header_license", "test/fixtures/manifest/multiple_license_headers/source_2.c": "bsd3_multi_header_license", "test/fixtures/manifest/with_license_file/source.c": "mit_license_file", - "test/fixtures/manifest/with_notices/source.c": "notices" + "test/fixtures/manifest/with_notices/source.c": "notices", + "test/fixtures/manifest/version/test.c": "version_test" } diff --git a/test/fixtures/manifest/manifest.yml b/test/fixtures/manifest/manifest.yml index 36ae7dd0..c3160adb 100644 --- a/test/fixtures/manifest/manifest.yml +++ b/test/fixtures/manifest/manifest.yml @@ -6,3 +6,4 @@ test/fixtures/manifest/multiple_license_headers/source.c: bsd3_multi_header_lice test/fixtures/manifest/multiple_license_headers/source_2.c: bsd3_multi_header_license test/fixtures/manifest/with_license_file/source.c: mit_license_file test/fixtures/manifest/with_notices/source.c: notices +test/fixtures/manifest/version/test.c: version_test diff --git a/test/fixtures/manifest/version/test.c b/test/fixtures/manifest/version/test.c new file mode 100644 index 00000000..83025866 --- /dev/null +++ b/test/fixtures/manifest/version/test.c @@ -0,0 +1,6 @@ +#include + +int main() +{ + printf("I'm a test!"); +} diff --git a/test/sources/go_test.rb b/test/sources/go_test.rb index 7e2d8850..6f4fb63e 100644 --- a/test/sources/go_test.rb +++ b/test/sources/go_test.rb @@ -170,19 +170,18 @@ end describe "without go module information" do - it "is nil when git is unavailable" do + it "is the latest git SHA of the package directory when configured" do Dir.chdir fixtures do - Licensed::Git.stub(:available?, false) do - dep = source.dependencies.detect { |d| d.name == "github.com/gorilla/context" } - assert_nil dep.version - end + dep = source.dependencies.detect { |d| d.name == "github.com/gorilla/context" } + assert_equal source.git_version([dep.path]), dep.version end end - it "is the latest git SHA of the package directory" do + it "is the hash of all contents in the package directory when configured" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::CONTENTS Dir.chdir fixtures do dep = source.dependencies.detect { |d| d.name == "github.com/gorilla/context" } - assert_match(/[a-f0-9]{40}/, dep.version) + assert_equal source.contents_hash(Dir["#{dep.path}/*"]), dep.version end end end diff --git a/test/sources/helpers/content_versioning_test.rb b/test/sources/helpers/content_versioning_test.rb new file mode 100644 index 00000000..e39405c9 --- /dev/null +++ b/test/sources/helpers/content_versioning_test.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true +require "test_helper" + +describe Licensed::Sources::ContentVersioning do + let(:fixtures) { File.expand_path("../../../fixtures/command", __FILE__) } + let(:config) { Licensed::Configuration.new } + let(:helper) do + obj = mock.extend Licensed::Sources::ContentVersioning + obj.stubs(:config).returns(config) + obj + end + + + describe "#contents_version" do + it "handles a content hashing strategy" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::CONTENTS + helper.expects(:contents_hash).with(["path1", "path2"]).returns("version") + helper.expects(:git_version).never + assert_equal "version", helper.contents_version("path1", "path2") + end + + it "handles a git commit SHA strategy" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::GIT + helper.expects(:contents_hash).never + helper.expects(:git_version).with(["path1", "path2"]).returns("version") + assert_equal "version", helper.contents_version("path1", "path2") + end + end + + describe "#version_strategy" do + it "specifies content hashing if configured" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::CONTENTS + assert_equal Licensed::Sources::ContentVersioning::CONTENTS, helper.version_strategy + end + + it "specifies git version if configured" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::GIT + assert_equal Licensed::Sources::ContentVersioning::GIT, helper.version_strategy + end + + it "defaults to git version if not configured and git is available" do + Licensed::Git.stubs(:available?).returns(true) + assert_equal Licensed::Sources::ContentVersioning::GIT, helper.version_strategy + end + + it "defaults to content hashing if not configured and git is not available" do + Licensed::Git.stubs(:available?).returns(false) + assert_equal Licensed::Sources::ContentVersioning::CONTENTS, helper.version_strategy + end + end + + describe "#git_version" do + it "gets a hash for the latest commit for the set of paths" do + Dir.chdir fixtures do + # the hash for "." in a folder should identify the latest commit + # regardless of what other files from that folder are included + assert_equal Licensed::Git.version("."), helper.git_version(Dir["*"].concat(["."])) + end + end + + it "handles files not tracked by git" do + Dir.chdir File.expand_path("../../../bin", fixtures) do + assert_nil helper.git_version(Dir["*"]) + end + end + + it "handles empty arrays" do + assert_nil helper.git_version([]) + end + + it "handles nil input" do + assert_nil helper.git_version(nil) + end + end + + describe "#contents_hash" do + it "gets a hash representing the contents of relative paths" do + Dir.chdir fixtures do + refute_nil helper.contents_hash(Dir["*"]) + end + end + + it "gets a hash representing the contents of absolute paths" do + refute_nil helper.contents_hash(Dir["#{fixtures}/*"]) + end + + it "is agnostic to the order of paths provided" do + Dir.chdir fixtures do + assert_equal helper.contents_hash(["bower.yml", "bundler.yml", "cabal.yml"]), + helper.contents_hash(["cabal.yml", "bundler.yml", "bower.yml"]) + end + end + + it "handles empty arrays" do + assert_nil helper.contents_hash([]) + end + + it "handles nil input" do + assert_nil helper.contents_hash(nil) + end + + it "handles nil paths" do + assert_nil helper.contents_hash([nil]) + end + + it "handles non-existant paths" do + assert_nil helper.contents_hash(["#{fixtures}-bad"]) + end + + it "handles non-file paths" do + assert_nil helper.contents_hash([fixtures]) + end + end +end diff --git a/test/sources/manifest_test.rb b/test/sources/manifest_test.rb index 815e8ba4..6ece215d 100644 --- a/test/sources/manifest_test.rb +++ b/test/sources/manifest_test.rb @@ -83,6 +83,23 @@ assert dep refute_empty dep.record.notices end + + it "uses the git commit SHA as the version if configured" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::GIT + dep = source.dependencies.detect { |d| d.name == "version_test" } + assert_equal source.git_version(source.packages["version_test"]), dep.version + end + + it "uses the git commit SHA as the version if not configured" do + dep = source.dependencies.detect { |d| d.name == "version_test" } + assert_equal source.git_version(source.packages["version_test"]), dep.version + end + + it "uses the file contents hash as the version if configured" do + config["version_strategy"] = Licensed::Sources::ContentVersioning::CONTENTS + dep = source.dependencies.detect { |d| d.name == "version_test" } + assert_equal source.contents_hash(source.packages["version_test"]), dep.version + end end describe "manifest" do