Support npm 7 in npm source

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Setup node
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v2
       with:
         node-version: 8
     - name: Install Bower
@@ -258,11 +258,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node_version: [ 8, 10, 12 ]
+        node_version: [ 10, 12, 14, 15 ]
     steps:
     - uses: actions/checkout@v2
     - name: Setup node
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v2
       with:
         node-version: ${{ matrix.node_version }}
     - name: Set up Ruby
@@ -371,7 +371,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Setup node
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v2
       with:
         node-version: 12
     - name: Install Yarn

.gitignore

@@ -17,6 +17,7 @@ test/fixtures/bower/bower_components
 
 test/fixtures/npm/node_modules
 test/fixtures/npm/package-lock.json
+test/fixtures/npm/package.json
 
 test/fixtures/go/src/*
 test/fixtures/go/pkg

lib/licensed/sources/npm.rb

@@ -4,6 +4,25 @@
 module Licensed
   module Sources
     class NPM < Source
+      class Dependency < ::Licensed::Dependency
+        # override license_metadata to pull homepage and summary information
+        # from a packages package.json file, if it exists
+        # this accounts for the lack of this information in npm 7's `npm list` output
+        def license_metadata
+          data = super
+          return data if !data["homepage"].to_s.empty? && !data["summary"].to_s.empty?
+
+          package_json_path = File.join(path, "package.json")
+          return data unless File.exist?(package_json_path)
+
+          package_json = JSON.parse(File.read(package_json_path))
+          data["homepage"] = package_json["homepage"]
+          data["summary"] = package_json["description"]
+
+          data
+        end
+      end
+
       def self.type
         "npm"
       end
@@ -50,6 +69,7 @@ def recursive_dependencies(dependencies, result = {})
         dependencies.each do |name, dependency|
           next if dependency["peerMissing"]
           next if yarn_lock_present && dependency["missing"]
+          dependency["name"] = name
           (result[name] ||= []) << dependency
           recursive_dependencies(dependency["dependencies"] || {}, result)
         end
@@ -59,22 +79,50 @@ def recursive_dependencies(dependencies, result = {})
       # Returns parsed package metadata returned from `npm list`
       def package_metadata
         return @package_metadata if defined?(@package_metadata)
+        @package_metadata = JSON.parse(package_metadata_command)
+      rescue JSON::ParserError => e
+        message = "Licensed was unable to parse the output from 'npm list'. JSON Error: #{e.message}"
+        npm_error = package_metadata_error
+        message = "#{message}. NPM Error: #{npm_error}" if npm_error
+        raise Licensed::Sources::Source::Error, message
+      end
 
-        @package_metadata = begin
-          JSON.parse(package_metadata_command)
-        rescue JSON::ParserError => e
-          raise Licensed::Sources::Source::Error,
-            "Licensed was unable to parse the output from 'npm list'. Please run 'npm list --json --long' and check for errors. Error: #{e.message}"
-        end
+      # Returns an error, if one exists, from running `npm list` to get package metadata
+      def package_metadata_error
+        Licensed::Shell.execute("npm", "list", *package_metadata_args)
+        return ""
+      rescue Licensed::Shell::Error => e
+        return e.message
       end
 
       # Returns the output from running `npm list` to get package metadata
       def package_metadata_command
         args = %w(--json --long)
-        args << "--production" unless include_non_production?
+        args.concat(package_metadata_args)
+
         Licensed::Shell.execute("npm", "list", *args, allow_failure: true)
       end
 
+      # Returns an array of arguments that should be used for all `npm list`
+      # calls, regardless of how the output is formatted
+      def package_metadata_args
+        args = []
+        args << "--production" unless include_non_production?
+
+        # on npm 7+, the --all argument is necessary to evaluate the project's
+        # full dependency tree
+        args << "--all" if npm_version >= Gem::Version.new("7.0.0")
+
+        return args
+      end
+
+      # Returns the currently installed version of npm as a Gem::Version object
+      def npm_version
+        @npm_version ||= begin
+          Gem::Version.new(Licensed::Shell.execute("npm", "-v").strip)
+        end
+      end
+
       # Returns true if a yarn.lock file exists in the current directory
       def yarn_lock_present
         @yarn_lock_present ||= File.exist?(config.pwd.join("yarn.lock"))

script/source-setup/npm

@@ -10,8 +10,25 @@ fi
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 cd $BASE_PATH/test/fixtures/npm
 
+FORCE=""
 if [ "$1" == "-f" ]; then
-  find . -not -regex "\.*" -and -not -name "package\.json" -print0 | xargs -0 rm -rf
+  FORCE=1
+fi
+
+NPM_MAJOR_VERSION="$(npm -v | cut -d'.' -f1)"
+if [ "$NPM_MAJOR_VERSION" -ge "7" ]; then
+  PACKAGE_JSON_SRC="package.json.npm7"
+else
+  PACKAGE_JSON_SRC="package.json.npm6"
+fi
+
+if [ ! -f "package.json" ] || [ "$(cat package.json | md5sum )" != "$(cat "$PACKAGE_JSON_SRC" | md5sum)" ]; then
+  FORCE=1
+  cp -f "$PACKAGE_JSON_SRC" package.json
+fi
+
+if [ -n "$FORCE" ]; then
+  find . -not -regex "\.*" -and -not -name "package\.json*" -print0 | xargs -0 rm -rf
 fi
 
 npm install

test/fixtures/npm/package.json.npm7

@@ -0,0 +1,14 @@
+{
+  "name": "fixtures",
+  "version": "1.0.0",
+  "dependencies": {
+    "@github/query-selector": "1.0.3",
+    "autoprefixer": "5.2.0"
+  },
+  "devDependencies": {
+    "string.prototype.startswith": "0.2.0"
+  },
+  "description": "npm test fixture",
+  "repository": "https://github.com/github/licensed",
+  "license": "MIT"
+}

test/sources/npm_test.rb

@@ -8,6 +8,7 @@
     let(:config) { Licensed::AppConfiguration.new({ "source_path" => Dir.pwd }) }
     let(:fixtures) { File.expand_path("../../fixtures/npm", __FILE__) }
     let(:source) { Licensed::Sources::NPM.new(config) }
+    let(:version) { Gem::Version.new(Licensed::Shell.execute("npm", "-v")) }
 
     describe "enabled?" do
       it "is true if package.json exists" do
@@ -35,17 +36,30 @@
           assert dep
           assert_equal "npm", dep.record["type"]
           assert_equal "5.2.0", dep.version
-          assert dep.record["homepage"]
+          if source.npm_version < Gem::Version.new("7.0.0")
+            assert dep.record["homepage"]
+          end
           assert dep.record["summary"]
         end
       end
 
+      it "includes homepage information if available" do
+        Dir.chdir fixtures do
+          dep = source.dependencies.detect { |d| d.name == "amdefine" }
+          assert dep
+          assert_equal "npm", dep.record["type"]
+          assert dep.record["homepage"]
+        end
+      end
+
       it "handles scoped dependency names" do
         Dir.chdir fixtures do
           dep = source.dependencies.detect { |d| d.name == "@github/query-selector" }
           assert dep
           assert_equal "1.0.3", dep.version
-          assert dep.record["homepage"]
+          if source.npm_version < Gem::Version.new("7.0.0")
+            assert dep.record["homepage"]
+          end
           assert dep.record["summary"]
         end
       end
@@ -77,6 +91,7 @@
       end
 
       it "does not include missing indirect peer dependencies" do
+        skip if source.npm_version >= Gem::Version.new("7.0.0")
         Dir.chdir fixtures do
           # peer dependency of @optimizely/js-sdk-datafile-manager, which is
           # an indirect dependency through @optimizely/optimizely-sdk
@@ -119,6 +134,9 @@
 
     describe "missing dependencies (glob is missing package)" do
       it "includes missing dependencies when yarn.lock is missing" do
+        # this test is incompatible with npm >=7
+        skip if source.npm_version >= Gem::Version.new("7.0.0")
+
         Dir.mktmpdir do |dir|
           FileUtils.cp_r(fixtures, dir)
           dir = File.join(dir, "npm")
@@ -132,6 +150,9 @@
       end
 
       it "excludes missing dependencies when yarn.lock is present" do
+        # this test is incompatible with npm >=7
+        skip if source.npm_version >= Gem::Version.new("7.0.0")
+
         Dir.mktmpdir do |dir|
           FileUtils.cp_r(fixtures, dir)
           dir = File.join(dir, "npm")
@@ -144,6 +165,25 @@
           end
         end
       end
+
+      it "raises Licensed::Sources::Source::Error on missing dependencies" do
+        # this test is incompatible with npm <7
+        skip if source.npm_version < Gem::Version.new("7.0.0")
+
+        Dir.mktmpdir do |dir|
+          FileUtils.cp_r(fixtures, dir)
+          dir = File.join(dir, "npm")
+          FileUtils.rm_rf(File.join(dir, "node_modules/wrappy"))
+
+          Dir.chdir dir do
+            error = assert_raises Licensed::Sources::Source::Error do
+              source.dependencies
+            end
+
+            assert error.message.include? "missing: wrappy@1"
+          end
+        end
+      end
     end
   end
 end