diff --git a/analyzer/src/funTest/assets/projects/external/git-repo-expected-output.yml b/analyzer/src/funTest/assets/projects/external/git-repo-expected-output.yml index 7a8d2fc916b0e..fd75939021c92 100644 --- a/analyzer/src/funTest/assets/projects/external/git-repo-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/external/git-repo-expected-output.yml @@ -4288,7 +4288,7 @@ analyzer: declared_licenses_processed: spdx_expression: "MIT" description: "TypeScript definitions for Node.js" - homepage_url: "" + homepage_url: "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/node" binary_artifact: url: "" hash: diff --git a/analyzer/src/funTest/assets/projects/synthetic/npm-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/npm-expected-output.yml index dc4d73155d009..0f470d22afd5f 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/npm-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/npm-expected-output.yml @@ -1165,7 +1165,7 @@ packages: declared_licenses_processed: spdx_expression: "MIT" description: "TypeScript definitions for Node.js" - homepage_url: "" + homepage_url: "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/node" binary_artifact: url: "" hash: diff --git a/analyzer/src/funTest/assets/projects/synthetic/yarn-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/yarn-expected-output.yml index 1ece70b5c0c1e..37faeefa047ce 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/yarn-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/yarn-expected-output.yml @@ -1134,7 +1134,7 @@ packages: declared_licenses_processed: spdx_expression: "MIT" description: "TypeScript definitions for Node.js" - homepage_url: "" + homepage_url: "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/node" binary_artifact: url: "" hash: diff --git a/analyzer/src/main/kotlin/managers/Npm.kt b/analyzer/src/main/kotlin/managers/Npm.kt index 84809ef6f5089..41a761b8e4e79 100644 --- a/analyzer/src/main/kotlin/managers/Npm.kt +++ b/analyzer/src/main/kotlin/managers/Npm.kt @@ -28,7 +28,6 @@ import com.vdurmont.semver4j.Requirement import java.io.File import java.io.IOException -import java.net.URLEncoder import java.util.SortedSet import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory @@ -39,8 +38,6 @@ import org.ossreviewtoolkit.analyzer.managers.utils.NpmModuleInfo import org.ossreviewtoolkit.analyzer.managers.utils.expandNpmShortcutUrl import org.ossreviewtoolkit.analyzer.managers.utils.hasNpmLockFile import org.ossreviewtoolkit.analyzer.managers.utils.mapDefinitionFilesForNpm -import org.ossreviewtoolkit.analyzer.managers.utils.readProxySettingsFromNpmRc -import org.ossreviewtoolkit.analyzer.managers.utils.readRegistryFromNpmRc import org.ossreviewtoolkit.analyzer.parseAuthorString import org.ossreviewtoolkit.downloader.VcsHost import org.ossreviewtoolkit.downloader.VersionControlSystem @@ -71,17 +68,13 @@ import org.ossreviewtoolkit.utils.common.realFile import org.ossreviewtoolkit.utils.common.stashDirectories import org.ossreviewtoolkit.utils.common.textValueOrEmpty import org.ossreviewtoolkit.utils.common.withoutPrefix -import org.ossreviewtoolkit.utils.core.OkHttpClientHelper -import org.ossreviewtoolkit.utils.core.installAuthenticatorAndProxySelector import org.ossreviewtoolkit.utils.core.log import org.ossreviewtoolkit.utils.spdx.SpdxConstants -const val PUBLIC_NPM_REGISTRY = "https://registry.npmjs.org" - object NpmCli : CommandLineTool { override fun command(workingDir: File?) = if (Os.isWindows) "npm.cmd" else "npm" - override fun getVersionRequirement() = Requirement.buildNPM("5.7.* - 7.20.*") + override fun getVersionRequirement() = Requirement.buildNPM("6.* - 7.20.*") } /** @@ -196,10 +189,10 @@ open class Npm( /** * Construct a [Package] by parsing its _package.json_ file and - if applicable - querying additional - * content from the [npmRegistry]. Result is a [Pair] with the raw identifier and the new package. + * content via the `npm view` command. The result is a [Pair] with the raw identifier and the new package. */ @Suppress("HttpUrlsUsage") - internal fun parsePackage(packageFile: File, npmRegistry: String): Pair { + internal fun parsePackage(packageFile: File): Pair { val packageDir = packageFile.parentFile log.debug { "Found a 'package.json' file in '$packageDir'." } @@ -246,34 +239,18 @@ open class Npm( || hash == Hash.NONE || vcsFromPackage == VcsInfo.EMPTY if (hasIncompleteData) { - // Download package info from registry.npmjs.org. - // TODO: check if unpkg.com can be used as a fallback in case npmjs.org is down. - val encodedName = if (rawName.startsWith("@")) { - "@${URLEncoder.encode(rawName.substringAfter('@'), "UTF-8")}" - } else { - rawName - } - - log.debug { "Resolving the package info for '${id.toCoordinates()}' via NPM registry." } - - OkHttpClientHelper.downloadText("$npmRegistry/$encodedName/$version").onSuccess { - val versionInfo = jsonMapper.readTree(it) + val process = NpmCli.run("view", "--json", "$rawName@$version") + val view = jsonMapper.readTree(process.stdoutFile) - if (description.isEmpty()) description = versionInfo["description"].textValueOrEmpty() - if (homepageUrl.isEmpty()) homepageUrl = versionInfo["homepage"].textValueOrEmpty() + if (description.isEmpty()) description = view["description"].textValueOrEmpty() + if (homepageUrl.isEmpty()) homepageUrl = view["homepage"].textValueOrEmpty() - versionInfo["dist"]?.let { dist -> - if (downloadUrl.isEmpty()) downloadUrl = dist["tarball"].textValueOrEmpty() - if (hash == Hash.NONE) hash = Hash.create(dist["shasum"].textValueOrEmpty()) - } - - vcsFromPackage = parseVcsInfo(versionInfo) - }.onFailure { - log.info { - "Could not retrieve package information for '$encodedName' from NPM registry " + - "$npmRegistry: ${it.message}" - } + view["dist"]?.let { dist -> + if (downloadUrl.isEmpty()) downloadUrl = dist["tarball"].textValueOrEmpty() + if (hash == Hash.NONE) hash = Hash.create(dist["shasum"].textValueOrEmpty()) } + + vcsFromPackage = parseVcsInfo(view) } } @@ -323,23 +300,8 @@ open class Npm( } } - private val ortProxySelector = installAuthenticatorAndProxySelector() - - private val npmRegistry: String - - init { - val npmRcFile = Os.userHomeDirectory.resolve(".npmrc") - npmRegistry = if (npmRcFile.isFile) { - val npmRcContent = npmRcFile.readText() - ortProxySelector.addProxies(managerName, readProxySettingsFromNpmRc(npmRcContent)) - readRegistryFromNpmRc(npmRcContent) ?: PUBLIC_NPM_REGISTRY - } else { - PUBLIC_NPM_REGISTRY - } - } - private val graphBuilder: DependencyGraphBuilder = - DependencyGraphBuilder(NpmDependencyHandler(npmRegistry)) + DependencyGraphBuilder(NpmDependencyHandler()) /** * Array of parameters passed to the install command when installing dependencies. @@ -356,10 +318,6 @@ open class Npm( NpmCli.checkVersion() } - override fun afterResolution(definitionFiles: List) { - ortProxySelector.removeProxyOrigin(managerName) - } - override fun createPackageManagerResult(projectResults: Map>) = PackageManagerResult(projectResults, graphBuilder.build(), graphBuilder.packages()) @@ -423,7 +381,7 @@ open class Npm( nodeModulesDir.walk().filter { it.name == "package.json" && isValidNodeModulesDirectory(nodeModulesDir, nodeModulesDirForPackageJson(it)) }.forEach { file -> - val (id, pkg) = parsePackage(file, npmRegistry) + val (id, pkg) = parsePackage(file) packages[id] = pkg } diff --git a/analyzer/src/main/kotlin/managers/utils/NpmDependencyHandler.kt b/analyzer/src/main/kotlin/managers/utils/NpmDependencyHandler.kt index c569c5a176e66..59c99390a703f 100644 --- a/analyzer/src/main/kotlin/managers/utils/NpmDependencyHandler.kt +++ b/analyzer/src/main/kotlin/managers/utils/NpmDependencyHandler.kt @@ -49,10 +49,7 @@ data class NpmModuleInfo( /** * A specialized [DependencyHandler] implementation for NPM. */ -class NpmDependencyHandler( - /** The URL for queries to the NPM registry. */ - private val npmRegistryUrl: String -) : DependencyHandler { +class NpmDependencyHandler : DependencyHandler { override fun identifierFor(dependency: NpmModuleInfo): Identifier = dependency.id override fun dependenciesFor(dependency: NpmModuleInfo): Collection = dependency.dependencies @@ -60,5 +57,5 @@ class NpmDependencyHandler( override fun linkageFor(dependency: NpmModuleInfo): PackageLinkage = PackageLinkage.DYNAMIC override fun createPackage(dependency: NpmModuleInfo, issues: MutableList): Package = - Npm.parsePackage(dependency.packageFile, npmRegistryUrl).second + Npm.parsePackage(dependency.packageFile).second } diff --git a/analyzer/src/test/kotlin/managers/utils/NpmDependencyHandlerTest.kt b/analyzer/src/test/kotlin/managers/utils/NpmDependencyHandlerTest.kt index 7b2c248ae42ab..cc1a836947a11 100644 --- a/analyzer/src/test/kotlin/managers/utils/NpmDependencyHandlerTest.kt +++ b/analyzer/src/test/kotlin/managers/utils/NpmDependencyHandlerTest.kt @@ -72,15 +72,12 @@ class NpmDependencyHandlerTest : StringSpec({ id shouldBe Identifier("NPM", "", "bonjour", "3.5.0") declaredLicenses should containExactly("MIT") authors should containExactly("Thomas Watson Steen") - homepageUrl shouldBe "https://github.com/watson/bonjour/local" - description shouldBe "A Bonjour/Zeroconf implementation in pure JavaScript (local)" + homepageUrl shouldBe "https://github.com/watson/bonjour" + description shouldBe "A Bonjour/Zeroconf implementation in pure JavaScript" } } }) -/** The URI to be used for queries to the NPM registry. */ -const val NPM_REGISTRY_URL = "https://npm-test.example.org" - /** * Construct a test identifier with the given [name]. */ @@ -100,4 +97,4 @@ private fun createModuleInfo( /** * Creates an [NpmDependencyHandler] instance to be used by test cases. */ -private fun createHandler() = NpmDependencyHandler(NPM_REGISTRY_URL) +private fun createHandler() = NpmDependencyHandler()