From bcaddf481c203c03ee6e4fbb915ddbd6be392436 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 18 Jul 2023 16:52:58 +0200 Subject: [PATCH 1/2] feat(helper-cli): Allow generating an analyzer result from package list The command defines a data model called `PackageList` which is read as input using any of the known file extensions. It outputs an analyzer result with a single project containing the provided dependencies as a flat list. When a project does not use a package manager, currently the only way of scanning it with ORT is to craft SPDX files and running the analyzer against them. This command tries to provide a minimalistic alternative to fulfill the need to scan any project which does not use any package manager (supported by ORT). The package metadata holds attributes which are usually relevant for license clearance, namely the excluded state and the type of linkage. So, it can be useful to do license clearance for arbitrary sets of packages. This is for now limited to detected license clearance, as providing other attributes like e.g. the declared licenses is not (yet) implemented. Signed-off-by: Frank Viernau --- ...r-result-from-pkg-list-expected-output.yml | 113 +++++++++++ .../src/funTest/assets/package-list.yml | 28 +++ ...yzerResultFromPackageListCommandFunTest.kt | 64 ++++++ helper-cli/src/main/kotlin/HelperMain.kt | 1 + ...ateAnalyzerResultFromPackageListCommand.kt | 182 ++++++++++++++++++ 5 files changed, 388 insertions(+) create mode 100644 helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml create mode 100644 helper-cli/src/funTest/assets/package-list.yml create mode 100644 helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt create mode 100644 helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt diff --git a/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml b/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml new file mode 100644 index 0000000000000..69d7709bb6a00 --- /dev/null +++ b/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml @@ -0,0 +1,113 @@ +--- +repository: + vcs: + type: "Git" + url: "https://github.com/example/project.git" + revision: "2222222222222222222222222222222222222222" + path: "vcs-path/project" + vcs_processed: + type: "Git" + url: "https://github.com/example/project.git" + revision: "2222222222222222222222222222222222222222" + path: "vcs-path/project" + config: + excludes: + scopes: + - pattern: "excluded" + reason: "DEV_DEPENDENCY_OF" +analyzer: + start_time: "1970-01-01T00:00:00Z" + end_time: "1970-01-01T00:00:00Z" + environment: + ort_version: "1.1.0-SNAPSHOT" + java_version: "17.0.8.1" + os: "Linux" + processors: 8 + max_memory: 536870912 + variables: {} + tool_versions: {} + config: + allow_dynamic_versions: false + skip_excluded: false + result: + projects: + - id: "Unmanaged::Example project name:" + definition_file_path: "" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "Git" + url: "https://github.com/example/project.git" + revision: "2222222222222222222222222222222222222222" + path: "vcs-path/project" + vcs_processed: + type: "Git" + url: "https://github.com/example/project.git" + revision: "2222222222222222222222222222222222222222" + path: "vcs-path/project" + homepage_url: "" + scopes: + - name: "excluded" + dependencies: + - id: "NPM::example-dependency-one:1.0.0" + - name: "main" + dependencies: + - id: "NPM::example-dependency-two:2.0.0" + linkage: "STATIC" + packages: + - id: "NPM::example-dependency-one:1.0.0" + purl: "" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://example.org/example-dependency-one.zip" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/example/depedency-one.git" + revision: "0000000000000000000000000000000000000000" + path: "vcs-path/dependency-one" + vcs_processed: + type: "Git" + url: "https://github.com/example/depedency-one.git" + revision: "0000000000000000000000000000000000000000" + path: "vcs-path/dependency-one" + - id: "NPM::example-dependency-two:2.0.0" + purl: "" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://example.org/example-dependency-two.zip" + hash: + value: "" + algorithm: "" + vcs: + type: "Git" + url: "https://github.com/example/depedency-1.git" + revision: "1111111111111111111111111111111111111111" + path: "vcs-path/dependency-two" + vcs_processed: + type: "Git" + url: "https://github.com/example/depedency-1.git" + revision: "1111111111111111111111111111111111111111" + path: "vcs-path/dependency-two" +scanner: null +advisor: null +evaluator: null +resolved_configuration: {} diff --git a/helper-cli/src/funTest/assets/package-list.yml b/helper-cli/src/funTest/assets/package-list.yml new file mode 100644 index 0000000000000..f7bb8cd252497 --- /dev/null +++ b/helper-cli/src/funTest/assets/package-list.yml @@ -0,0 +1,28 @@ +--- +projectName: "Example project name" +projectVcs: + type: "Git" + url: "https://github.com/example/project.git" + revision: "2222222222222222222222222222222222222222" + path: "vcs-path/project" +dependencies: + - id: "NPM::example-dependency-one:1.0.0" + vcs: + type: "Git" + url: "https://github.com/example/depedency-one.git" + revision: "0000000000000000000000000000000000000000" + path: "vcs-path/dependency-one" + sourceArtifact: + url: "https://example.org/example-dependency-one.zip" + isExcluded: true + isDynamicallyLinked: true + - id: "NPM::example-dependency-two:2.0.0" + vcs: + type: "Git" + url: "https://github.com/example/depedency-1.git" + revision: "1111111111111111111111111111111111111111" + path: "vcs-path/dependency-two" + sourceArtifact: + url: "https://example.org/example-dependency-two.zip" + isExcluded: false + isDynamicallyLinked: false diff --git a/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt b/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt new file mode 100644 index 0000000000000..6613e1dddd298 --- /dev/null +++ b/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.helper.commands + +import com.github.ajalt.clikt.core.ProgramResult + +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.shouldBe + +import org.ossreviewtoolkit.helper.HelperMain +import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.readValue +import org.ossreviewtoolkit.model.toYaml +import org.ossreviewtoolkit.utils.ort.Environment +import org.ossreviewtoolkit.utils.ort.createOrtTempDir +import org.ossreviewtoolkit.utils.test.getAssetFile + +class CreateAnalyzerResultFromPackageListCommandFunTest : WordSpec({ + "The command" should { + "generate the expected analyzer result file" { + val inputFile = getAssetFile("package-list.yml") + val outputFile = createOrtTempDir().resolve("analyzer-result.yml") + val expectedOutputFile = getAssetFile("create-analyzer-result-from-pkg-list-expected-output.yml") + + runMain( + "create-analyzer-result-from-package-list", + "--package-list-file", + inputFile.absolutePath, + "--ort-file", + outputFile.absolutePath + ) + + outputFile.readText() shouldBe expectedOutputFile.readValue().patchEnvironment().toYaml() + } + } +}) + +private fun runMain(vararg args: String) { + @Suppress("SwallowedException") + try { + HelperMain().parse(args.asList()) + } catch (e: ProgramResult) { + // Ignore exceptions that just propagate the program result. + } +} + +private fun OrtResult.patchEnvironment(): OrtResult = copy(analyzer = analyzer?.copy(environment = Environment())) diff --git a/helper-cli/src/main/kotlin/HelperMain.kt b/helper-cli/src/main/kotlin/HelperMain.kt index 10334046c867c..6c65931daf843 100644 --- a/helper-cli/src/main/kotlin/HelperMain.kt +++ b/helper-cli/src/main/kotlin/HelperMain.kt @@ -79,6 +79,7 @@ internal class HelperMain : CliktCommand( subcommands( ConvertOrtFileCommand(), CreateAnalyzerResultCommand(), + CreateAnalyzerResultFromPackageListCommand(), DevCommand(), ExtractRepositoryConfigurationCommand(), GenerateTimeoutErrorResolutionsCommand(), diff --git a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt new file mode 100644 index 0000000000000..7ffa6f7bef7ae --- /dev/null +++ b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.helper.commands + +import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.module.kotlin.readValue + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.convert +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.file + +import org.ossreviewtoolkit.helper.utils.writeOrtResult +import org.ossreviewtoolkit.model.AnalyzerResult +import org.ossreviewtoolkit.model.AnalyzerRun +import org.ossreviewtoolkit.model.Hash +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageLinkage +import org.ossreviewtoolkit.model.PackageReference +import org.ossreviewtoolkit.model.Project +import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.Repository +import org.ossreviewtoolkit.model.Scope +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.Excludes +import org.ossreviewtoolkit.model.config.RepositoryConfiguration +import org.ossreviewtoolkit.model.config.ScopeExclude +import org.ossreviewtoolkit.model.config.ScopeExcludeReason +import org.ossreviewtoolkit.model.mapper +import org.ossreviewtoolkit.model.orEmpty +import org.ossreviewtoolkit.utils.common.expandTilde +import org.ossreviewtoolkit.utils.ort.Environment + +internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand( + "A command which turns a package list file into an analyzer result." +) { + private val packageListFile by option( + "--package-list-file", "-i", + help = "The package list file to read the packages metadata and the project metadata from." + ).convert { it.expandTilde() } + .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) + .convert { it.absoluteFile.normalize() } + .required() + + private val ortFile by option( + "--ort-file", "-o", + help = "The ORT file to write the generated analyzer result to." + ).convert { it.expandTilde() } + .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) + .convert { it.absoluteFile.normalize() } + .required() + + override fun run() { + val packageList = packageListFile.mapper().copy().apply { + // Use camel case already now (even if in all places snake case is used), because there is a plan + // to migrate from snake case to camel case in context of + // https://github.com/oss-review-toolkit/ort/issues/3904. + propertyNamingStrategy = PropertyNamingStrategies.LOWER_CAMEL_CASE + }.readValue(packageListFile) + + val projectName = packageList.projectName?.takeUnless { it.isBlank() } ?: DEFAULT_PROJECT_NAME + val projectVcs = packageList.projectVcs.toVcsInfo() + + val project = Project.EMPTY.copy( + id = Identifier("$PROJECT_TYPE::$projectName:"), + vcs = projectVcs, + vcsProcessed = projectVcs.normalize(), + scopeDependencies = setOfNotNull( + packageList.dependencies.filterNot { it.isExcluded }.toScope(MAIN_SCOPE_NAME), + packageList.dependencies.filter { it.isExcluded }.toScope(EXCLUDED_SCOPE_NAME) + ) + ) + + val ortResult = OrtResult( + analyzer = AnalyzerRun.EMPTY.copy( + result = AnalyzerResult( + projects = setOf(project), + packages = packageList.dependencies.mapTo(mutableSetOf()) { it.toPackage() } + ), + environment = Environment() + ), + repository = Repository( + vcs = projectVcs.normalize(), + config = RepositoryConfiguration( + excludes = Excludes( + scopes = listOf( + ScopeExclude(EXCLUDED_SCOPE_NAME, ScopeExcludeReason.DEV_DEPENDENCY_OF) + ) + ) + ) + ) + ) + + writeOrtResult(ortResult, ortFile) + } +} + +private const val DEFAULT_PROJECT_NAME = "unknown" +private const val EXCLUDED_SCOPE_NAME = "excluded" +private const val MAIN_SCOPE_NAME = "main" +private const val PROJECT_TYPE = "Unmanaged" // This refers to the package manager (plugin) named "Unmanaged". + +private data class PackageList( + val projectName: String? = null, + val projectVcs: Vcs? = null, + val dependencies: List = emptyList() +) + +private data class Dependency( + val id: Identifier, + val vcs: Vcs? = null, + val sourceArtifact: SourceArtifact? = null, + val isExcluded: Boolean = false, + val isDynamicallyLinked: Boolean = false +) + +private data class SourceArtifact( + val url: String, + val hash: Hash? = null +) + +private data class Vcs( + val type: String? = null, + val url: String? = null, + val revision: String? = null, + val path: String? = null +) + +private fun Vcs?.toVcsInfo(): VcsInfo = + if (this == null) { + VcsInfo.EMPTY + } else { + VcsInfo( + type = type?.let { VcsType.forName(it) } ?: VcsType.UNKNOWN, + url = url.orEmpty(), + revision = revision.orEmpty(), + path = path.orEmpty() + ) + } + +private fun Collection.toScope(name: String): Scope = + Scope( + name = name, + dependencies = mapTo(mutableSetOf()) { dependency -> + PackageReference( + id = dependency.id, + linkage = PackageLinkage.STATIC.takeUnless { dependency.isDynamicallyLinked } ?: PackageLinkage.DYNAMIC + ) + } + ) + +private fun Dependency.toPackage(): Package { + val vcsInfo = vcs.toVcsInfo() + + return Package.EMPTY.copy( + id = id, + sourceArtifact = sourceArtifact?.let { RemoteArtifact(url = it.url, it.hash ?: Hash.NONE) }.orEmpty(), + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize() + ) +} From 3f8030f2816cc8fbaf9e2cefa9d41091bfcdf93b Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Thu, 20 Jul 2023 13:21:00 +0200 Subject: [PATCH 2/2] feat(helper-cli): Inlcude package curations into generated ORT file As the analyzer is responsible for adding all package curations from all configurated providers to the resolved configuration in the ORT file, do so as well when generating the analyzer result from a package list file. This allows to use the standard package curation workflow with `CreateAnalyzerResultFromPackageListCommand`. Signed-off-by: Frank Viernau --- ...yzerResultFromPackageListCommandFunTest.kt | 11 ++++++++--- ...ateAnalyzerResultFromPackageListCommand.kt | 19 ++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt b/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt index 6613e1dddd298..ce3c7dc6cf5f8 100644 --- a/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt +++ b/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt @@ -26,8 +26,8 @@ import io.kotest.matchers.shouldBe import org.ossreviewtoolkit.helper.HelperMain import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.ResolvedConfiguration import org.ossreviewtoolkit.model.readValue -import org.ossreviewtoolkit.model.toYaml import org.ossreviewtoolkit.utils.ort.Environment import org.ossreviewtoolkit.utils.ort.createOrtTempDir import org.ossreviewtoolkit.utils.test.getAssetFile @@ -47,7 +47,8 @@ class CreateAnalyzerResultFromPackageListCommandFunTest : WordSpec({ outputFile.absolutePath ) - outputFile.readText() shouldBe expectedOutputFile.readValue().patchEnvironment().toYaml() + outputFile.readValue().patchAnalyzerResult() shouldBe + expectedOutputFile.readValue().patchAnalyzerResult() } } }) @@ -61,4 +62,8 @@ private fun runMain(vararg args: String) { } } -private fun OrtResult.patchEnvironment(): OrtResult = copy(analyzer = analyzer?.copy(environment = Environment())) +private fun OrtResult.patchAnalyzerResult(): OrtResult = + copy( + analyzer = analyzer?.copy(environment = Environment()), + resolvedConfiguration = ResolvedConfiguration() + ) diff --git a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt index 7ffa6f7bef7ae..06a5c9e40299a 100644 --- a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt +++ b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt @@ -24,6 +24,7 @@ import com.fasterxml.jackson.module.kotlin.readValue import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.options.convert +import com.github.ajalt.clikt.parameters.options.default import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.options.required import com.github.ajalt.clikt.parameters.types.file @@ -44,13 +45,18 @@ import org.ossreviewtoolkit.model.Scope import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.Excludes +import org.ossreviewtoolkit.model.config.OrtConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.config.ScopeExclude import org.ossreviewtoolkit.model.config.ScopeExcludeReason import org.ossreviewtoolkit.model.mapper import org.ossreviewtoolkit.model.orEmpty +import org.ossreviewtoolkit.model.utils.addPackageCurations +import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory import org.ossreviewtoolkit.utils.common.expandTilde import org.ossreviewtoolkit.utils.ort.Environment +import org.ossreviewtoolkit.utils.ort.ORT_CONFIG_FILENAME +import org.ossreviewtoolkit.utils.ort.ortConfigDirectory internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand( "A command which turns a package list file into an analyzer result." @@ -71,6 +77,14 @@ internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand( .convert { it.absoluteFile.normalize() } .required() + private val configFile by option( + "--config", + help = "The path to the ORT configuration file that configures the scan results storage." + ).convert { it.expandTilde() } + .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) + .convert { it.absoluteFile.normalize() } + .default(ortConfigDirectory.resolve(ORT_CONFIG_FILENAME)) + override fun run() { val packageList = packageListFile.mapper().copy().apply { // Use camel case already now (even if in all places snake case is used), because there is a plan @@ -92,6 +106,9 @@ internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand( ) ) + val ortConfig = OrtConfiguration.load(emptyMap(), configFile) + val packageCurationProviders = PackageCurationProviderFactory.create(ortConfig.packageCurationProviders) + val ortResult = OrtResult( analyzer = AnalyzerRun.EMPTY.copy( result = AnalyzerResult( @@ -110,7 +127,7 @@ internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand( ) ) ) - ) + ).addPackageCurations(packageCurationProviders) writeOrtResult(ortResult, ortFile) }