feat(helper-cli): Allow generating an analyzer result from package list

The command defines a data model called `PackageList` which is read as input using any of the known file extensions. It outputs an analyzer result with a single project containing the provided dependencies as a flat list. When a project does not use a package manager, currently the only way of scanning it with ORT is to craft SPDX files and running the analyzer against them. This command tries to provide a minimalistic alternative to fulfill the need to scan any project which does not use any package manager (supported by ORT). The package metadata holds attributes which are usually relevant for license clearance, namely the excluded state and the type of linkage. So, it can be useful to do license clearance for arbitrary sets of packages. Note: This is a first experimental version. The `VcsInfo` class is deliberately not used for the VCS info in order to avoid nesting, which makes it a bit easier to generate the package list file. Signed-off-by: Frank Viernau <frank_viernau@epam.com>
oss-review-toolkit · Sep 6, 2023 · fcc1287 · fcc1287
1 parent 723c314
commit fcc1287
Show file tree

Hide file tree

Showing 5 changed files with 371 additions and 0 deletions.
diff --git a/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml b/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml
@@ -0,0 +1,113 @@
+---
+repository:
+  vcs:
+    type: "Git"
+    url: "https://github.com/example/project.git"
+    revision: "2222222222222222222222222222222222222222"
+    path: "vcs-path/project"
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/example/project.git"
+    revision: "2222222222222222222222222222222222222222"
+    path: "vcs-path/project"
+  config:
+    excludes:
+      scopes:
+      - pattern: "excluded"
+        reason: "DEV_DEPENDENCY_OF"
+analyzer:
+  start_time: "1970-01-01T00:00:00Z"
+  end_time: "1970-01-01T00:00:00Z"
+  environment:
+    ort_version: "1.1.0-SNAPSHOT"
+    java_version: "17.0.8.1"
+    os: "Linux"
+    processors: 8
+    max_memory: 536870912
+    variables: {}
+    tool_versions: {}
+  config:
+    allow_dynamic_versions: false
+    skip_excluded: false
+  result:
+    projects:
+    - id: "Unmanaged::Example project name:"
+      definition_file_path: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      vcs:
+        type: "Git"
+        url: "https://github.com/example/project.git"
+        revision: "2222222222222222222222222222222222222222"
+        path: "vcs-path/project"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/example/project.git"
+        revision: "2222222222222222222222222222222222222222"
+        path: "vcs-path/project"
+      homepage_url: ""
+      scopes:
+      - name: "excluded"
+        dependencies:
+        - id: "NPM::example-dependency-one:1.0.0"
+      - name: "main"
+        dependencies:
+        - id: "NPM::example-dependency-two:2.0.0"
+          linkage: "STATIC"
+    packages:
+    - id: "NPM::example-dependency-one:1.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: "https://example.org/example-dependency-one.zip"
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/example/depedency-one.git"
+        revision: "0000000000000000000000000000000000000000"
+        path: "vcs-path/dependency-one"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/example/depedency-one.git"
+        revision: "0000000000000000000000000000000000000000"
+        path: "vcs-path/dependency-one"
+    - id: "NPM::example-dependency-two:2.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: "https://example.org/example-dependency-two.zip"
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/example/depedency-1.git"
+        revision: "1111111111111111111111111111111111111111"
+        path: "vcs-path/dependency-two"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/example/depedency-1.git"
+        revision: "1111111111111111111111111111111111111111"
+        path: "vcs-path/dependency-two"
+scanner: null
+advisor: null
+evaluator: null
+resolved_configuration: {}
diff --git a/helper-cli/src/funTest/assets/package-list.yml b/helper-cli/src/funTest/assets/package-list.yml
@@ -0,0 +1,23 @@
+---
+name: "Example project name"
+vcs_type: "Git"
+vcs_url: "https://github.com/example/project.git"
+vcs_revision: "2222222222222222222222222222222222222222"
+vcs_path: "vcs-path/project"
+dependencies:
+  - id: "NPM::example-dependency-one:1.0.0"
+    vcs_type: "Git"
+    vcs_url: "https://github.com/example/depedency-one.git"
+    vcs_revision: "0000000000000000000000000000000000000000"
+    vcs_path: "vcs-path/dependency-one"
+    source_artifact_url: "https://example.org/example-dependency-one.zip"
+    is_excluded: true
+    is_dynamically_linked: true
+  - id: "NPM::example-dependency-two:2.0.0"
+    vcs_type: "Git"
+    vcs_url: "https://github.com/example/depedency-1.git"
+    vcs_revision: "1111111111111111111111111111111111111111"
+    vcs_path: "vcs-path/dependency-two"
+    source_artifact_url: "https://example.org/example-dependency-two.zip"
+    is_excluded: false
+    is_dynamically_linked: false
diff --git a/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt b/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.helper.commands
+
+import com.github.ajalt.clikt.core.ProgramResult
+
+import io.kotest.core.spec.style.WordSpec
+import io.kotest.matchers.shouldBe
+
+import org.ossreviewtoolkit.helper.HelperMain
+import org.ossreviewtoolkit.model.OrtResult
+import org.ossreviewtoolkit.model.readValue
+import org.ossreviewtoolkit.model.toYaml
+import org.ossreviewtoolkit.utils.common.redirectStdout
+import org.ossreviewtoolkit.utils.ort.Environment
+import org.ossreviewtoolkit.utils.ort.createOrtTempDir
+import org.ossreviewtoolkit.utils.test.getAssetFile
+
+class CreateAnalyzerResultFromPackageListCommandFunTest : WordSpec({
+    "The command" should {
+        "generate the expected analyzer result file" {
+            val inputFile = getAssetFile("package-list.yml")
+            val outputFile = createOrtTempDir().resolve("analyzer-result.yml")
+            val expectedOutputFile = getAssetFile("create-analyzer-result-from-pkg-list-expected-output.yml")
+
+            runMain(
+                "create-analyzer-result-from-package-list",
+                "--package-list-file",
+                inputFile.absolutePath,
+                "--ort-file",
+                outputFile.absolutePath
+            )
+
+            outputFile.readText() shouldBe expectedOutputFile.readValue<OrtResult>().patchEnvironment().toYaml()
+        }
+    }
+})
+
+private fun runMain(vararg args: String) =
+    redirectStdout {
+        @Suppress("SwallowedException")
+        try {
+            HelperMain().parse(args.asList())
+        } catch (e: ProgramResult) {
+            // Ignore exceptions that just propagate the program result.
+        }
+    }.lineSequence()
+
+private fun OrtResult.patchEnvironment(): OrtResult = copy(analyzer = analyzer!!.copy(environment = Environment()))
diff --git a/helper-cli/src/main/kotlin/HelperMain.kt b/helper-cli/src/main/kotlin/HelperMain.kt
@@ -79,6 +79,7 @@ internal class HelperMain : CliktCommand(
         subcommands(
             ConvertOrtFileCommand(),
             CreateAnalyzerResultCommand(),
+            CreateAnalyzerResultFromPackageListCommand(),
             DevCommand(),
             ExtractRepositoryConfigurationCommand(),
             GenerateTimeoutErrorResolutionsCommand(),

diff --git a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.helper.commands
+
+import com.fasterxml.jackson.module.kotlin.readValue
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.options.convert
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.options.required
+import com.github.ajalt.clikt.parameters.types.file
+
+import org.ossreviewtoolkit.helper.utils.writeOrtResult
+import org.ossreviewtoolkit.model.AnalyzerResult
+import org.ossreviewtoolkit.model.AnalyzerRun
+import org.ossreviewtoolkit.model.Hash
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.OrtResult
+import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.PackageLinkage
+import org.ossreviewtoolkit.model.PackageReference
+import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.RemoteArtifact
+import org.ossreviewtoolkit.model.Repository
+import org.ossreviewtoolkit.model.Scope
+import org.ossreviewtoolkit.model.VcsInfo
+import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.Excludes
+import org.ossreviewtoolkit.model.config.RepositoryConfiguration
+import org.ossreviewtoolkit.model.config.ScopeExclude
+import org.ossreviewtoolkit.model.config.ScopeExcludeReason
+import org.ossreviewtoolkit.model.mapper
+import org.ossreviewtoolkit.model.orEmpty
+import org.ossreviewtoolkit.utils.common.expandTilde
+import org.ossreviewtoolkit.utils.ort.Environment
+
+internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand(
+    "A command which turns a simple definition file into an analyzer result."
+) {
+    private val packageListFile by option(
+        "--package-list-file", "-i",
+        help = "The package list file to read the packages metadata and the project metadata from."
+    ).convert { it.expandTilde() }
+        .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false)
+        .convert { it.absoluteFile.normalize() }
+        .required()
+
+    private val ortFile by option(
+        "--ort-file", "-o",
+        help = "The ORT file to write the generated analyzer result to."
+    ).convert { it.expandTilde() }
+        .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false)
+        .convert { it.absoluteFile.normalize() }
+        .required()
+
+    override fun run() {
+        val packageList = packageListFile.mapper().readValue<PackageList>(packageListFile)
+
+        val projectVcs = VcsInfo(
+            type = packageList.vcsType ?: VcsType.UNKNOWN,
+            url = packageList.vcsUrl.orEmpty(),
+            revision = packageList.vcsRevision.orEmpty(),
+            path = packageList.vcsPath.orEmpty()
+        )
+
+        val projectName = packageList.name?.takeUnless { it.isBlank() } ?: DEFAULT_PROJECT_NAME
+
+        val project = Project.EMPTY.copy(
+            id = Identifier("Unmanaged::$projectName:"),
+            vcs = projectVcs,
+            vcsProcessed = projectVcs.normalize(),
+            scopeDependencies = setOfNotNull(
+                packageList.dependencies.filterNot { it.isExcluded }.toScope(MAIN_SCOPE_NAME),
+                packageList.dependencies.filter { it.isExcluded }.toScope(EXCLUDED_SCOPE_NAME)
+            )
+        )
+
+        val ortResult = OrtResult(
+            analyzer = AnalyzerRun.EMPTY.copy(
+                result = AnalyzerResult(
+                    projects = setOf(project),
+                    packages = packageList.dependencies.mapTo(mutableSetOf()) { it.toPackage() }
+                ),
+                environment = Environment()
+            ),
+            repository = Repository(
+                vcs = projectVcs.normalize(),
+                config = RepositoryConfiguration(
+                    excludes = Excludes(
+                        scopes = listOf(
+                            ScopeExclude(EXCLUDED_SCOPE_NAME, ScopeExcludeReason.DEV_DEPENDENCY_OF)
+                        )
+                    )
+                )
+            )
+        )
+
+        writeOrtResult(ortResult, ortFile)
+    }
+}
+
+private const val DEFAULT_PROJECT_NAME = "unknown"
+private const val EXCLUDED_SCOPE_NAME = "excluded"
+private const val MAIN_SCOPE_NAME = "main"
+
+private data class PackageList(
+    val name: String? = null,
+    val vcsType: VcsType? = null,
+    val vcsUrl: String? = null,
+    val vcsRevision: String? = null,
+    val vcsPath: String? = null,
+    val dependencies: List<Dependency> = emptyList()
+)
+
+private data class Dependency(
+    val id: Identifier,
+    val vcsType: VcsType? = null,
+    val vcsUrl: String? = null,
+    val vcsRevision: String? = null,
+    val vcsPath: String? = null,
+    val sourceArtifactUrl: String? = null,
+    val isExcluded: Boolean = false,
+    val isDynamicallyLinked: Boolean = false
+)
+
+private fun Collection<Dependency>.toScope(name: String): Scope =
+    Scope(
+        name = name,
+        dependencies = mapTo(mutableSetOf()) { dependency ->
+            PackageReference(
+                id = dependency.id,
+                linkage = PackageLinkage.STATIC.takeUnless { dependency.isDynamicallyLinked } ?: PackageLinkage.DYNAMIC
+            )
+        }
+    )
+
+private fun Dependency.toPackage(): Package {
+    val vcs = VcsInfo(
+        type = vcsType ?: VcsType.UNKNOWN,
+        url = vcsUrl.orEmpty(),
+        revision = vcsRevision.orEmpty(),
+        path = vcsPath.orEmpty()
+    )
+
+    return Package.EMPTY.copy(
+        id = id,
+        sourceArtifact = sourceArtifactUrl?.let { RemoteArtifact(url = it, Hash.NONE) }.orEmpty(),
+        vcs = vcs,
+        vcsProcessed = vcs.normalize()
+    )
+}