Refactor the scanner API to support reading more than ScanCode results from ClearlyDefined

plugins/scanners/askalono/src/main/kotlin/Askalono.kt

@@ -61,25 +61,21 @@ class Askalono internal constructor(
         // askalono 0.2.0-beta.1
         output.removePrefix("askalono ")
 
-    override fun scanPath(path: File, context: ScanContext): ScanSummary {
-        val startTime = Instant.now()
-
+    override fun runScanner(path: File, context: ScanContext): String {
         val process = run(
             "--format", "json",
             "crawl", path.absolutePath
         )
 
-        val endTime = Instant.now()
-
         return with(process) {
             if (stderr.isNotBlank()) logger.debug { stderr }
             if (isError) throw ScanException(errorMessage)
 
-            generateSummary(startTime, endTime, stdout)
+            stdout
         }
     }
 
-    private fun generateSummary(startTime: Instant, endTime: Instant, result: String): ScanSummary {
+    override fun createSummary(result: String, startTime: Instant, endTime: Instant): ScanSummary {
         val licenseFindings = mutableSetOf<LicenseFinding>()
 
         result.lines().forEach { line ->

plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt

@@ -31,7 +31,7 @@ import org.ossreviewtoolkit.model.Severity
 import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
-import org.ossreviewtoolkit.model.readTree
+import org.ossreviewtoolkit.model.jsonMapper
 import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory
 import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper
 import org.ossreviewtoolkit.scanner.ScanContext
@@ -68,33 +68,27 @@ class BoyterLc internal constructor(
         // licensechecker version 1.1.1
         output.removePrefix("licensechecker version ")
 
-    override fun scanPath(path: File, context: ScanContext): ScanSummary {
-        val startTime = Instant.now()
-
+    override fun runScanner(path: File, context: ScanContext): String {
         val resultFile = createOrtTempDir().resolve("result.json")
         val process = run(
             *CONFIGURATION_OPTIONS.toTypedArray(),
             "--output", resultFile.absolutePath,
             path.absolutePath
         )
 
-        val endTime = Instant.now()
-
         return with(process) {
             if (stderr.isNotBlank()) logger.debug { stderr }
             if (isError) throw ScanException(errorMessage)
 
-            generateSummary(startTime, endTime, resultFile).also {
-                resultFile.parentFile.safeDeleteRecursively(force = true)
-            }
+            resultFile.readText().also { resultFile.parentFile.safeDeleteRecursively(force = true) }
         }
     }
 
-    private fun generateSummary(startTime: Instant, endTime: Instant, resultFile: File): ScanSummary {
+    override fun createSummary(result: String, startTime: Instant, endTime: Instant): ScanSummary {
         val licenseFindings = mutableSetOf<LicenseFinding>()
-        val result = resultFile.readTree()
+        val json = jsonMapper.readTree(result)
 
-        result.flatMapTo(licenseFindings) { file ->
+        json.flatMapTo(licenseFindings) { file ->
             val filePath = File(file["Directory"].textValue(), file["Filename"].textValue())
             file["LicenseGuesses"].map {
                 LicenseFinding(

plugins/scanners/licensee/src/main/kotlin/Licensee.kt

@@ -60,26 +60,22 @@ class Licensee internal constructor(
 
     override fun getVersionArguments() = "version"
 
-    override fun scanPath(path: File, context: ScanContext): ScanSummary {
-        val startTime = Instant.now()
-
+    override fun runScanner(path: File, context: ScanContext): String {
         val process = run(
             "detect",
             *CONFIGURATION_OPTIONS.toTypedArray(),
             path.absolutePath
         )
 
-        val endTime = Instant.now()
-
         return with(process) {
             if (stderr.isNotBlank()) logger.debug { stderr }
             if (isError) throw ScanException(errorMessage)
 
-            generateSummary(startTime, endTime, stdout)
+            stdout
         }
     }
 
-    private fun generateSummary(startTime: Instant, endTime: Instant, result: String): ScanSummary {
+    override fun createSummary(result: String, startTime: Instant, endTime: Instant): ScanSummary {
         val licenseFindings = mutableSetOf<LicenseFinding>()
 
         val json = jsonMapper.readTree(result)

scanner/src/main/kotlin/CommandLinePathScannerWrapper.kt

@@ -19,8 +19,12 @@
 
 package org.ossreviewtoolkit.scanner
 
+import java.io.File
+import java.time.Instant
+
 import org.apache.logging.log4j.kotlin.Logging
 
+import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerDetails
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 
@@ -36,4 +40,22 @@ abstract class CommandLinePathScannerWrapper(name: String) : PathScannerWrapper,
     abstract val configuration: String
 
     override val details by lazy { ScannerDetails(name, getVersion(), configuration) }
+
+    final override fun scanPath(path: File, context: ScanContext): ScanSummary {
+        val startTime = Instant.now()
+        val result = runScanner(path, context)
+        val endTime = Instant.now()
+        return createSummary(result, startTime, endTime)
+    }
+
+    /**
+     * Run the scanner on the given [path] with the given [context].
+     */
+    abstract fun runScanner(path: File, context: ScanContext): String
+
+    /**
+     * Create a [ScanSummary] from the scan [result] in a scanner-native format. If the [result] itself does not contain
+     * time information, [startTime] and [endTime] may be used instead.
+     */
+    abstract fun createSummary(result: String, startTime: Instant, endTime: Instant): ScanSummary
 }

scanner/src/main/kotlin/scanners/scancode/ScanCode.kt

@@ -20,6 +20,7 @@
 package org.ossreviewtoolkit.scanner.scanners.scancode
 
 import java.io.File
+import java.time.Instant
 
 import kotlin.math.max
 
@@ -29,7 +30,7 @@ import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerDetails
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
-import org.ossreviewtoolkit.model.readTree
+import org.ossreviewtoolkit.model.jsonMapper
 import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory
 import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper
 import org.ossreviewtoolkit.scanner.ScanContext
@@ -127,6 +128,18 @@ class ScanCode internal constructor(
     override fun command(workingDir: File?) =
         listOfNotNull(workingDir, if (Os.isWindows) "scancode.bat" else "scancode").joinToString(File.separator)
 
+    override fun getVersion(workingDir: File?): String =
+        // The release candidate version names lack a hyphen in between the minor version and the extension, e.g.
+        // 3.2.1rc2. Insert that hyphen for compatibility with Semver.
+        super.getVersion(workingDir).let {
+            val index = it.indexOf("rc")
+            if (index != -1) {
+                "${it.substring(0, index)}-${it.substring(index)}"
+            } else {
+                it
+            }
+        }
+
     override fun getVersionRequirement(): RangesList = RangesListFactory.create(">=3.0.0")
 
     override fun transformVersion(output: String): String {
@@ -139,26 +152,31 @@ class ScanCode internal constructor(
         }.orEmpty()
     }
 
-    override fun scanPath(path: File, context: ScanContext): ScanSummary {
+    override fun runScanner(path: File, context: ScanContext): String {
         val resultFile = createOrtTempDir().resolve("result.json")
         val process = runScanCode(path, resultFile)
 
-        val result = resultFile.readTree()
-        resultFile.parentFile.safeDeleteRecursively(force = true)
+        return with(process) {
+            if (stderr.isNotBlank()) logger.debug { stderr }
+
+            // Do not throw yet if the process exited with an error as some errors might turn out to be tolerable during
+            // parsing.
+
+            resultFile.readText().also { resultFile.parentFile.safeDeleteRecursively(force = true) }
+        }
+    }
 
+    override fun createSummary(result: String, startTime: Instant, endTime: Instant): ScanSummary {
+        val json = jsonMapper.readTree(result)
         val parseLicenseExpressions = scanCodeConfiguration["parseLicenseExpressions"].isTrue()
-        val summary = generateSummary(result, parseLicenseExpressions)
+        val summary = generateSummary(json, parseLicenseExpressions)
 
         val issues = summary.issues.toMutableList()
 
         mapUnknownIssues(issues)
         mapTimeoutErrors(issues)
 
-        return with(process) {
-            if (stderr.isNotBlank()) logger.debug { stderr }
-
-            summary.copy(issues = issues)
-        }
+        return summary.copy(issues = issues)
     }
 
     /**
@@ -174,16 +192,4 @@ class ScanCode internal constructor(
         OUTPUT_FORMAT_OPTION,
         resultFile.absolutePath
     )
-
-    override fun getVersion(workingDir: File?): String =
-        // The release candidate version names lack a hyphen in between the minor version and the extension, e.g.
-        // 3.2.1rc2. Insert that hyphen for compatibility with Semver.
-        super.getVersion(workingDir).let {
-            val index = it.indexOf("rc")
-            if (index != -1) {
-                "${it.substring(0, index)}-${it.substring(index)}"
-            } else {
-                it
-            }
-        }
 }

scanner/src/main/kotlin/scanners/scancode/ScanCodeResultParser.kt

@@ -17,8 +17,6 @@
  * License-Filename: LICENSE
  */
 
-@file:Suppress("TooManyFunctions")
-
 package org.ossreviewtoolkit.scanner.scanners.scancode
 
 import com.fasterxml.jackson.databind.JsonNode
@@ -31,7 +29,6 @@ import org.ossreviewtoolkit.model.CopyrightFinding
 import org.ossreviewtoolkit.model.Issue
 import org.ossreviewtoolkit.model.LicenseFinding
 import org.ossreviewtoolkit.model.ScanSummary
-import org.ossreviewtoolkit.model.ScannerDetails
 import org.ossreviewtoolkit.model.Severity
 import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.createAndLogIssue
@@ -111,44 +108,6 @@ internal fun generateSummary(result: JsonNode, parseExpressions: Boolean = true)
     )
 }
 
-/**
- * Generate details for the given raw ScanCode [result].
- */
-internal fun generateScannerDetails(result: JsonNode): ScannerDetails {
-    val header = result["headers"].single()
-    val version = header["tool_version"].textValueOrEmpty()
-    val config = generateScannerOptions(header["options"])
-    return ScannerDetails(ScanCode.SCANNER_NAME, version, config)
-}
-
-/**
- * Convert the JSON node with ScanCode [options] to a string that corresponds to the options as they have been passed on
- * the command line.
- */
-private fun generateScannerOptions(options: JsonNode?): String {
-    fun addValues(list: MutableList<String>, node: JsonNode, key: String) {
-        if (node.isEmpty) {
-            list += key
-            list += node.asText()
-        } else {
-            node.forEach {
-                list += key
-                list += it.asText()
-            }
-        }
-    }
-
-    return options?.let {
-        val optionList = mutableListOf<String>()
-
-        it.fieldNames().asSequence().forEach { option ->
-            addValues(optionList, it[option], option)
-        }
-
-        optionList.joinToString(separator = " ")
-    }.orEmpty()
-}
-
 private fun getInputPath(result: JsonNode): String {
     val header = result["headers"].single()
     val input = header["options"]["input"]