From eb676c44ed6e5ba6dedd03ded6acbbca4814cfc2 Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Wed, 29 Mar 2023 14:54:52 +0200 Subject: [PATCH 1/5] feat(fossid-webapp): Extract the function `createSingleIssueSummary` This reduces code duplication and will allow to create single issue summaries in other locations of the FossID scanner. Signed-off-by: Nicolas Nobelis --- .../src/main/kotlin/scanners/fossid/FossId.kt | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scanner/src/main/kotlin/scanners/fossid/FossId.kt b/scanner/src/main/kotlin/scanners/fossid/FossId.kt index 3382b1f1c4752..f7f94c1a05812 100644 --- a/scanner/src/main/kotlin/scanners/fossid/FossId.kt +++ b/scanner/src/main/kotlin/scanners/fossid/FossId.kt @@ -206,11 +206,22 @@ class FossId internal constructor( } } + /** + * Create a [ScanSummary] containing a single [issue], started at [startTime] and finished at [endTime]. + */ + private fun createSingleIssueSummary( + startTime: Instant, + endTime: Instant = Instant.now(), + issue: Issue + ) = ScanSummary( + startTime, endTime, "", sortedSetOf(), sortedSetOf(), listOf(issue) + ) + override fun scanPackage(pkg: Package, context: ScanContext): ScanResult { val (result, duration) = measureTimedValue { fun createSingleIssueResult(issue: Issue, provenance: Provenance): ScanResult { val time = Instant.now() - val summary = ScanSummary(time, time, "", sortedSetOf(), sortedSetOf(), listOf(issue)) + val summary = createSingleIssueSummary(time, time, issue) return ScanResult(provenance, details, summary) } @@ -284,9 +295,7 @@ class FossId internal constructor( "Scan results need to be inspected on the server instance.", severity = Severity.HINT ) - val summary = ScanSummary( - startTime, Instant.now(), "", sortedSetOf(), sortedSetOf(), listOf(issue) - ) + val summary = createSingleIssueSummary(startTime, issue = issue) ScanResult( provenance, @@ -302,7 +311,7 @@ class FossId internal constructor( source = name, message = "Failed to scan package '${pkg.id.toCoordinates()}' from $url." ) - val summary = ScanSummary(startTime, Instant.now(), "", sortedSetOf(), sortedSetOf(), listOf(issue)) + val summary = createSingleIssueSummary(startTime, issue = issue) if (!config.keepFailedScans) { createdScans.forEach { code -> From 5532c68721d5cfb13317c34f2180054294b158fa Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Thu, 30 Mar 2023 10:36:31 +0200 Subject: [PATCH 2/5] feat(model): Extend the model to capture Snippets from snippet scanners Snippet scanners such as ScanOSS [1] and FossID [2] can identify code snippets potentially coming from a third party source. To do so, they scan the Internet for source code and build a Knowledge Base (KB). Then, the source code to check for snippets is scanned and compared against this KB. Snippet Findings are difference in nature from License and Copyright findings as they reference a third party sourcecode. Therefore, this commit adds a new property ORT data model in the `ScanSummary` to carry these snippet findings. This model has been created by comparing the results from FossID and ScanOSS and trying to find a common abstraction. This is currently the minimal model required to handle snippets. Further properties will be added in the future. Blackduck [3] is another scanner considered for integration in ORT [4] which supports snippets. However since it does not deliver snippets through its API, it was not considered when designing the snippet data model for ORT. Fixes: #3265. [1]: https://www.scanoss.com/ [2]: https://fossid.com/ [3]: https://www.synopsys.com/software-integrity/security-testing/software-composition-analysis.html [4]: https://github.com/oss-review-toolkit/ort/issues/4632 Signed-off-by: Nicolas Nobelis --- model/src/main/kotlin/ScanSummary.kt | 13 ++++- model/src/main/kotlin/utils/Snippet.kt | 53 +++++++++++++++++++ model/src/main/kotlin/utils/SnippetFinding.kt | 39 ++++++++++++++ .../main/kotlin/utils/SortedSetConverters.kt | 5 ++ ...my-expected-output-for-analyzer-result.yml | 5 ++ .../provenance/NestedProvenanceScanResult.kt | 24 +++++++++ .../src/main/kotlin/scanners/fossid/FossId.kt | 2 +- 7 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 model/src/main/kotlin/utils/Snippet.kt create mode 100644 model/src/main/kotlin/utils/SnippetFinding.kt diff --git a/model/src/main/kotlin/ScanSummary.kt b/model/src/main/kotlin/ScanSummary.kt index 9ab9bf4edb37c..aa2a37d92c2f2 100644 --- a/model/src/main/kotlin/ScanSummary.kt +++ b/model/src/main/kotlin/ScanSummary.kt @@ -23,12 +23,15 @@ import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.annotation.JsonSerialize import java.time.Instant import java.util.SortedSet import org.ossreviewtoolkit.model.config.LicenseFilePatterns import org.ossreviewtoolkit.model.utils.RootLicenseMatcher +import org.ossreviewtoolkit.model.utils.SnippetFinding +import org.ossreviewtoolkit.model.utils.SnippetFindingSortedSetConverter import org.ossreviewtoolkit.utils.common.FileMatcher import org.ossreviewtoolkit.utils.spdx.SpdxExpression @@ -66,6 +69,13 @@ data class ScanSummary( @JsonProperty("copyrights") val copyrightFindings: SortedSet, + /** + * The detected snippet findings. + */ + @JsonProperty("snippets") + @JsonSerialize(converter = SnippetFindingSortedSetConverter::class) + val snippetFindings: Set = emptySet(), + /** * The list of issues that occurred during the scan. This property is not serialized if the list is empty to reduce * the size of the result file. If there are no issues at all, [ScannerRun.hasIssues] already contains that @@ -84,7 +94,8 @@ data class ScanSummary( endTime = Instant.EPOCH, packageVerificationCode = "", licenseFindings = sortedSetOf(), - copyrightFindings = sortedSetOf() + copyrightFindings = sortedSetOf(), + snippetFindings = sortedSetOf() ) } diff --git a/model/src/main/kotlin/utils/Snippet.kt b/model/src/main/kotlin/utils/Snippet.kt new file mode 100644 index 0000000000000..e1c4a62b644ec --- /dev/null +++ b/model/src/main/kotlin/utils/Snippet.kt @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (See ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.utils + +import org.ossreviewtoolkit.model.Provenance +import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.utils.spdx.SpdxExpression + +data class Snippet( + /** + * The matching score between the code being scanned and the code snippet. This is scanner specific (e.g. for + * ScanOSS this is a percentage). + */ + val score: Float, + + /** + * The text location in the snippet that has matched. + */ + val location: TextLocation, + + /** + * The provenance of the snippet, either an artifact or a repository. + */ + val provenance: Provenance, + + /** + * The purl representing the author/vendor, artifact, version of the code snippet. If the snippet scanner does not + * natively support purls, it will be generated by ORT. + */ + val purl: String, + + /** + * The license of the component the code snippet is commit from. + */ + val licenses: SpdxExpression +) diff --git a/model/src/main/kotlin/utils/SnippetFinding.kt b/model/src/main/kotlin/utils/SnippetFinding.kt new file mode 100644 index 0000000000000..67c6523b1e2bf --- /dev/null +++ b/model/src/main/kotlin/utils/SnippetFinding.kt @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.utils + +import org.ossreviewtoolkit.model.TextLocation + +/** + * A class representing a snippet finding for a source file. A snippet finding is a code snippet from another origin, + * matching the code being scanned. + * It is meant to be reviewed by an operator as it could be a false positive. + */ +data class SnippetFinding( + /** + * The text location in the scanned source file where the snippet has matched. + */ + val sourceLocation: TextLocation, + + /** + * The corresponding snippet. + */ + val snippet: Snippet +) diff --git a/model/src/main/kotlin/utils/SortedSetConverters.kt b/model/src/main/kotlin/utils/SortedSetConverters.kt index b4780aee3f907..8f75d682c7db8 100644 --- a/model/src/main/kotlin/utils/SortedSetConverters.kt +++ b/model/src/main/kotlin/utils/SortedSetConverters.kt @@ -36,4 +36,9 @@ class ProjectSortedSetConverter : StdConverter, SortedSet> override fun convert(value: Set) = value.toSortedSet(compareBy { it.id }) } +class SnippetFindingSortedSetConverter : StdConverter, SortedSet>() { + override fun convert(value: Set) = + value.toSortedSet(compareBy { it.sourceLocation.path }.thenByDescending { it.snippet.purl }) +} + // TODO: Add more converters to get rid of Comparable implementations that just serve sorted output. diff --git a/scanner/src/funTest/assets/dummy-expected-output-for-analyzer-result.yml b/scanner/src/funTest/assets/dummy-expected-output-for-analyzer-result.yml index 9c24eb3714402..c3330ef99a632 100644 --- a/scanner/src/funTest/assets/dummy-expected-output-for-analyzer-result.yml +++ b/scanner/src/funTest/assets/dummy-expected-output-for-analyzer-result.yml @@ -237,6 +237,7 @@ scanner: package_verification_code: "" licenses: [] copyrights: [] + snippets: [] issues: - timestamp: "1970-01-01T00:00:00Z" source: "scanner" @@ -281,6 +282,7 @@ scanner: start_line: -1 end_line: -1 copyrights: [] + snippets: [] Maven:org.apache.commons:commons-lang3:3.5: - provenance: source_artifact: @@ -308,6 +310,7 @@ scanner: start_line: -1 end_line: -1 copyrights: [] + snippets: [] Maven:org.apache.commons:commons-text:1.1: - provenance: source_artifact: @@ -335,6 +338,7 @@ scanner: start_line: -1 end_line: -1 copyrights: [] + snippets: [] Maven:org.hamcrest:hamcrest-core:1.3: - provenance: source_artifact: @@ -362,6 +366,7 @@ scanner: start_line: -1 end_line: -1 copyrights: [] + snippets: [] storage_stats: num_reads: 0 num_hits: 0 diff --git a/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt b/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt index 03a6cc7de222b..a892a687b38b1 100644 --- a/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt +++ b/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt @@ -29,6 +29,7 @@ import org.ossreviewtoolkit.model.OrtResult import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary +import org.ossreviewtoolkit.model.utils.SnippetFinding /** * A class that contains all [ScanResult]s for a [NestedProvenance]. @@ -87,6 +88,7 @@ data class NestedProvenanceScanResult( val licenseFindings = scanResultsForScanner.mergeLicenseFindings() val copyrightFindings = scanResultsForScanner.mergeCopyrightFindings() + val snippetFindings = scanResultsForScanner.mergeSnippetFindings() ScanResult( provenance = nestedProvenance.root, @@ -97,6 +99,7 @@ data class NestedProvenanceScanResult( packageVerificationCode = "", licenseFindings = licenseFindings, copyrightFindings = copyrightFindings, + snippetFindings = snippetFindings, issues = issues ), additionalData = allScanResults.map { it.additionalData }.reduce { acc, map -> acc + map } @@ -130,6 +133,27 @@ data class NestedProvenanceScanResult( return findings } + private fun Map>.mergeSnippetFindings(): Set { + val findingsByPath = mapKeys { getPath(it.key) }.mapValues { (_, scanResults) -> + mutableSetOf().also { acc -> + scanResults.forEach { acc += it.summary.snippetFindings } + } + } + + val allFindings = mutableSetOf() + + findingsByPath.forEach { (path, findings) -> + val prefix = if (path.isEmpty()) path else "$path/" + + allFindings += findings.map { finding -> + val newPath = "$prefix${finding.sourceLocation.path}" + finding.copy(sourceLocation = finding.sourceLocation.copy(path = newPath)) + } + } + + return allFindings + } + private fun getPath(provenance: KnownProvenance) = nestedProvenance.getPath(provenance) fun filterByIgnorePatterns(ignorePatterns: List): NestedProvenanceScanResult = diff --git a/scanner/src/main/kotlin/scanners/fossid/FossId.kt b/scanner/src/main/kotlin/scanners/fossid/FossId.kt index f7f94c1a05812..dfcca886d596e 100644 --- a/scanner/src/main/kotlin/scanners/fossid/FossId.kt +++ b/scanner/src/main/kotlin/scanners/fossid/FossId.kt @@ -214,7 +214,7 @@ class FossId internal constructor( endTime: Instant = Instant.now(), issue: Issue ) = ScanSummary( - startTime, endTime, "", sortedSetOf(), sortedSetOf(), listOf(issue) + startTime, endTime, "", sortedSetOf(), sortedSetOf(), issues = listOf(issue) ) override fun scanPackage(pkg: Package, context: ScanContext): ScanResult { From 0666631d3e2215fedad25d0af8e11109be399eef Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Fri, 21 Apr 2023 10:22:07 +0200 Subject: [PATCH 3/5] feat(model): Add GitHub and GitLab to Purl types FossID returns packages containing URLs of these two websites. Since FossID does not natively support Purls, a mapping needs to be done from the native URL to the Purl, using the `PackageProvider` enum. Such mapping will be added in a future commit. See https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst. Signed-off-by: Nicolas Nobelis --- model/src/main/kotlin/utils/Extensions.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/model/src/main/kotlin/utils/Extensions.kt b/model/src/main/kotlin/utils/Extensions.kt index f9c5e8a0a8815..ed35cd076522d 100644 --- a/model/src/main/kotlin/utils/Extensions.kt +++ b/model/src/main/kotlin/utils/Extensions.kt @@ -143,6 +143,8 @@ enum class PurlType(private val value: String) { DEBIAN("deb"), DRUPAL("drupal"), GEM("gem"), + GITHUB("github"), + GITLAB("gitlab"), GOLANG("golang"), MAVEN("maven"), NPM("npm"), From e8be2f765d7bed8ec7be36572526715baa0123a6 Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Thu, 30 Mar 2023 10:38:05 +0200 Subject: [PATCH 4/5] feat(fossid-webapp): Map FossID snippets to the `ScanSummary` When FossId identifies a file matching snippets, it is a pending file. An operator needs to log to FossID UI and use the license of a snippet or manually enter difference license information. Then the file is marked as "identified". Currently, the FossID scanner in ORT returns the list of all pending files in `ScanSummary` issues, with a severity of `HINT`. This commit maps the snippets of pending files using the newly-created snippet data model. The pending files are still listed as issues: This will be removed in a future commit as it is a breaking change. Signed-off-by: Nicolas Nobelis --- .../src/main/kotlin/scanners/fossid/FossId.kt | 114 +++++++++++++++++- .../scanners/fossid/FossIdScanResults.kt | 4 +- .../test/kotlin/scanners/fossid/FossIdTest.kt | 104 +++++++++++++++- 3 files changed, 217 insertions(+), 5 deletions(-) diff --git a/scanner/src/main/kotlin/scanners/fossid/FossId.kt b/scanner/src/main/kotlin/scanners/fossid/FossId.kt index dfcca886d596e..87e879ca305bc 100644 --- a/scanner/src/main/kotlin/scanners/fossid/FossId.kt +++ b/scanner/src/main/kotlin/scanners/fossid/FossId.kt @@ -27,6 +27,9 @@ import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import kotlin.time.measureTimedValue +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull @@ -48,6 +51,7 @@ import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles import org.ossreviewtoolkit.clients.fossid.listPendingFiles import org.ossreviewtoolkit.clients.fossid.listScansForProject +import org.ossreviewtoolkit.clients.fossid.listSnippets import org.ossreviewtoolkit.clients.fossid.model.Project import org.ossreviewtoolkit.clients.fossid.model.Scan import org.ossreviewtoolkit.clients.fossid.model.rules.RuleScope @@ -56,28 +60,40 @@ import org.ossreviewtoolkit.clients.fossid.model.status.DownloadStatus import org.ossreviewtoolkit.clients.fossid.model.status.ScanStatus import org.ossreviewtoolkit.clients.fossid.runScan import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.model.ArtifactProvenance +import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageProvider import org.ossreviewtoolkit.model.Provenance +import org.ossreviewtoolkit.model.RemoteArtifact import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.ScannerDetails import org.ossreviewtoolkit.model.Severity +import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.Options import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.createAndLogIssue +import org.ossreviewtoolkit.model.utils.PurlType +import org.ossreviewtoolkit.model.utils.Snippet +import org.ossreviewtoolkit.model.utils.SnippetFinding import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory import org.ossreviewtoolkit.scanner.PackageScannerWrapper import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerCriteria +import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.enumSetOf import org.ossreviewtoolkit.utils.common.replaceCredentialsInUri import org.ossreviewtoolkit.utils.ort.showStackTrace +import org.ossreviewtoolkit.utils.spdx.SpdxConstants +import org.ossreviewtoolkit.utils.spdx.toSpdx /** * A wrapper for [FossID](https://fossid.com/). @@ -746,7 +762,23 @@ class FossId internal constructor( "${pendingFiles.size} pending files have been returned for scan '$scanCode'." } - return RawResults(identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles, pendingFiles) + val snippets = runBlocking(Dispatchers.IO) { + pendingFiles.map { + async { + logger.info { "Listing snippet for $it..." } + val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, it) + .checkResponse("list snippets") + val snippets = checkNotNull(snippetResponse.data) { + "Snippet could not be listed. Response was ${snippetResponse.message}." + } + logger.info { "${snippets.size} snippets." } + + it to snippets.toSet() + } + }.awaitAll().toMap() + } + + return RawResults(identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles, pendingFiles, snippets) } /** @@ -760,10 +792,61 @@ class FossId internal constructor( scanId: String ): ScanResult { // TODO: Maybe get issues from FossID (see has_failed_scan_files, get_failed_files and maybe get_scan_log). + + // TODO: Deprecation: Remove the pending files in issues. This is a breaking change. val issues = rawResults.listPendingFiles.mapTo(mutableListOf()) { Issue(source = name, message = "Pending identification for '$it'.", severity = Severity.HINT) } + val snippetFindings = mutableSetOf() + val fakeLocation = TextLocation(".", TextLocation.UNKNOWN_LINE) + snippetFindings += rawResults.listSnippets.flatMap { (file, rawSnippets) -> + val snippets = rawSnippets.map { + val license = it.artifactLicense?.let { + runCatching { + LicenseFinding.createAndMap( + it, + fakeLocation, + detectedLicenseMapping = scannerConfig.detectedLicenseMapping + ).license + }.onFailure { spdxException -> + issues += FossId.createAndLogIssue( + source = "FossId", + message = "Failed to parse license '$it' as an SPDX expression:" + + " ${spdxException.collectMessages()}" + ) + }.getOrNull() + } ?: SpdxConstants.NOASSERTION.toSpdx() + + // FossID does not return the hash of the remote artifact. Instead, it returns the MD5 hash of the + // matched file in the remote artifact as part of the "match_file_id" property. + val snippetProvenance = it.url?.let { url -> + ArtifactProvenance(RemoteArtifact(url, Hash.NONE)) + } ?: UnknownProvenance + val purlType = it.url?.let { url -> urlToPackageType(url, issues)?.toString() } ?: "generic" + + // TODO: FossID doesn't return the line numbers of the match, only the character range. One must use + // another call "getMatchedLine" to retrieve the matched line numbers. Unfortunately, this is a + // call per snippet which is too expensive. When it is available for a batch of snippets, it can + // be used here. + Snippet( + it.score.toFloat(), + TextLocation(it.file, TextLocation.UNKNOWN_LINE), + snippetProvenance, + "pkg:$purlType/${it.author}/${it.artifact}@${it.version}", + license + ) + } + + val sourceLocation = TextLocation(file, TextLocation.UNKNOWN_LINE) + snippets.map { + SnippetFinding( + sourceLocation, + it + ) + } + } + val ignoredFiles = rawResults.listIgnoredFiles.associateBy { it.path } val (licenseFindings, copyrightFindings) = rawResults.markedAsIdentifiedFiles.ifEmpty { @@ -776,6 +859,7 @@ class FossId internal constructor( packageVerificationCode = "", licenseFindings = licenseFindings.toSortedSet(), copyrightFindings = copyrightFindings.toSortedSet(), + snippetFindings = snippetFindings, issues = issues ) @@ -786,4 +870,32 @@ class FossId internal constructor( mapOf(SCAN_CODE_KEY to scanCode, SCAN_ID_KEY to scanId, SERVER_URL_KEY to config.serverUrl) ) } + + /** + * Return the [PurlType] as determined from the given [url], or null if there is no match, in which case an issue + * will be added to [issues]. + */ + private fun urlToPackageType(url: String, issues: MutableList): PurlType? = + when (val provider = PackageProvider.get(url)) { + PackageProvider.COCOAPODS -> PurlType.COCOAPODS + PackageProvider.CRATES_IO -> PurlType.CARGO + PackageProvider.DEBIAN -> PurlType.DEBIAN + PackageProvider.GITHUB -> PurlType.GITHUB + PackageProvider.GITLAB -> PurlType.GITLAB + PackageProvider.GOLANG -> PurlType.GOLANG + PackageProvider.MAVEN_CENTRAL, PackageProvider.MAVEN_GOOGLE -> PurlType.MAVEN + PackageProvider.NPM_JS -> PurlType.NPM + PackageProvider.NUGET -> PurlType.NUGET + PackageProvider.PACKAGIST -> PurlType.COMPOSER + PackageProvider.PYPI -> PurlType.PYPI + PackageProvider.RUBYGEMS -> PurlType.GEM + + else -> { + issues += FossId.createAndLogIssue( + source = "FossId", + message = "Cannot determine PURL type for url '$url' and provider '$provider'." + ) + null + } + } } diff --git a/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt b/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt index 3553ba764be08..fc420014ea72c 100644 --- a/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt +++ b/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt @@ -22,6 +22,7 @@ package org.ossreviewtoolkit.scanner.scanners.fossid import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile +import org.ossreviewtoolkit.clients.fossid.model.result.Snippet import org.ossreviewtoolkit.clients.fossid.model.summary.Summarizable import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.Issue @@ -37,7 +38,8 @@ internal data class RawResults( val identifiedFiles: List, val markedAsIdentifiedFiles: List, val listIgnoredFiles: List, - val listPendingFiles: List + val listPendingFiles: List, + val listSnippets: Map> ) /** diff --git a/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt b/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt index bf74903377c8c..947f3d1bb3fe6 100644 --- a/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt +++ b/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt @@ -62,6 +62,7 @@ import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles import org.ossreviewtoolkit.clients.fossid.listPendingFiles import org.ossreviewtoolkit.clients.fossid.listScansForProject +import org.ossreviewtoolkit.clients.fossid.listSnippets import org.ossreviewtoolkit.clients.fossid.model.Scan import org.ossreviewtoolkit.clients.fossid.model.identification.common.LicenseMatchType import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile @@ -69,6 +70,8 @@ import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredF import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.License import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.LicenseFile import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile +import org.ossreviewtoolkit.clients.fossid.model.result.MatchType +import org.ossreviewtoolkit.clients.fossid.model.result.Snippet import org.ossreviewtoolkit.clients.fossid.model.rules.IgnoreRule import org.ossreviewtoolkit.clients.fossid.model.rules.RuleScope import org.ossreviewtoolkit.clients.fossid.model.rules.RuleType @@ -78,23 +81,29 @@ import org.ossreviewtoolkit.clients.fossid.model.status.UnversionedScanDescripti import org.ossreviewtoolkit.clients.fossid.runScan import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.downloader.vcs.Git +import org.ossreviewtoolkit.model.ArtifactProvenance import org.ossreviewtoolkit.model.CopyrightFinding +import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.Issue import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.PackageType +import org.ossreviewtoolkit.model.RemoteArtifact import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.utils.Snippet as OrtSnippet +import org.ossreviewtoolkit.model.utils.SnippetFinding import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SCAN_CODE_KEY import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SCAN_ID_KEY import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SERVER_URL_KEY import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.convertGitUrlToProjectName +import org.ossreviewtoolkit.utils.spdx.SpdxExpression @Suppress("LargeClass") class FossIdTest : WordSpec({ @@ -314,6 +323,7 @@ class FossIdTest : WordSpec({ summary.licenseFindings shouldContainExactlyInAnyOrder expectedLicenseFindings } + // TODO: Deprecation: Remove the pending files in issues. This is a breaking change. "report pending files as issues" { val projectCode = projectCode(PROJECT) val scanCode = scanCode(PROJECT, null) @@ -328,19 +338,57 @@ class FossIdTest : WordSpec({ .expectCheckScanStatus(scanCode, ScanStatus.FINISHED) .expectCreateScan(projectCode, scanCode, vcsInfo, "") .expectDownload(scanCode) - .mockFiles(scanCode, pendingRange = 4..5) + .mockFiles(scanCode, pendingRange = 4..5, snippetRange = 1..5) val fossId = createFossId(config) val summary = fossId.scan(createPackage(pkgId, vcsInfo)).summary - val expectedIssues = listOf(createPendingFile(4), createPendingFile(5)).map { + val pendingFilesIssues = listOf(createPendingFile(4), createPendingFile(5)).map { Issue(Instant.EPOCH, "FossId", "Pending identification for '$it'.", Severity.HINT) } + val urlMappingIssues = (1..5).map { + Issue( + Instant.EPOCH, + "FossId", + "Cannot determine PURL type for url 'url$it' and provider 'null'.", + Severity.ERROR + ) + } + // Add the mapping issues from the snippet fake URLs: 5 issues for each pending file. + val expectedIssues = pendingFilesIssues + urlMappingIssues + urlMappingIssues summary.issues.map { it.copy(timestamp = Instant.EPOCH) } shouldBe expectedIssues } + "report pending files as snippets" { + val projectCode = projectCode(PROJECT) + val scanCode = scanCode(PROJECT, null) + val config = createConfig(deltaScans = false) + val vcsInfo = createVcsInfo() + val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode) + val pkgId = createIdentifier(index = 42) + + FossIdRestService.create(config.serverUrl) + .expectProjectRequest(projectCode) + .expectListScans(projectCode, listOf(scan)) + .expectCheckScanStatus(scanCode, ScanStatus.FINISHED) + .expectCreateScan(projectCode, scanCode, vcsInfo, "") + .expectDownload(scanCode) + .mockFiles(scanCode, pendingRange = 1..5, snippetRange = 1..5) + + val fossId = createFossId(config) + + val summary = fossId.scan(createPackage(pkgId, vcsInfo)).summary + + val expectedPendingFile = (1..5).map(::createPendingFile).toSet() + val expectedSnippetFindings = (1..5).map(::createSnippetFindings).flatten() + + summary.snippetFindings shouldHaveSize expectedPendingFile.size * 5 + summary.snippetFindings.map { it.sourceLocation.path }.toSet() shouldBe expectedPendingFile + summary.snippetFindings shouldBe expectedSnippetFindings + } + "create a new project if none exists yet" { val projectCode = projectCode(PROJECT) val scanCode = scanCode(PROJECT, null) @@ -1238,6 +1286,52 @@ private fun createIgnoredFile(index: Int): IgnoredFile = */ private fun createPendingFile(index: Int): String = "/pending/file/$index" +/** + * Generate a FossID snippet based on the given [index]. + */ +private fun createSnippet(index: Int): Snippet = Snippet( + index, + "created$index", + index, + index, + index, + MatchType.PARTIAL, + "reason$index", + "author$index", + "artifact$index", + "version$index", + "MIT", + "releaseDate$index", + "mirror$index", + "file$index", + "fileLicense$index", + "url$index", + "hits$index", + index, + "updated$index", + "cpe$index", + "$index", + "matchField$index", + "classification$index", + "highlighting$index" +) + +/** + * Generate a ORT snippet finding based on the given [index]. + */ +private fun createSnippetFindings(index: Int): Set = (1..5).map { snippetIndex -> + SnippetFinding( + TextLocation("/pending/file/$index", TextLocation.UNKNOWN_LINE), + OrtSnippet( + snippetIndex.toFloat(), + TextLocation("file$snippetIndex", TextLocation.UNKNOWN_LINE), + ArtifactProvenance(RemoteArtifact("url$snippetIndex", Hash.NONE)), + "pkg:generic/author$snippetIndex/artifact$snippetIndex@version$snippetIndex", + SpdxExpression.Companion.parse("MIT") + ) + ) +}.toSet() + /** * Prepare this service mock to answer a request for a project with the given [projectCode]. Return a response with * the given [status] and [error]. @@ -1348,12 +1442,14 @@ private fun FossIdServiceWithVersion.mockFiles( identifiedRange: IntRange = IntRange.EMPTY, markedRange: IntRange = IntRange.EMPTY, ignoredRange: IntRange = IntRange.EMPTY, - pendingRange: IntRange = IntRange.EMPTY + pendingRange: IntRange = IntRange.EMPTY, + snippetRange: IntRange = IntRange.EMPTY ): FossIdServiceWithVersion { val identifiedFiles = identifiedRange.map(::createIdentifiedFile) val markedFiles = markedRange.map(::createMarkedIdentifiedFile) val ignoredFiles = ignoredRange.map(::createIgnoredFile) val pendingFiles = pendingRange.map(::createPendingFile) + val snippets = snippetRange.map(::createSnippet) coEvery { listIdentifiedFiles(USER, API_KEY, scanCode) } returns PolymorphicResponseBody( @@ -1367,6 +1463,8 @@ private fun FossIdServiceWithVersion.mockFiles( PolymorphicResponseBody(status = 1, data = PolymorphicList(ignoredFiles)) coEvery { listPendingFiles(USER, API_KEY, scanCode) } returns PolymorphicResponseBody(status = 1, data = PolymorphicList(pendingFiles)) + coEvery { listSnippets(USER, API_KEY, scanCode, any()) } returns + PolymorphicResponseBody(status = 1, data = PolymorphicList(snippets)) return this } From d1fb1daf6187551ebc119600cc229fbb8df07e80 Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Mon, 3 Apr 2023 08:49:46 +0200 Subject: [PATCH 5/5] feat(scanoss): Map the snippets to the `ScanSummary` This commits maps the snippets in a ScanOSS response using the newly-created snippet data model. Please note that the snippet's license in the test data file has been manipulated to be a license not present in the other identifications of this file. This allows to demonstrate that license findings and snippet findings are disjoint in ORT, even if they are returned together by ScanOSS. Signed-off-by: Nicolas Nobelis --- .../scanners/scanoss/ScanOssResultParser.kt | 68 ++- .../scanoss-semver4j-3.1.0-with-snippet.json | 415 ++++++++++++++++++ .../scanoss/ScanOssResultParserTest.kt | 57 +++ .../scanoss/ScanOssScannerDirectoryTest.kt | 34 +- 4 files changed, 564 insertions(+), 10 deletions(-) create mode 100644 scanner/src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json diff --git a/scanner/src/main/kotlin/scanners/scanoss/ScanOssResultParser.kt b/scanner/src/main/kotlin/scanners/scanoss/ScanOssResultParser.kt index d52117e8d13e0..022b16f85c87e 100644 --- a/scanner/src/main/kotlin/scanners/scanoss/ScanOssResultParser.kt +++ b/scanner/src/main/kotlin/scanners/scanoss/ScanOssResultParser.kt @@ -23,11 +23,17 @@ import java.io.File import java.time.Instant import org.ossreviewtoolkit.clients.scanoss.FullScanResponse +import org.ossreviewtoolkit.clients.scanoss.model.IdentificationType import org.ossreviewtoolkit.clients.scanoss.model.ScanResponse import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.LicenseFinding +import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.utils.Snippet +import org.ossreviewtoolkit.model.utils.SnippetFinding import org.ossreviewtoolkit.utils.spdx.SpdxConstants import org.ossreviewtoolkit.utils.spdx.SpdxExpression import org.ossreviewtoolkit.utils.spdx.calculatePackageVerificationCode @@ -64,11 +70,25 @@ internal fun generateSummary( ): ScanSummary { val licenseFindings = mutableListOf() val copyrightFindings = mutableListOf() + val snippetFindings = mutableSetOf() result.forEach { (_, scanResponses) -> scanResponses.forEach { scanResponse -> - licenseFindings += getLicenseFindings(scanResponse, detectedLicenseMapping) - copyrightFindings += getCopyrightFindings(scanResponse) + if (scanResponse.id == IdentificationType.FILE) { + licenseFindings += getLicenseFindings(scanResponse, detectedLicenseMapping) + copyrightFindings += getCopyrightFindings(scanResponse) + } + + if (scanResponse.id == IdentificationType.SNIPPET) { + val file = requireNotNull(scanResponse.file) + val lines = requireNotNull(scanResponse.lines) + val sourceLocation = convertLines(file, lines) + val snippets = getSnippets(scanResponse) + + snippets.forEach { + snippetFindings += SnippetFinding(sourceLocation, it) + } + } } } @@ -78,6 +98,7 @@ internal fun generateSummary( packageVerificationCode = verificationCode, licenseFindings = licenseFindings.toSortedSet(), copyrightFindings = copyrightFindings.toSortedSet(), + snippetFindings = snippetFindings, issues = emptyList() ) } @@ -131,3 +152,46 @@ private fun getCopyrightFindings(scanResponse: ScanResponse): List { + val matched = requireNotNull(scanResponse.matched) + val fileUrl = requireNotNull(scanResponse.fileUrl) + val ossLines = requireNotNull(scanResponse.ossLines) + val url = requireNotNull(scanResponse.url) + val purls = requireNotNull(scanResponse.purl) + + val licenses = scanResponse.licenses.map { license -> + SpdxExpression.parse(license.name) + }.toSet() + + val score = matched.substringBeforeLast("%").toFloat() + val snippetLocation = convertLines(fileUrl, ossLines) + // TODO: No resolved revision is available. Should a ArtifactProvenance be created instead ? + val snippetProvenance = RepositoryProvenance(VcsInfo(VcsType.UNKNOWN, url, ""), ".") + + return purls.map { + Snippet( + score, + snippetLocation, + snippetProvenance, + it, + licenses.distinct().reduce(SpdxExpression::and).sorted() + ) + }.toSet() +} + +/** + * Split a [lineRange] returned by ScanOSS such as 1-321 into a [TextLocation] for the given [file]. + */ +private fun convertLines(file: String, lineRange: String): TextLocation { + val splitLines = lineRange.split("-") + return if (splitLines.size == 2) { + TextLocation(file, splitLines.first().toInt(), splitLines.last().toInt()) + } else { + TextLocation(file, splitLines.first().toInt()) + } +} diff --git a/scanner/src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json b/scanner/src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json new file mode 100644 index 0000000000000..a553470bd4782 --- /dev/null +++ b/scanner/src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json @@ -0,0 +1,415 @@ +{ + "src/main/java/com/vdurmont/semver4j/Range.java": [ + { + "component": "semver4j", + "file": "com/vdurmont/semver4j/Range.java", + "file_hash": "8b917844b6bb13a7a377d091cac9e231", + "file_url": "https://osskb.org/api/file_contents/8b917844b6bb13a7a377d091cac9e231", + "id": "file", + "latest": "2.2.0-graylog.1", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/Apache-2.0.txt", + "copyleft": "no", + "name": "Apache-2.0", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "yes", + "source": "license_file", + "url": "https://spdx.org/licenses/Apache-2.0.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-2-Clause.txt", + "copyleft": "no", + "name": "BSD-2-Clause", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "license_file", + "url": "https://spdx.org/licenses/BSD-2-Clause.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/EPL-2.0.txt", + "copyleft": "yes", + "name": "EPL-2.0", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "yes", + "source": "license_file", + "url": "https://spdx.org/licenses/EPL-2.0.html" + }, + { + "name": "SSPL", + "source": "license_file", + "url": "https://spdx.org/licenses/SSPL.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:maven/org.graylog.repackaged/semver4j", + "pkg:maven/org.graylog2/graylog2-server", + "pkg:github/graylog2/graylog2-server", + "pkg:maven/org.graylog/graylog-parent", + "pkg:maven/org.graylog/graylog-project-parent", + "pkg:maven/org.graylog.repackaged/os-platform-finder", + "pkg:maven/org.graylog.telemetry/graylog-telemetry-plugin", + "pkg:maven/org.graylog.plugins/graylog-plugin-parent", + "pkg:maven/org.graylog.shaded/kafka09", + "pkg:maven/org.graylog.shaded/elasticsearch5" + ], + "release_date": "2018-07-04", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "8b917844b6bb13a7a377d091cac9e231", + "status": "pending", + "url": "https://mvnrepository.com/artifact/org.graylog.repackaged/semver4j", + "url_hash": "9b0606cb3068e838edb89aeb2158d3be", + "vendor": "org.graylog.repackaged", + "version": "2.2.0-graylog.1" + } + ], + "src/main/java/com/vdurmont/semver4j/Requirement.java":[ + { + "id": "snippet", + "lines": "1-710", + "oss_lines": "1-710", + "matched": "98%", + "file_hash": "6ff2427335b985212c9b79dfa795799f", + "source_hash": "bd4bff27f540f4f2c9de012acc4b48a3", + "file_url": "https://osskb.org/api/file_contents/6ff2427335b985212c9b79dfa795799f", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "vendor": "vdurmont", + "component": "semver4j", + "version": "3.1.0", + "latest": "3.1.0", + "url": "https://github.com/vdurmont/semver4j", + "status": "pending", + "release_date": "2019-09-13", + "file": "src/main/java/com/vdurmont/semver4j/Requirement.java", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "licenses": [ + { + "name": "CC-BY-SA-2.0", + "patent_hints": "no", + "copyleft": "no", + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "source": "component_declared", + "url": "https://scancode-licensedb.aboutcode.org/cc-by-sa-2.0.LICENSE" + } + ], + "server": { + "version": "5.2.5", + "kb_version": { + "monthly": "23.03", + "daily": "23.03.19" + } + } + } + ], + "src/main/java/com/vdurmont/semver4j/Semver.java": [ + { + "component": "semver4j", + "file": "src/main/java/com/vdurmont/semver4j/Semver.java", + "file_hash": "a51addd1e051f8ff4feb0fc879726e2c", + "file_url": "https://osskb.org/api/file_contents/a51addd1e051f8ff4feb0fc879726e2c", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-09-13", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "a51addd1e051f8ff4feb0fc879726e2c", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "vendor": "vdurmont", + "version": "3.1.0" + } + ], + "src/main/java/com/vdurmont/semver4j/SemverException.java": [ + { + "id": "none", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + } + } + ], + "src/main/java/com/vdurmont/semver4j/Tokenizer.java": [ + { + "component": "semver4j", + "file": "src/main/java/com/vdurmont/semver4j/Tokenizer.java", + "file_hash": "72189e2d96994bde6483571eb6bf1825", + "file_url": "https://osskb.org/api/file_contents/72189e2d96994bde6483571eb6bf1825", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-09-13", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "72189e2d96994bde6483571eb6bf1825", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "vendor": "vdurmont", + "version": "3.1.0" + } + ], + "src/test/java/com/vdurmont/semver4j/NpmSemverTest.java": [ + { + "component": "semver4j", + "file": "src/test/java/com/vdurmont/semver4j/NpmSemverTest.java", + "file_hash": "8fa5ef7baab5071e49b2393905231420", + "file_url": "https://osskb.org/api/file_contents/8fa5ef7baab5071e49b2393905231420", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-09-13", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "8fa5ef7baab5071e49b2393905231420", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "vendor": "vdurmont", + "version": "3.1.0" + } + ], + "src/test/java/com/vdurmont/semver4j/RangeTest.java": [ + { + "component": "semver4j", + "file": "src/test/java/com/vdurmont/semver4j/RangeTest.java", + "file_hash": "7069133e87c482fe935a7a54ca5b5a5d", + "file_url": "https://osskb.org/api/file_contents/7069133e87c482fe935a7a54ca5b5a5d", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-08-16", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "7069133e87c482fe935a7a54ca5b5a5d", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "7cc91d1fce4275d9ef74fb5ae1bd4f63", + "vendor": "vdurmont", + "version": "3.0.0" + } + ], + "src/test/java/com/vdurmont/semver4j/RequirementTest.java": [ + { + "component": "semver4j", + "file": "src/test/java/com/vdurmont/semver4j/RequirementTest.java", + "file_hash": "5e9fbaa3bde70a3a5a077fe6d65e36de", + "file_url": "https://osskb.org/api/file_contents/5e9fbaa3bde70a3a5a077fe6d65e36de", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-09-13", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "5e9fbaa3bde70a3a5a077fe6d65e36de", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "vendor": "vdurmont", + "version": "3.1.0" + } + ], + "src/test/java/com/vdurmont/semver4j/SemverTest.java": [ + { + "component": "semver4j", + "file": "src/test/java/com/vdurmont/semver4j/SemverTest.java", + "file_hash": "ef32d9061afb0be4d07285813321e306", + "file_url": "https://osskb.org/api/file_contents/ef32d9061afb0be4d07285813321e306", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-09-13", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "ef32d9061afb0be4d07285813321e306", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "vendor": "vdurmont", + "version": "3.1.0" + } + ], + "src/test/java/com/vdurmont/semver4j/TokenizerTest.java": [ + { + "component": "semver4j", + "file": "src/test/java/com/vdurmont/semver4j/TokenizerTest.java", + "file_hash": "4498810681d4cbdc111ddbf4015b351e", + "file_url": "https://osskb.org/api/file_contents/4498810681d4cbdc111ddbf4015b351e", + "id": "file", + "latest": "3.1.0", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2023-03-26T02:11:00+00:00", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/vdurmont/semver4j" + ], + "release_date": "2019-09-13", + "server": { + "kb_version": { + "daily": "23.03.19", + "monthly": "23.03" + }, + "version": "5.2.5" + }, + "source_hash": "4498810681d4cbdc111ddbf4015b351e", + "status": "pending", + "url": "https://github.com/vdurmont/semver4j", + "url_hash": "b92cd7e4d588747588d1b5fe1f8c664c", + "vendor": "vdurmont", + "version": "3.1.0" + } + ] +} diff --git a/scanner/src/test/kotlin/scanners/scanoss/ScanOssResultParserTest.kt b/scanner/src/test/kotlin/scanners/scanoss/ScanOssResultParserTest.kt index c69b472365b37..8dbb0ed38d80f 100644 --- a/scanner/src/test/kotlin/scanners/scanoss/ScanOssResultParserTest.kt +++ b/scanner/src/test/kotlin/scanners/scanoss/ScanOssResultParserTest.kt @@ -23,6 +23,8 @@ import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.collections.haveSize import io.kotest.matchers.collections.shouldContain +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.should import java.io.File @@ -34,8 +36,14 @@ import org.ossreviewtoolkit.clients.scanoss.FullScanResponse import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.LicenseFinding +import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.utils.Snippet +import org.ossreviewtoolkit.model.utils.SnippetFinding import org.ossreviewtoolkit.utils.spdx.SpdxConstants +import org.ossreviewtoolkit.utils.spdx.SpdxExpression class ScanOssResultParserTest : WordSpec({ "generateSummary()" should { @@ -76,5 +84,54 @@ class ScanOssResultParserTest : WordSpec({ ) ) } + + "properly summarize Semver4j 3.1.0 with snippet findings" { + val result = File("src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json").inputStream().use { + ScanOssService.JSON.decodeFromStream(it) + } + + val time = Instant.now() + val summary = generateSummary(time, time, SpdxConstants.NONE, result, emptyMap()) + + summary.licenses.map { it.toString() } should containExactlyInAnyOrder( + "Apache-2.0", + "BSD-2-Clause", + "EPL-2.0", + "LicenseRef-scanoss-SSPL", + "MIT" + ) + + summary.licenseFindings should haveSize(11) + summary.licenseFindings shouldContain LicenseFinding( + license = "Apache-2.0", + location = TextLocation( + path = "com/vdurmont/semver4j/Range.java", + startLine = TextLocation.UNKNOWN_LINE, + endLine = TextLocation.UNKNOWN_LINE + ), + score = 100.0f + ) + + summary.snippetFindings shouldHaveSize (1) + summary.snippetFindings.shouldContainExactly( + SnippetFinding( + TextLocation("src/main/java/com/vdurmont/semver4j/Requirement.java", 1, 710), + Snippet( + 98.0f, + TextLocation( + "https://osskb.org/api/file_contents/6ff2427335b985212c9b79dfa795799f", + 1, + 710 + ), + RepositoryProvenance( + VcsInfo(VcsType.UNKNOWN, "https://github.com/vdurmont/semver4j", ""), + "." + ), + "pkg:github/vdurmont/semver4j", + SpdxExpression.parse("CC-BY-SA-2.0") + ) + ) + ) + } } }) diff --git a/scanner/src/test/kotlin/scanners/scanoss/ScanOssScannerDirectoryTest.kt b/scanner/src/test/kotlin/scanners/scanoss/ScanOssScannerDirectoryTest.kt index 025047b26959f..65662d15bd0b4 100644 --- a/scanner/src/test/kotlin/scanners/scanoss/ScanOssScannerDirectoryTest.kt +++ b/scanner/src/test/kotlin/scanners/scanoss/ScanOssScannerDirectoryTest.kt @@ -24,6 +24,7 @@ import com.github.tomakehurst.wiremock.core.WireMockConfiguration import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.containExactlyInAnyOrder +import io.kotest.matchers.collections.shouldContainExactly import io.kotest.matchers.should import io.mockk.every @@ -35,10 +36,16 @@ import java.util.UUID import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType +import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.model.utils.Snippet +import org.ossreviewtoolkit.model.utils.SnippetFinding import org.ossreviewtoolkit.scanner.ScanContext +import org.ossreviewtoolkit.utils.spdx.SpdxExpression private val TEST_DIRECTORY_TO_SCAN = File("src/test/assets/scanoss/filesToScan") @@ -91,14 +98,6 @@ class ScanOssScannerDirectoryTest : StringSpec({ with(summary) { licenseFindings should containExactlyInAnyOrder( - LicenseFinding( - license = "Apache-2.0", - location = TextLocation( - path = "utils/src/main/kotlin/ArchiveUtils.kt", - line = TextLocation.UNKNOWN_LINE - ), - score = 99.0f - ), LicenseFinding( license = "Apache-2.0", location = TextLocation( @@ -108,6 +107,25 @@ class ScanOssScannerDirectoryTest : StringSpec({ score = 100.0f ) ) + + snippetFindings.shouldContainExactly( + SnippetFinding( + TextLocation("utils/src/main/kotlin/ArchiveUtils.kt", 1, 240), + Snippet( + 99.0f, + TextLocation( + "https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095", + 128, + 367 + ), + RepositoryProvenance( + VcsInfo(VcsType.UNKNOWN, "https://github.com/scanoss/ort", ""), "." + ), + "pkg:github/scanoss/ort", + SpdxExpression.parse("Apache-2.0") + ) + ) + ) } } })