feat(fossid-webapp): Map FossID snippets to the ScanSummary

nnobelis · nnobelis · commit fc090ad1e30a · 2023-04-12T16:04:01.000+02:00
When FossId identifies a file matching snippets, it is a pending file.
An operator needs to log to FossID UI and use the license of a snippet
or manually enter difference license information. Then the file is
marked as "identified".

Currently, the FossID scanner in ORT returns the list of all pending
files in `ScanSummary` issues, with a severity of `HINT`. This commit
maps the snippets of pending files using the newly-created snippet data
model.
The pending files are still listed as issues: this will be removed in a
future commit as it is a breaking change.

Signed-off-by: Nicolas Nobelis &lt;nicolas.nobelis@bosch.io&gt;
diff --git a/scanner/src/main/kotlin/scanners/fossid/FossId.kt b/scanner/src/main/kotlin/scanners/fossid/FossId.kt
@@ -27,6 +27,9 @@ import kotlin.time.Duration.Companion.minutes
 import kotlin.time.Duration.Companion.seconds
 import kotlin.time.measureTimedValue
 
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeoutOrNull
@@ -48,6 +51,7 @@ import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles
 import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles
 import org.ossreviewtoolkit.clients.fossid.listPendingFiles
 import org.ossreviewtoolkit.clients.fossid.listScansForProject
+import org.ossreviewtoolkit.clients.fossid.listSnippets
 import org.ossreviewtoolkit.clients.fossid.model.Project
 import org.ossreviewtoolkit.clients.fossid.model.Scan
 import org.ossreviewtoolkit.clients.fossid.model.rules.RuleScope
@@ -56,20 +60,25 @@ import org.ossreviewtoolkit.clients.fossid.model.status.DownloadStatus
 import org.ossreviewtoolkit.clients.fossid.model.status.ScanStatus
 import org.ossreviewtoolkit.clients.fossid.runScan
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.model.Hash
 import org.ossreviewtoolkit.model.Issue
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.Provenance
+import org.ossreviewtoolkit.model.RemoteArtifact
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerDetails
 import org.ossreviewtoolkit.model.Severity
+import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.UnknownProvenance
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.model.config.Options
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
 import org.ossreviewtoolkit.model.createAndLogIssue
+import org.ossreviewtoolkit.model.utils.DetectedSnippet
+import org.ossreviewtoolkit.model.utils.SnippetFinding
 import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory
 import org.ossreviewtoolkit.scanner.PackageScannerWrapper
 import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper
@@ -78,6 +87,9 @@ import org.ossreviewtoolkit.scanner.ScannerCriteria
 import org.ossreviewtoolkit.utils.common.enumSetOf
 import org.ossreviewtoolkit.utils.common.replaceCredentialsInUri
 import org.ossreviewtoolkit.utils.ort.showStackTrace
+import org.ossreviewtoolkit.utils.spdx.SpdxConstants
+import org.ossreviewtoolkit.utils.spdx.SpdxExpression
+import org.ossreviewtoolkit.utils.spdx.toSpdx
 
 /**
  * A wrapper for [FossID](https://fossid.com/).
@@ -746,7 +758,23 @@ class FossId internal constructor(
             "${pendingFiles.size} pending files have been returned for scan '$scanCode'."
         }
 
-        return RawResults(identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles, pendingFiles)
+        val snippets = runBlocking(Dispatchers.IO) {
+            pendingFiles.map {
+                logger.info { "Listing snippet for $it..." }
+                async {
+                    val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, it)
+                        .checkResponse("list snippets")
+                    val snippets = requireNotNull(snippetResponse.data) {
+                        "Snippet could not be listed. Response was ${snippetResponse.message}."
+                    }
+                    logger.info { "${snippets.size} snippets." }
+
+                    it to snippets.toSet()
+                }
+            }.awaitAll().toMap()
+        }
+
+        return RawResults(identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles, pendingFiles, snippets)
     }
 
     /**
@@ -760,10 +788,45 @@ class FossId internal constructor(
         scanId: String
     ): ScanResult {
         // TODO: Maybe get issues from FossID (see has_failed_scan_files, get_failed_files and maybe get_scan_log).
+
+        // TODO: Deprecation: Remove the pending files in issues. This is a breaking change.
         val issues = rawResults.listPendingFiles.mapTo(mutableListOf()) {
             Issue(source = name, message = "Pending identification for '$it'.", severity = Severity.HINT)
         }
 
+        val snippets = rawResults.listSnippets.mapValues { entry ->
+            entry.value.map {
+                val license = it.artifactLicense?.let { license ->
+                    val licenseExpression = runCatching { SpdxExpression.parse(license) }.getOrNull()
+
+                    val validatedLicense = when {
+                        licenseExpression == null -> SpdxConstants.NOASSERTION.toSpdx()
+                        licenseExpression.isValid() -> licenseExpression
+                        else -> "${SpdxConstants.LICENSE_REF_PREFIX}scanoss-$license".toSpdx()
+                    }
+
+                    validatedLicense
+                } ?: SpdxConstants.NOASSERTION.toSpdx()
+                // TODO: FossId doesn't return the line numbers of the match, only the character range. One must use
+                //  another call "getMatchedLine" to retrieve the matched line numbers. Unfortunately, this is a call
+                //  per snippet which is too expensive. When it is available for a batch of snippets, it can be used
+                //  here.
+                val sourceLocation = TextLocation(it.file, TextLocation.UNKNOWN_LINE)
+                val snippetLocation = TextLocation(it.url.orEmpty(), TextLocation.UNKNOWN_LINE)
+                SnippetFinding(
+                    it.score.toFloat(),
+                    sourceLocation,
+                    DetectedSnippet(
+                        snippetLocation,
+                        null,
+                        RemoteArtifact(it.url.orEmpty(), Hash.NONE),
+                        setOf("pkg:fossid/${it.author}/${it.artifact}@${it.version}"),
+                        license
+                    )
+                )
+            }.toSet()
+        }
+
         val ignoredFiles = rawResults.listIgnoredFiles.associateBy { it.path }
 
         val (licenseFindings, copyrightFindings) = rawResults.markedAsIdentifiedFiles.ifEmpty {
@@ -776,6 +839,7 @@ class FossId internal constructor(
             packageVerificationCode = "",
             licenseFindings = licenseFindings.toSortedSet(),
             copyrightFindings = copyrightFindings.toSortedSet(),
+            snippetFindings = snippets,
             issues = issues
         )
 
diff --git a/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt b/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt
@@ -22,6 +22,7 @@ package org.ossreviewtoolkit.scanner.scanners.fossid
 import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile
+import org.ossreviewtoolkit.clients.fossid.model.result.Snippet
 import org.ossreviewtoolkit.clients.fossid.model.summary.Summarizable
 import org.ossreviewtoolkit.model.CopyrightFinding
 import org.ossreviewtoolkit.model.Issue
@@ -37,7 +38,8 @@ internal data class RawResults(
     val identifiedFiles: List<IdentifiedFile>,
     val markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
     val listIgnoredFiles: List<IgnoredFile>,
-    val listPendingFiles: List<String>
+    val listPendingFiles: List<String>,
+    val listSnippets: Map<String, Set<Snippet>>
 )
 
 /**
diff --git a/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt b/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt
@@ -62,13 +62,16 @@ import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles
 import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles
 import org.ossreviewtoolkit.clients.fossid.listPendingFiles
 import org.ossreviewtoolkit.clients.fossid.listScansForProject
+import org.ossreviewtoolkit.clients.fossid.listSnippets
 import org.ossreviewtoolkit.clients.fossid.model.Scan
 import org.ossreviewtoolkit.clients.fossid.model.identification.common.LicenseMatchType
 import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.License
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.LicenseFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile
+import org.ossreviewtoolkit.clients.fossid.model.result.MatchType
+import org.ossreviewtoolkit.clients.fossid.model.result.Snippet
 import org.ossreviewtoolkit.clients.fossid.model.rules.IgnoreRule
 import org.ossreviewtoolkit.clients.fossid.model.rules.RuleScope
 import org.ossreviewtoolkit.clients.fossid.model.rules.RuleType
@@ -79,22 +82,27 @@ import org.ossreviewtoolkit.clients.fossid.runScan
 import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.downloader.vcs.Git
 import org.ossreviewtoolkit.model.CopyrightFinding
+import org.ossreviewtoolkit.model.Hash
 import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Issue
 import org.ossreviewtoolkit.model.LicenseFinding
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageType
+import org.ossreviewtoolkit.model.RemoteArtifact
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.Severity
 import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
+import org.ossreviewtoolkit.model.utils.DetectedSnippet
+import org.ossreviewtoolkit.model.utils.SnippetFinding
 import org.ossreviewtoolkit.scanner.ScanContext
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SCAN_CODE_KEY
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SCAN_ID_KEY
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SERVER_URL_KEY
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.convertGitUrlToProjectName
+import org.ossreviewtoolkit.utils.spdx.SpdxExpression
 
 @Suppress("LargeClass")
 class FossIdTest : WordSpec({
@@ -314,6 +322,7 @@ class FossIdTest : WordSpec({
             summary.licenseFindings shouldContainExactlyInAnyOrder expectedLicenseFindings
         }
 
+        // TODO: Deprecation: Remove the pending files in issues. This is a breaking change.
         "report pending files as issues" {
             val projectCode = projectCode(PROJECT)
             val scanCode = scanCode(PROJECT, null)
@@ -328,7 +337,7 @@ class FossIdTest : WordSpec({
                 .expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
                 .expectCreateScan(projectCode, scanCode, vcsInfo, "")
                 .expectDownload(scanCode)
-                .mockFiles(scanCode, pendingRange = 4..5)
+                .mockFiles(scanCode, pendingRange = 4..5, snippetRange = 1..5)
 
             val fossId = createFossId(config)
 
@@ -341,6 +350,35 @@ class FossIdTest : WordSpec({
             summary.issues.map { it.copy(timestamp = Instant.EPOCH) } shouldBe expectedIssues
         }
 
+        "report pending files as snippets" {
+            val projectCode = projectCode(PROJECT)
+            val scanCode = scanCode(PROJECT, null)
+            val config = createConfig(deltaScans = false)
+            val vcsInfo = createVcsInfo()
+            val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
+            val pkgId = createIdentifier(index = 42)
+
+            FossIdRestService.create(config.serverUrl)
+                .expectProjectRequest(projectCode)
+                .expectListScans(projectCode, listOf(scan))
+                .expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
+                .expectCreateScan(projectCode, scanCode, vcsInfo, "")
+                .expectDownload(scanCode)
+                .mockFiles(scanCode, pendingRange = 1..5, snippetRange = 1..5)
+
+            val fossId = createFossId(config)
+
+            val summary = fossId.scan(createPackage(pkgId, vcsInfo)).summary
+
+            val expectedPendingFile = (1..5).map(::createPendingFile).toSet()
+            val expectedSnippetFindings = (1..5).map(::createSnippetFinding).toSet()
+
+            summary.snippetFindings.keys shouldBe expectedPendingFile
+            summary.snippetFindings.values.forEach {
+                it shouldBe expectedSnippetFindings
+            }
+        }
+
         "create a new project if none exists yet" {
             val projectCode = projectCode(PROJECT)
             val scanCode = scanCode(PROJECT, null)
@@ -1238,6 +1276,51 @@ private fun createIgnoredFile(index: Int): IgnoredFile =
  */
 private fun createPendingFile(index: Int): String = "/pending/file/$index"
 
+/**
+ * Generate a FossID snippet based on the given [index].
+ */
+private fun createSnippet(index: Int): Snippet = Snippet(
+    index,
+    "created$index",
+    index,
+    index,
+    index,
+    MatchType.PARTIAL,
+    "reason$index",
+    "author$index",
+    "artifact$index",
+    "version$index",
+    "MIT",
+    "releaseDate$index",
+    "mirror$index",
+    "file$index",
+    "fileLicense$index",
+    "url$index",
+    "hits$index",
+    index,
+    "updated$index",
+    "cpe$index",
+    "$index",
+    "matchField$index",
+    "classification$index",
+    "highlighting$index"
+)
+
+/**
+ * Generate a ORT snippet finding based on the given [index].
+ */
+private fun createSnippetFinding(index: Int): SnippetFinding = SnippetFinding(
+    index.toFloat(),
+    TextLocation("file$index", TextLocation.UNKNOWN_LINE),
+    DetectedSnippet(
+        TextLocation("url$index", TextLocation.UNKNOWN_LINE),
+        null,
+        RemoteArtifact("url$index", Hash.NONE),
+        setOf("pkg:fossid/author$index/artifact$index@version$index"),
+        SpdxExpression.Companion.parse("MIT")
+    )
+)
+
 /**
  * Prepare this service mock to answer a request for a project with the given [projectCode]. Return a response with
  * the given [status] and [error].
@@ -1348,12 +1431,14 @@ private fun FossIdServiceWithVersion.mockFiles(
     identifiedRange: IntRange = IntRange.EMPTY,
     markedRange: IntRange = IntRange.EMPTY,
     ignoredRange: IntRange = IntRange.EMPTY,
-    pendingRange: IntRange = IntRange.EMPTY
+    pendingRange: IntRange = IntRange.EMPTY,
+    snippetRange: IntRange = IntRange.EMPTY
 ): FossIdServiceWithVersion {
     val identifiedFiles = identifiedRange.map(::createIdentifiedFile)
     val markedFiles = markedRange.map(::createMarkedIdentifiedFile)
     val ignoredFiles = ignoredRange.map(::createIgnoredFile)
     val pendingFiles = pendingRange.map(::createPendingFile)
+    val snippets = snippetRange.map(::createSnippet)
 
     coEvery { listIdentifiedFiles(USER, API_KEY, scanCode) } returns
             PolymorphicResponseBody(
@@ -1367,6 +1452,8 @@ private fun FossIdServiceWithVersion.mockFiles(
             PolymorphicResponseBody(status = 1, data = PolymorphicList(ignoredFiles))
     coEvery { listPendingFiles(USER, API_KEY, scanCode) } returns
             PolymorphicResponseBody(status = 1, data = PolymorphicList(pendingFiles))
+    coEvery { listSnippets(USER, API_KEY, scanCode, any()) } returns
+            PolymorphicResponseBody(status = 1, data = PolymorphicList(snippets))
 
     return this
 }
