feat(fossid-webapp): Map FossID snippets to the ScanSummary

nnobelis · nnobelis · commit 3c616709a301 · 2023-04-04T13:43:31.000+02:00
When FossId identifies a file matching snippets, it is a pending file.
An operator needs to log to FossID UI and use the license of a snippet
or manually enter difference license information. Then the file is
marked as "identified".

Currently, the FossID scanner in ORT returns the list of all pending
files in `ScanSummary` issues, with a severity of `HINT`. This commit
maps the snippets of pending files using the newly-created snippet data
model.
The pending files are still listed as issues: this will be removed in a
future commit as it is a breaking change.

Signed-off-by: Nicolas Nobelis &lt;nicolas.nobelis@bosch.io&gt;
diff --git a/scanner/src/main/kotlin/scanners/fossid/FossId.kt b/scanner/src/main/kotlin/scanners/fossid/FossId.kt
@@ -27,6 +27,9 @@ import kotlin.time.Duration.Companion.minutes
 import kotlin.time.Duration.Companion.seconds
 import kotlin.time.measureTimedValue
 
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeoutOrNull
@@ -48,6 +51,7 @@ import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles
 import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles
 import org.ossreviewtoolkit.clients.fossid.listPendingFiles
 import org.ossreviewtoolkit.clients.fossid.listScansForProject
+import org.ossreviewtoolkit.clients.fossid.listSnippets
 import org.ossreviewtoolkit.clients.fossid.model.Project
 import org.ossreviewtoolkit.clients.fossid.model.Scan
 import org.ossreviewtoolkit.clients.fossid.model.rules.RuleScope
@@ -64,12 +68,14 @@ import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerDetails
 import org.ossreviewtoolkit.model.Severity
+import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.UnknownProvenance
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.model.config.Options
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
 import org.ossreviewtoolkit.model.createAndLogIssue
+import org.ossreviewtoolkit.model.utils.SnippetFinding
 import org.ossreviewtoolkit.scanner.AbstractScannerWrapperFactory
 import org.ossreviewtoolkit.scanner.PackageScannerWrapper
 import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper
@@ -78,6 +84,9 @@ import org.ossreviewtoolkit.scanner.ScannerCriteria
 import org.ossreviewtoolkit.utils.common.enumSetOf
 import org.ossreviewtoolkit.utils.common.replaceCredentialsInUri
 import org.ossreviewtoolkit.utils.ort.showStackTrace
+import org.ossreviewtoolkit.utils.spdx.SpdxConstants
+import org.ossreviewtoolkit.utils.spdx.SpdxExpression
+import org.ossreviewtoolkit.utils.spdx.toSpdx
 
 /**
  * A wrapper for [FossID](https://fossid.com/).
@@ -746,7 +755,23 @@ class FossId internal constructor(
             "${pendingFiles.size} pending files have been returned for scan '$scanCode'."
         }
 
-        return RawResults(identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles, pendingFiles)
+        val snippets = runBlocking(Dispatchers.IO) {
+            pendingFiles.map {
+                logger.info { "Listing snippet for $it..." }
+                async {
+                    val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, it)
+                        .checkResponse("list snippets")
+                    val snippets = requireNotNull(snippetResponse.data) {
+                        "Snippet could not be listed. Response was ${snippetResponse.message}."
+                    }
+                    logger.info { "${snippets.size} snippets." }
+
+                    it to snippets.toSet()
+                }
+            }.awaitAll().toMap()
+        }
+
+        return RawResults(identifiedFiles, markedAsIdentifiedFiles, listIgnoredFiles, pendingFiles, snippets)
     }
 
     /**
@@ -760,10 +785,43 @@ class FossId internal constructor(
         scanId: String
     ): ScanResult {
         // TODO: Maybe get issues from FossID (see has_failed_scan_files, get_failed_files and maybe get_scan_log).
+
+        // TODO: Deprecation: Remove the pending files in issues. This is a breaking change.
         val issues = rawResults.listPendingFiles.mapTo(mutableListOf()) {
             Issue(source = name, message = "Pending identification for '$it'.", severity = Severity.HINT)
         }
 
+        val snippets = rawResults.listSnippets.mapValues { entry ->
+            entry.value.map {
+                val license = it.artifactLicense?.let { license ->
+                    val licenseExpression = runCatching { SpdxExpression.parse(license) }.getOrNull()
+
+                    val validatedLicense = when {
+                        licenseExpression == null -> SpdxConstants.NOASSERTION.toSpdx()
+                        licenseExpression.isValid() -> licenseExpression
+                        else -> "${SpdxConstants.LICENSE_REF_PREFIX}scanoss-$license".toSpdx()
+                    }
+
+                    setOf(validatedLicense)
+                } ?: emptySet()
+                // TODO: FossId doesn't return the line numbers of the match, only the character range. One must use
+                //  another call "getMatchedLine" to retrieve the matched line numbers. Unfortunately, this is a call
+                //  per snippet which is too expensive. When it is available for a batch of snippets, it can be used
+                //  here.
+                val sourceLocation = TextLocation(it.file, -1)
+                val snippetLocation = TextLocation(it.url.orEmpty(), -1)
+                SnippetFinding(
+                    it.author,
+                    it.artifact,
+                    it.version,
+                    license,
+                    it.score.toFloat(),
+                    sourceLocation,
+                    snippetLocation
+                )
+            }.toSet()
+        }
+
         val ignoredFiles = rawResults.listIgnoredFiles.associateBy { it.path }
 
         val (licenseFindings, copyrightFindings) = rawResults.markedAsIdentifiedFiles.ifEmpty {
@@ -776,6 +834,7 @@ class FossId internal constructor(
             packageVerificationCode = "",
             licenseFindings = licenseFindings.toSortedSet(),
             copyrightFindings = copyrightFindings.toSortedSet(),
+            snippetFindings = snippets,
             issues = issues
         )
 
diff --git a/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt b/scanner/src/main/kotlin/scanners/fossid/FossIdScanResults.kt
@@ -22,6 +22,7 @@ package org.ossreviewtoolkit.scanner.scanners.fossid
 import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile
+import org.ossreviewtoolkit.clients.fossid.model.result.Snippet
 import org.ossreviewtoolkit.clients.fossid.model.summary.Summarizable
 import org.ossreviewtoolkit.model.CopyrightFinding
 import org.ossreviewtoolkit.model.Issue
@@ -37,7 +38,8 @@ internal data class RawResults(
     val identifiedFiles: List<IdentifiedFile>,
     val markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
     val listIgnoredFiles: List<IgnoredFile>,
-    val listPendingFiles: List<String>
+    val listPendingFiles: List<String>,
+    val listSnippets: Map<String, Set<Snippet>>
 )
 
 /**
diff --git a/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt b/scanner/src/test/kotlin/scanners/fossid/FossIdTest.kt
@@ -62,13 +62,16 @@ import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles
 import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles
 import org.ossreviewtoolkit.clients.fossid.listPendingFiles
 import org.ossreviewtoolkit.clients.fossid.listScansForProject
+import org.ossreviewtoolkit.clients.fossid.listSnippets
 import org.ossreviewtoolkit.clients.fossid.model.Scan
 import org.ossreviewtoolkit.clients.fossid.model.identification.common.LicenseMatchType
 import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.License
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.LicenseFile
 import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.MarkedAsIdentifiedFile
+import org.ossreviewtoolkit.clients.fossid.model.result.MatchType
+import org.ossreviewtoolkit.clients.fossid.model.result.Snippet
 import org.ossreviewtoolkit.clients.fossid.model.rules.IgnoreRule
 import org.ossreviewtoolkit.clients.fossid.model.rules.RuleScope
 import org.ossreviewtoolkit.clients.fossid.model.rules.RuleType
@@ -90,11 +93,13 @@ import org.ossreviewtoolkit.model.TextLocation
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
+import org.ossreviewtoolkit.model.utils.SnippetFinding
 import org.ossreviewtoolkit.scanner.ScanContext
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SCAN_CODE_KEY
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SCAN_ID_KEY
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.SERVER_URL_KEY
 import org.ossreviewtoolkit.scanner.scanners.fossid.FossId.Companion.convertGitUrlToProjectName
+import org.ossreviewtoolkit.utils.spdx.SpdxExpression
 
 @Suppress("LargeClass")
 class FossIdTest : WordSpec({
@@ -314,6 +319,7 @@ class FossIdTest : WordSpec({
             summary.licenseFindings shouldContainExactlyInAnyOrder expectedLicenseFindings
         }
 
+        // TODO: Deprecation: Remove the pending files in issues. This is a breaking change.
         "report pending files as issues" {
             val projectCode = projectCode(PROJECT)
             val scanCode = scanCode(PROJECT, null)
@@ -328,7 +334,7 @@ class FossIdTest : WordSpec({
                 .expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
                 .expectCreateScan(projectCode, scanCode, vcsInfo, "")
                 .expectDownload(scanCode)
-                .mockFiles(scanCode, pendingRange = 4..5)
+                .mockFiles(scanCode, pendingRange = 4..5, snippetRange = 1..5)
 
             val fossId = createFossId(config)
 
@@ -341,6 +347,35 @@ class FossIdTest : WordSpec({
             summary.issues.map { it.copy(timestamp = Instant.EPOCH) } shouldBe expectedIssues
         }
 
+        "report pending files as snippets" {
+            val projectCode = projectCode(PROJECT)
+            val scanCode = scanCode(PROJECT, null)
+            val config = createConfig(deltaScans = false)
+            val vcsInfo = createVcsInfo()
+            val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode)
+            val pkgId = createIdentifier(index = 42)
+
+            FossIdRestService.create(config.serverUrl)
+                .expectProjectRequest(projectCode)
+                .expectListScans(projectCode, listOf(scan))
+                .expectCheckScanStatus(scanCode, ScanStatus.FINISHED)
+                .expectCreateScan(projectCode, scanCode, vcsInfo, "")
+                .expectDownload(scanCode)
+                .mockFiles(scanCode, pendingRange = 1..5, snippetRange = 1..5)
+
+            val fossId = createFossId(config)
+
+            val summary = fossId.scan(createPackage(pkgId, vcsInfo)).summary
+
+            val expectedPendingFile = (1..5).map(::createPendingFile).toSet()
+            val expectedSnippetFindings = (1..5).map(::createSnippetFinding).toSet()
+
+            summary.snippetFindings.keys shouldBe expectedPendingFile
+            summary.snippetFindings.values.forEach {
+                it shouldBe expectedSnippetFindings
+            }
+        }
+
         "create a new project if none exists yet" {
             val projectCode = projectCode(PROJECT)
             val scanCode = scanCode(PROJECT, null)
@@ -1238,6 +1273,49 @@ private fun createIgnoredFile(index: Int): IgnoredFile =
  */
 private fun createPendingFile(index: Int): String = "/pending/file/$index"
 
+/**
+ * Generate a FossID snippet based on the given [index].
+ */
+private fun createSnippet(index: Int): Snippet = Snippet(
+    index,
+    "created$index",
+    index,
+    index,
+    index,
+    MatchType.PARTIAL,
+    "reason$index",
+    "author$index",
+    "artifact$index",
+    "version$index",
+    "MIT",
+    "releaseDate$index",
+    "mirror$index",
+    "file$index",
+    "fileLicense$index",
+    "url$index",
+    "hits$index",
+    index,
+    "updated$index",
+    "cpe$index",
+    "$index",
+    "matchField$index",
+    "classification$index",
+    "highlighting$index"
+)
+
+/**
+ * Generate a ORT snippet finding based on the given [index].
+ */
+private fun createSnippetFinding(index: Int): SnippetFinding = SnippetFinding(
+    "author$index",
+    "artifact$index",
+    "version$index",
+    setOf(SpdxExpression.Companion.parse("MIT")),
+    index.toFloat(),
+    TextLocation("file$index", -1),
+    TextLocation("url$index", -1)
+)
+
 /**
  * Prepare this service mock to answer a request for a project with the given [projectCode]. Return a response with
  * the given [status] and [error].
@@ -1348,12 +1426,14 @@ private fun FossIdServiceWithVersion.mockFiles(
     identifiedRange: IntRange = IntRange.EMPTY,
     markedRange: IntRange = IntRange.EMPTY,
     ignoredRange: IntRange = IntRange.EMPTY,
-    pendingRange: IntRange = IntRange.EMPTY
+    pendingRange: IntRange = IntRange.EMPTY,
+    snippetRange: IntRange = IntRange.EMPTY
 ): FossIdServiceWithVersion {
     val identifiedFiles = identifiedRange.map(::createIdentifiedFile)
     val markedFiles = markedRange.map(::createMarkedIdentifiedFile)
     val ignoredFiles = ignoredRange.map(::createIgnoredFile)
     val pendingFiles = pendingRange.map(::createPendingFile)
+    val snippets = snippetRange.map(::createSnippet)
 
     coEvery { listIdentifiedFiles(USER, API_KEY, scanCode) } returns
             PolymorphicResponseBody(
@@ -1367,6 +1447,8 @@ private fun FossIdServiceWithVersion.mockFiles(
             PolymorphicResponseBody(status = 1, data = PolymorphicList(ignoredFiles))
     coEvery { listPendingFiles(USER, API_KEY, scanCode) } returns
             PolymorphicResponseBody(status = 1, data = PolymorphicList(pendingFiles))
+    coEvery { listSnippets(USER, API_KEY, scanCode, any()) } returns
+            PolymorphicResponseBody(status = 1, data = PolymorphicList(snippets))
 
     return this
 }
