Skip to content

Commit 40fa386

Browse files
mnonnenmachermnonnenmacher
committed
feat(scanner)!: Migrate to new plugin API
Migrate the `ScannerWrapper` plugins to the new plugin API. One major difference is that the configuration properties for the scan result matcher are not automatically supported for all plugins anymore. Instead, plugins have to decide which of them they want to make configurable. The previous mechanism was difficult to translate to the new plugin API and also not all plugins need to support all of those options. The plugin API requires list options to be comma-separated, therefore the ScanCode command line options now have to be provided comma-separated instead of separated by whitespace. Signed-off-by: Martin Nonnenmacher <martin.nonnenmacher@bosch.com>
1 parent f71062c commit 40fa386

File tree

48 files changed

+626
-777
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+626
-777
lines changed

‎model/src/main/resources/reference.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,10 +237,10 @@ ort:
237237
options:
238238
# Command line options that affect the ScanCode output. If changed, stored scan results that were created with
239239
# different options are not reused.
240-
commandLine: '--copyright --license --info --strip-root --timeout 300'
240+
commandLine: '--copyright,--license,--info,--strip-root,--timeout,300'
241241

242242
# Command line options that do not affect the ScanCode output.
243-
commandLineNonConfig: '--processes 4'
243+
commandLineNonConfig: '--processes,4'
244244

245245
# Use per-file license findings instead of per-line ones.
246246
preferFileLicense: false

‎model/src/test/kotlin/config/OrtConfigurationTest.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,8 @@ class OrtConfigurationTest : WordSpec({
241241
config shouldNotBeNull {
242242
get("ScanCode") shouldNotBeNull {
243243
options shouldContainExactly mapOf(
244-
"commandLine" to "--copyright --license --info --strip-root --timeout 300",
245-
"commandLineNonConfig" to "--processes 4",
244+
"commandLine" to "--copyright,--license,--info,--strip-root,--timeout,300",
245+
"commandLineNonConfig" to "--processes,4",
246246
"preferFileLicense" to "false",
247247
"minVersion" to "3.2.1-rc2",
248248
"maxVersion" to "32.0.0"

‎plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ import org.ossreviewtoolkit.model.config.OrtConfiguration
5151
import org.ossreviewtoolkit.model.utils.DefaultResolutionProvider
5252
import org.ossreviewtoolkit.model.utils.mergeLabels
5353
import org.ossreviewtoolkit.plugins.api.OrtPlugin
54+
import org.ossreviewtoolkit.plugins.api.PluginConfig
5455
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
5556
import org.ossreviewtoolkit.plugins.commands.api.OrtCommand
5657
import org.ossreviewtoolkit.plugins.commands.api.OrtCommandFactory
@@ -167,34 +168,34 @@ class ScannerCommand(descriptor: PluginDescriptor = ScannerCommandFactory.descri
167168

168169
@Suppress("ForbiddenMethodCall")
169170
private fun runScanners(
170-
scannerWrapperFactories: List<ScannerWrapperFactory<*>>,
171-
projectScannerWrapperFactories: List<ScannerWrapperFactory<*>>,
171+
scannerWrapperFactories: List<ScannerWrapperFactory>,
172+
projectScannerWrapperFactories: List<ScannerWrapperFactory>,
172173
ortConfig: OrtConfiguration
173174
): OrtResult {
174175
val packageScannerWrappers = scannerWrapperFactories
175176
.takeIf { PackageType.PACKAGE in packageTypes }.orEmpty()
176177
.map {
177-
val config = ortConfig.scanner.config?.get(it.type)
178-
it.create(config?.options.orEmpty(), config?.secrets.orEmpty())
178+
val config = ortConfig.scanner.config?.get(it.descriptor.id)
179+
it.create(PluginConfig(config?.options.orEmpty(), config?.secrets.orEmpty()))
179180
}
180181

181182
val projectScannerWrappers = projectScannerWrapperFactories
182183
.takeIf { PackageType.PROJECT in packageTypes }.orEmpty()
183184
.map {
184-
val config = ortConfig.scanner.config?.get(it.type)
185-
it.create(config?.options.orEmpty(), config?.secrets.orEmpty())
185+
val config = ortConfig.scanner.config?.get(it.descriptor.id)
186+
it.create(PluginConfig(config?.options.orEmpty(), config?.secrets.orEmpty()))
186187
}
187188

188189
if (projectScannerWrappers.isNotEmpty()) {
189190
echo("Scanning projects with:")
190-
echo(projectScannerWrappers.joinToString { "\t${it.name} (version ${it.version})" })
191+
echo(projectScannerWrappers.joinToString { "\t${it.descriptor.displayName} (version ${it.version})" })
191192
} else {
192193
echo("Projects will not be scanned.")
193194
}
194195

195196
if (packageScannerWrappers.isNotEmpty()) {
196197
echo("Scanning packages with:")
197-
echo(packageScannerWrappers.joinToString { "\t${it.name} (version ${it.version})" })
198+
echo(packageScannerWrappers.joinToString { "\t${it.descriptor.displayName} (version ${it.version})" })
198199
} else {
199200
echo("Packages will not be scanned.")
200201
}

‎plugins/scanners/askalono/build.gradle.kts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
plugins {
2121
// Apply precompiled plugins.
22-
id("ort-library-conventions")
22+
id("ort-plugin-conventions")
2323

2424
// Apply third-party plugins.
2525
alias(libs.plugins.kotlinSerialization)
@@ -32,5 +32,7 @@ dependencies {
3232
implementation(libs.kotlinx.serialization.core)
3333
implementation(libs.kotlinx.serialization.json)
3434

35+
ksp(projects.scanner)
36+
3537
funTestApi(testFixtures(projects.scanner))
3638
}

‎plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,9 @@ package org.ossreviewtoolkit.plugins.scanners.askalono
2222
import org.ossreviewtoolkit.model.LicenseFinding
2323
import org.ossreviewtoolkit.model.TextLocation
2424
import org.ossreviewtoolkit.scanner.AbstractPathScannerWrapperFunTest
25-
import org.ossreviewtoolkit.scanner.ScannerWrapperConfig
2625

2726
class AskalonoFunTest : AbstractPathScannerWrapperFunTest() {
28-
override val scanner = Askalono("Askalono", ScannerWrapperConfig.EMPTY)
27+
override val scanner = AskalonoFactory.create()
2928

3029
override val expectedFileLicenses = listOf(
3130
LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 1.0f)

‎plugins/scanners/askalono/src/main/kotlin/Askalono.kt‎

Lines changed: 64 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,16 @@ import org.ossreviewtoolkit.model.LicenseFinding
3232
import org.ossreviewtoolkit.model.ScanSummary
3333
import org.ossreviewtoolkit.model.Severity
3434
import org.ossreviewtoolkit.model.TextLocation
35+
import org.ossreviewtoolkit.plugins.api.OrtPlugin
36+
import org.ossreviewtoolkit.plugins.api.OrtPluginOption
37+
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
3538
import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper
3639
import org.ossreviewtoolkit.scanner.ScanContext
3740
import org.ossreviewtoolkit.scanner.ScanException
3841
import org.ossreviewtoolkit.scanner.ScannerMatcher
39-
import org.ossreviewtoolkit.scanner.ScannerWrapperConfig
42+
import org.ossreviewtoolkit.scanner.ScannerMatcherConfig
4043
import org.ossreviewtoolkit.scanner.ScannerWrapperFactory
4144
import org.ossreviewtoolkit.utils.common.CommandLineTool
42-
import org.ossreviewtoolkit.utils.common.Options
4345
import org.ossreviewtoolkit.utils.common.Os
4446

4547
private const val CONFIDENCE_NOTICE = "Confidence threshold not high enough for any known license"
@@ -56,23 +58,69 @@ object AskalonoCommand : CommandLineTool {
5658
output.removePrefix("askalono ")
5759
}
5860

59-
class Askalono internal constructor(name: String, private val wrapperConfig: ScannerWrapperConfig) :
60-
LocalPathScannerWrapper(name) {
61-
class Factory : ScannerWrapperFactory<Unit>("Askalono") {
62-
override fun create(config: Unit, wrapperConfig: ScannerWrapperConfig) = Askalono(type, wrapperConfig)
63-
64-
override fun parseConfig(options: Options, secrets: Options) = Unit
65-
}
66-
61+
data class AskalonoConfig(
62+
/**
63+
* A regular expression to match the scanner name when looking up scan results in the storage.
64+
*/
65+
val regScannerName: String?,
66+
67+
/**
68+
* The minimum version of stored scan results to use.
69+
*/
70+
val minVersion: String?,
71+
72+
/**
73+
* The maximum version of stored scan results to use.
74+
*/
75+
val maxVersion: String?,
76+
77+
/**
78+
* The configuration to use for the scanner. Only scan results with the same configuration are used when looking up
79+
* scan results in the storage.
80+
*/
81+
val configuration: String?,
82+
83+
/**
84+
* Whether to read scan results from the storage.
85+
*/
86+
@OrtPluginOption(defaultValue = "true")
87+
val readFromStorage: Boolean,
88+
89+
/**
90+
* Whether to write scan results to the storage.
91+
*/
92+
@OrtPluginOption(defaultValue = "true")
93+
val writeToStorage: Boolean
94+
)
95+
96+
@OrtPlugin(
97+
displayName = "askalono",
98+
description = "askalono is a library and command-line tool to help detect license texts. It's designed to be " +
99+
"fast, accurate, and to support a wide variety of license texts.",
100+
factory = ScannerWrapperFactory::class
101+
)
102+
class Askalono(
103+
override val descriptor: PluginDescriptor = AskalonoFactory.descriptor,
104+
config: AskalonoConfig
105+
) : LocalPathScannerWrapper() {
67106
override val configuration = ""
68107

69-
override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) }
108+
override val matcher by lazy {
109+
ScannerMatcher.create(
110+
details,
111+
ScannerMatcherConfig(
112+
config.regScannerName,
113+
config.minVersion,
114+
config.maxVersion,
115+
config.configuration
116+
)
117+
)
118+
}
70119

71120
override val version by lazy { AskalonoCommand.getVersion() }
72121

73-
override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) }
74-
75-
override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) }
122+
override val readFromStorage = config.readFromStorage
123+
override val writeToStorage = config.writeToStorage
76124

77125
override fun runScanner(path: File, context: ScanContext): String {
78126
val process = AskalonoCommand.run(
@@ -95,7 +143,7 @@ class Askalono internal constructor(name: String, private val wrapperConfig: Sca
95143

96144
val issues = mutableListOf(
97145
Issue(
98-
source = name,
146+
source = descriptor.id,
99147
message = "This scanner is not capable of detecting copyright statements.",
100148
severity = Severity.HINT
101149
)
@@ -112,7 +160,7 @@ class Askalono internal constructor(name: String, private val wrapperConfig: Sca
112160

113161
if (it.error != null) {
114162
issues += Issue(
115-
source = name,
163+
source = descriptor.id,
116164
message = it.error,
117165
severity = if (it.error == CONFIDENCE_NOTICE) Severity.HINT else Severity.ERROR
118166
)

‎plugins/scanners/askalono/src/main/resources/META-INF/services/org.ossreviewtoolkit.scanner.ScannerWrapperFactory‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎plugins/scanners/boyterlc/build.gradle.kts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
plugins {
2121
// Apply precompiled plugins.
22-
id("ort-library-conventions")
22+
id("ort-plugin-conventions")
2323

2424
// Apply third-party plugins.
2525
alias(libs.plugins.kotlinSerialization)
@@ -32,5 +32,7 @@ dependencies {
3232
implementation(libs.kotlinx.serialization.core)
3333
implementation(libs.kotlinx.serialization.json)
3434

35+
ksp(projects.scanner)
36+
3537
funTestApi(testFixtures(projects.scanner))
3638
}

‎plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,9 @@ package org.ossreviewtoolkit.plugins.scanners.boyterlc
2222
import org.ossreviewtoolkit.model.LicenseFinding
2323
import org.ossreviewtoolkit.model.TextLocation
2424
import org.ossreviewtoolkit.scanner.AbstractPathScannerWrapperFunTest
25-
import org.ossreviewtoolkit.scanner.ScannerWrapperConfig
2625

2726
class BoyterLcFunTest : AbstractPathScannerWrapperFunTest() {
28-
override val scanner = BoyterLc("BoyterLc", ScannerWrapperConfig.EMPTY)
27+
override val scanner = BoyterLcFactory.create()
2928

3029
override val expectedFileLicenses = listOf(
3130
LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 0.98388565f),

‎plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt‎

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,16 @@ import org.ossreviewtoolkit.model.LicenseFinding
3131
import org.ossreviewtoolkit.model.ScanSummary
3232
import org.ossreviewtoolkit.model.Severity
3333
import org.ossreviewtoolkit.model.TextLocation
34+
import org.ossreviewtoolkit.plugins.api.OrtPlugin
35+
import org.ossreviewtoolkit.plugins.api.OrtPluginOption
36+
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
3437
import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper
3538
import org.ossreviewtoolkit.scanner.ScanContext
3639
import org.ossreviewtoolkit.scanner.ScanException
3740
import org.ossreviewtoolkit.scanner.ScannerMatcher
38-
import org.ossreviewtoolkit.scanner.ScannerWrapperConfig
41+
import org.ossreviewtoolkit.scanner.ScannerMatcherConfig
3942
import org.ossreviewtoolkit.scanner.ScannerWrapperFactory
4043
import org.ossreviewtoolkit.utils.common.CommandLineTool
41-
import org.ossreviewtoolkit.utils.common.Options
4244
import org.ossreviewtoolkit.utils.common.Os
4345
import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
4446
import org.ossreviewtoolkit.utils.ort.createOrtTempDir
@@ -55,30 +57,76 @@ object BoyterLcCommand : CommandLineTool {
5557
output.removePrefix("licensechecker version ")
5658
}
5759

58-
class BoyterLc internal constructor(name: String, private val wrapperConfig: ScannerWrapperConfig) :
59-
LocalPathScannerWrapper(name) {
60+
data class BoyterLcConfig(
61+
/**
62+
* A regular expression to match the scanner name when looking up scan results in the storage.
63+
*/
64+
val regScannerName: String?,
65+
66+
/**
67+
* The minimum version of stored scan results to use.
68+
*/
69+
val minVersion: String?,
70+
71+
/**
72+
* The maximum version of stored scan results to use.
73+
*/
74+
val maxVersion: String?,
75+
76+
/**
77+
* The configuration to use for the scanner. Only scan results with the same configuration are used when looking up
78+
* scan results in the storage.
79+
*/
80+
val configuration: String?,
81+
82+
/**
83+
* Whether to read scan results from the storage.
84+
*/
85+
@OrtPluginOption(defaultValue = "true")
86+
val readFromStorage: Boolean,
87+
88+
/**
89+
* Whether to write scan results to the storage.
90+
*/
91+
@OrtPluginOption(defaultValue = "true")
92+
val writeToStorage: Boolean
93+
)
94+
95+
@OrtPlugin(
96+
displayName = "BoyterLc",
97+
description = "A command line application which scans directories and identifies what software license things " +
98+
"are under.",
99+
factory = ScannerWrapperFactory::class
100+
)
101+
class BoyterLc(
102+
override val descriptor: PluginDescriptor = BoyterLcFactory.descriptor,
103+
config: BoyterLcConfig
104+
) : LocalPathScannerWrapper() {
60105
companion object {
61106
val CONFIGURATION_OPTIONS = listOf(
62107
"--confidence", "0.95", // Cut-off value to only get most relevant matches.
63108
"--format", "json"
64109
)
65110
}
66111

67-
class Factory : ScannerWrapperFactory<Unit>("BoyterLc") {
68-
override fun create(config: Unit, wrapperConfig: ScannerWrapperConfig) = BoyterLc(type, wrapperConfig)
69-
70-
override fun parseConfig(options: Options, secrets: Options) = Unit
71-
}
72-
73112
override val configuration = CONFIGURATION_OPTIONS.joinToString(" ")
74113

75-
override val matcher by lazy { ScannerMatcher.create(details, wrapperConfig.matcherConfig) }
114+
override val matcher by lazy {
115+
ScannerMatcher.create(
116+
details,
117+
ScannerMatcherConfig(
118+
config.regScannerName,
119+
config.minVersion,
120+
config.maxVersion,
121+
config.configuration
122+
)
123+
)
124+
}
76125

77126
override val version by lazy { BoyterLcCommand.getVersion() }
78127

79-
override val readFromStorage by lazy { wrapperConfig.readFromStorageWithDefault(matcher) }
80-
81-
override val writeToStorage by lazy { wrapperConfig.writeToStorageWithDefault(matcher) }
128+
override val readFromStorage = config.readFromStorage
129+
override val writeToStorage = config.writeToStorage
82130

83131
override fun runScanner(path: File, context: ScanContext): String {
84132
val resultFile = createOrtTempDir().resolve("result.json")
@@ -116,7 +164,7 @@ class BoyterLc internal constructor(name: String, private val wrapperConfig: Sca
116164
licenseFindings = licenseFindings,
117165
issues = listOf(
118166
Issue(
119-
source = name,
167+
source = descriptor.id,
120168
message = "This scanner is not capable of detecting copyright statements.",
121169
severity = Severity.HINT
122170
)

0 commit comments

Comments
 (0)