Skip to content

Commit 2fa54e8

Browse files
bishaboshabishabosha
authored
Export diagnostics (including unused warnings) to SemanticDB (#17835)
fix #17535 ## What this PR does This PR splits `ExtractSemanticDB` to `PostTyper` and `PostInlining` (as `ChechUnused` does). - The `PostTyper` phase - It extracts SemanticDB information such as symbol definitions, symbol occurrences, type information, and synthetics. - ~**This phase does not write the information to a `.semanticdb` file**; instead, it attaches the SemanticDB information to the top-level tree.~ - And write `.semanticdb` file. - The `PostInlining` phase - It extracts diagnostics from `ctx.reporter` and attaches them to the SemanticDB information extracted in the `PostTyper` phase. - Afterwards, it updates the SemanticDB to a `.semanticdb` file (if there's warning in the file). - **We need to run this phase after the `CheckUnused.PostInlining` phase so that we can extract the warnings generated by "-Wunused".** Also, - Reporter now stores `warnings` in addition to `errors` - Tweaked SemanticDBTest to show the generated diagnostics to `metac.expect` ### Concerns - Since we attach the SemanticDB information to the top-level tree, it lives in-memory during the compilation which may leads to more memory usage for compilation - Now, Reporter stores all warnings in addition to errors (to convey the warnings across phases), which also may cause some more memory consumption.
2 parents bf994f9 + 275e6fa commit 2fa54e8

File tree

11 files changed

+325
-69
lines changed

11 files changed

+325
-69
lines changed

‎compiler/src/dotty/tools/dotc/Compiler.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class Compiler {
3939
List(new CheckShadowing) :: // Check shadowing elements
4040
List(new YCheckPositions) :: // YCheck positions
4141
List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
42-
List(new semanticdb.ExtractSemanticDB) :: // Extract info into .semanticdb files
42+
List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
4343
List(new PostTyper) :: // Additional checks and cleanups after type checking
4444
List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only)
4545
List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks
@@ -72,6 +72,7 @@ class Compiler {
7272
new ExpandSAMs, // Expand single abstract method closures to anonymous classes
7373
new ElimRepeated, // Rewrite vararg parameters and arguments
7474
new RefChecks) :: // Various checks mostly related to abstract members and overriding
75+
List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file
7576
List(new init.Checker) :: // Check initialization of objects
7677
List(new ProtectedAccessors, // Add accessors for protected members
7778
new ExtensionMethods, // Expand methods of value classes with extension methods

‎compiler/src/dotty/tools/dotc/reporting/Reporter.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,14 @@ abstract class Reporter extends interfaces.ReporterResult {
109109

110110
private var errors: List[Error] = Nil
111111

112+
private var warnings: List[Warning] = Nil
113+
112114
/** All errors reported by this reporter (ignoring outer reporters) */
113115
def allErrors: List[Error] = errors
114116

117+
/** All warnings reported by this reporter (ignoring outer reporters) */
118+
def allWarnings: List[Warning] = warnings
119+
115120
/** Were sticky errors reported? Overridden in StoreReporter. */
116121
def hasStickyErrors: Boolean = false
117122

@@ -153,7 +158,9 @@ abstract class Reporter extends interfaces.ReporterResult {
153158
markReported(dia)
154159
withMode(Mode.Printing)(doReport(dia))
155160
dia match {
156-
case _: Warning => _warningCount += 1
161+
case w: Warning =>
162+
warnings = w :: warnings
163+
_warningCount += 1
157164
case e: Error =>
158165
errors = e :: errors
159166
_errorCount += 1

‎compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package dotty.tools.dotc.semanticdb
2+
3+
import dotty.tools.dotc.reporting.Diagnostic
4+
import dotty.tools.dotc.{semanticdb => s}
5+
import dotty.tools.dotc.interfaces.Diagnostic.{ERROR, INFO, WARNING}
6+
import dotty.tools.dotc.core.Contexts.Context
7+
import scala.annotation.internal.sharable
8+
9+
object DiagnosticOps:
10+
@sharable private val asciiColorCodes = "\u001B\\[[;\\d]*m".r
11+
extension (d: Diagnostic)
12+
def toSemanticDiagnostic: s.Diagnostic =
13+
val severity = d.level match
14+
case ERROR => s.Diagnostic.Severity.ERROR
15+
case WARNING => s.Diagnostic.Severity.WARNING
16+
case INFO => s.Diagnostic.Severity.INFORMATION
17+
case _ => s.Diagnostic.Severity.INFORMATION
18+
val msg = asciiColorCodes.replaceAllIn(d.msg.message, m => "")
19+
s.Diagnostic(
20+
range = Scala3.range(d.pos.span, d.pos.source),
21+
severity = severity,
22+
message = msg
23+
)

‎compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala‎

Lines changed: 156 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,34 @@ import transform.SymUtils._
2121

2222
import scala.collection.mutable
2323
import scala.annotation.{ threadUnsafe => tu, tailrec }
24+
import scala.jdk.CollectionConverters._
2425
import scala.PartialFunction.condOpt
26+
import typer.ImportInfo.withRootImports
2527

2628
import dotty.tools.dotc.{semanticdb => s}
2729
import dotty.tools.io.{AbstractFile, JarArchive}
30+
import dotty.tools.dotc.semanticdb.DiagnosticOps.*
31+
import scala.util.{Using, Failure, Success}
32+
2833

2934
/** Extract symbol references and uses to semanticdb files.
3035
* See https://scalameta.org/docs/semanticdb/specification.html#symbol-1
3136
* for a description of the format.
32-
* TODO: Also extract type information
37+
*
38+
* Here, we define two phases for "ExtractSemanticDB", "PostTyper" and "PostInlining".
39+
*
40+
* The "PostTyper" phase extracts SemanticDB information such as symbol
41+
* definitions, symbol occurrences, type information, and synthetics
42+
* and write .semanticdb file.
43+
*
44+
* The "PostInlining" phase extracts diagnostics from "ctx.reporter" and
45+
* attaches them to the SemanticDB information extracted in the "PostTyper" phase.
46+
* We need to run this phase after the "CheckUnused.PostInlining" phase
47+
* so that we can extract the warnings generated by "-Wunused".
3348
*/
34-
class ExtractSemanticDB extends Phase:
35-
import Scala3.{_, given}
49+
class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends Phase:
3650

37-
override val phaseName: String = ExtractSemanticDB.name
51+
override val phaseName: String = ExtractSemanticDB.phaseNamePrefix + phaseMode.toString()
3852

3953
override val description: String = ExtractSemanticDB.description
4054

@@ -46,14 +60,145 @@ class ExtractSemanticDB extends Phase:
4660
// Check not needed since it does not transform trees
4761
override def isCheckable: Boolean = false
4862

49-
override def run(using Context): Unit =
50-
val unit = ctx.compilationUnit
51-
val extractor = Extractor()
52-
extractor.extract(unit.tpdTree)
53-
ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList)
63+
override def runOn(units: List[CompilationUnit])(using ctx: Context): List[CompilationUnit] = {
64+
val sourceRoot = ctx.settings.sourceroot.value
65+
val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics
66+
if (appendDiagnostics)
67+
val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source)
68+
units.flatMap { unit =>
69+
warnings.get(unit.source).map { ws =>
70+
val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports
71+
val outputDir =
72+
ExtractSemanticDB.semanticdbPath(
73+
unit.source,
74+
ExtractSemanticDB.semanticdbOutDir(using unitCtx),
75+
sourceRoot
76+
)
77+
(outputDir, ws.map(_.toSemanticDiagnostic))
78+
}
79+
}.asJava.parallelStream().forEach { case (out, warnings) =>
80+
ExtractSemanticDB.appendDiagnostics(warnings, out)
81+
}
82+
else
83+
val writeSemanticdbText = ctx.settings.semanticdbText.value
84+
units.foreach { unit =>
85+
val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports
86+
val outputDir =
87+
ExtractSemanticDB.semanticdbPath(
88+
unit.source,
89+
ExtractSemanticDB.semanticdbOutDir(using unitCtx),
90+
sourceRoot
91+
)
92+
val extractor = ExtractSemanticDB.Extractor()
93+
extractor.extract(unit.tpdTree)(using unitCtx)
94+
ExtractSemanticDB.write(
95+
unit.source,
96+
extractor.occurrences.toList,
97+
extractor.symbolInfos.toList,
98+
extractor.synthetics.toList,
99+
outputDir,
100+
sourceRoot,
101+
writeSemanticdbText
102+
)
103+
}
104+
units
105+
}
106+
107+
def run(using Context): Unit = unsupported("run")
108+
end ExtractSemanticDB
109+
110+
object ExtractSemanticDB:
111+
import java.nio.file.Path
112+
import java.nio.file.Files
113+
import java.nio.file.Paths
114+
115+
val phaseNamePrefix: String = "extractSemanticDB"
116+
val description: String = "extract info into .semanticdb files"
117+
118+
enum PhaseMode:
119+
case ExtractSemanticInfo
120+
case AppendDiagnostics
121+
122+
class ExtractSemanticInfo extends ExtractSemanticDB(PhaseMode.ExtractSemanticInfo)
123+
124+
class AppendDiagnostics extends ExtractSemanticDB(PhaseMode.AppendDiagnostics)
125+
126+
private def semanticdbTarget(using Context): Option[Path] =
127+
Option(ctx.settings.semanticdbTarget.value)
128+
.filterNot(_.isEmpty)
129+
.map(Paths.get(_))
130+
131+
/** Destination for generated classfiles */
132+
private def outputDirectory(using Context): AbstractFile =
133+
ctx.settings.outputDir.value
134+
135+
/** Output directory for SemanticDB files */
136+
private def semanticdbOutDir(using Context): Path =
137+
semanticdbTarget.getOrElse(outputDirectory.jpath)
138+
139+
private def absolutePath(path: Path): Path = path.toAbsolutePath.normalize
140+
141+
private def write(
142+
source: SourceFile,
143+
occurrences: List[SymbolOccurrence],
144+
symbolInfos: List[SymbolInformation],
145+
synthetics: List[Synthetic],
146+
outpath: Path,
147+
sourceRoot: String,
148+
semanticdbText: Boolean
149+
): Unit =
150+
Files.createDirectories(outpath.getParent())
151+
val doc: TextDocument = TextDocument(
152+
schema = Schema.SEMANTICDB4,
153+
language = Language.SCALA,
154+
uri = Tools.mkURIstring(Paths.get(relPath(source, sourceRoot))),
155+
text = if semanticdbText then String(source.content) else "",
156+
md5 = internal.MD5.compute(String(source.content)),
157+
symbols = symbolInfos,
158+
occurrences = occurrences,
159+
synthetics = synthetics,
160+
)
161+
val docs = TextDocuments(List(doc))
162+
val out = Files.newOutputStream(outpath)
163+
try
164+
val stream = internal.SemanticdbOutputStream.newInstance(out)
165+
docs.writeTo(stream)
166+
stream.flush()
167+
finally
168+
out.close()
169+
end write
170+
171+
private def appendDiagnostics(
172+
diagnostics: Seq[Diagnostic],
173+
outpath: Path
174+
): Unit =
175+
Using.Manager { use =>
176+
val in = use(Files.newInputStream(outpath))
177+
val sin = internal.SemanticdbInputStream.newInstance(in)
178+
val docs = TextDocuments.parseFrom(sin)
179+
180+
val out = use(Files.newOutputStream(outpath))
181+
val sout = internal.SemanticdbOutputStream.newInstance(out)
182+
TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout)
183+
sout.flush()
184+
} match
185+
case Failure(ex) => // failed somehow, should we say something?
186+
case Success(_) => // success to update semanticdb, say nothing
187+
end appendDiagnostics
188+
189+
private def relPath(source: SourceFile, sourceRoot: String) =
190+
SourceFile.relativePath(source, sourceRoot)
191+
192+
private def semanticdbPath(source: SourceFile, base: Path, sourceRoot: String): Path =
193+
absolutePath(base)
194+
.resolve("META-INF")
195+
.resolve("semanticdb")
196+
.resolve(relPath(source, sourceRoot))
197+
.resolveSibling(source.name + ".semanticdb")
54198

55199
/** Extractor of symbol occurrences from trees */
56200
class Extractor extends TreeTraverser:
201+
import Scala3.{_, given}
57202
given s.SemanticSymbolBuilder = s.SemanticSymbolBuilder()
58203
val synth = SyntheticsExtractor()
59204
given converter: s.TypeOps = s.TypeOps()
@@ -465,55 +610,5 @@ class ExtractSemanticDB extends Phase:
465610
registerSymbol(vparam.symbol, symkinds)
466611
traverse(vparam.tpt)
467612
tparams.foreach(tp => traverse(tp.rhs))
468-
469-
470-
object ExtractSemanticDB:
471-
import java.nio.file.Path
472-
import java.nio.file.Files
473-
import java.nio.file.Paths
474-
475-
val name: String = "extractSemanticDB"
476-
val description: String = "extract info into .semanticdb files"
477-
478-
private def semanticdbTarget(using Context): Option[Path] =
479-
Option(ctx.settings.semanticdbTarget.value)
480-
.filterNot(_.isEmpty)
481-
.map(Paths.get(_))
482-
483-
private def semanticdbText(using Context): Boolean =
484-
ctx.settings.semanticdbText.value
485-
486-
private def outputDirectory(using Context): AbstractFile = ctx.settings.outputDir.value
487-
488-
def write(
489-
source: SourceFile,
490-
occurrences: List[SymbolOccurrence],
491-
symbolInfos: List[SymbolInformation],
492-
synthetics: List[Synthetic],
493-
)(using Context): Unit =
494-
def absolutePath(path: Path): Path = path.toAbsolutePath.normalize
495-
val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value)
496-
val outpath = absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath))
497-
.resolve("META-INF")
498-
.resolve("semanticdb")
499-
.resolve(relPath)
500-
.resolveSibling(source.name + ".semanticdb")
501-
Files.createDirectories(outpath.getParent())
502-
val doc: TextDocument = TextDocument(
503-
schema = Schema.SEMANTICDB4,
504-
language = Language.SCALA,
505-
uri = Tools.mkURIstring(Paths.get(relPath)),
506-
text = if semanticdbText then String(source.content) else "",
507-
md5 = internal.MD5.compute(String(source.content)),
508-
symbols = symbolInfos,
509-
occurrences = occurrences,
510-
synthetics = synthetics,
511-
)
512-
val docs = TextDocuments(List(doc))
513-
val out = Files.newOutputStream(outpath)
514-
try
515-
val stream = internal.SemanticdbOutputStream.newInstance(out)
516-
docs.writeTo(stream)
517-
stream.flush()
518-
finally
519-
out.close()
613+
end Extractor
614+
end ExtractSemanticDB

‎compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ object Scala3:
2929

3030
private val WILDCARDTypeName = nme.WILDCARD.toTypeName
3131

32-
def range(span: Span, treeSource: SourceFile)(using Context): Option[Range] =
32+
def range(span: Span, treeSource: SourceFile): Option[Range] =
3333
def lineCol(offset: Int) = (treeSource.offsetToLine(offset), treeSource.column(offset))
3434
val (startLine, startCol) = lineCol(span.start)
3535
val (endLine, endCol) = lineCol(span.end)
@@ -486,6 +486,8 @@ object Scala3:
486486

487487
given Ordering[SymbolInformation] = Ordering.by[SymbolInformation, String](_.symbol)(IdentifierOrdering())
488488

489+
given Ordering[Diagnostic] = (x, y) => compareRange(x.range, y.range)
490+
489491
given Ordering[Synthetic] = (x, y) => compareRange(x.range, y.range)
490492

491493
/**

‎compiler/src/dotty/tools/dotc/semanticdb/Tools.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ object Tools:
6969
sb.append("Language => ").append(languageString(doc.language)).nl
7070
sb.append("Symbols => ").append(doc.symbols.length).append(" entries").nl
7171
sb.append("Occurrences => ").append(doc.occurrences.length).append(" entries").nl
72+
if doc.diagnostics.nonEmpty then
73+
sb.append("Diagnostics => ").append(doc.diagnostics.length).append(" entries").nl
7274
if doc.synthetics.nonEmpty then
7375
sb.append("Synthetics => ").append(doc.synthetics.length).append(" entries").nl
7476
sb.nl
@@ -78,6 +80,10 @@ object Tools:
7880
sb.append("Occurrences:").nl
7981
doc.occurrences.sorted.foreach(processOccurrence)
8082
sb.nl
83+
if doc.diagnostics.nonEmpty then
84+
sb.append("Diagnostics:").nl
85+
doc.diagnostics.sorted.foreach(d => processDiag(d))
86+
sb.nl
8187
if doc.synthetics.nonEmpty then
8288
sb.append("Synthetics:").nl
8389
doc.synthetics.sorted.foreach(s => processSynth(s, synthPrinter))
@@ -108,6 +114,20 @@ object Tools:
108114
private def processSynth(synth: Synthetic, printer: SyntheticPrinter)(using sb: StringBuilder): Unit =
109115
sb.append(printer.pprint(synth)).nl
110116

117+
private def processDiag(d: Diagnostic)(using sb: StringBuilder): Unit =
118+
d.range match
119+
case Some(range) => processRange(sb, range)
120+
case _ => sb.append("[):")
121+
sb.append(" ")
122+
d.severity match
123+
case Diagnostic.Severity.ERROR => sb.append("[error]")
124+
case Diagnostic.Severity.WARNING => sb.append("[warning]")
125+
case Diagnostic.Severity.INFORMATION => sb.append("[info]")
126+
case _ => sb.append("[unknown]")
127+
sb.append(" ")
128+
sb.append(d.message)
129+
sb.nl
130+
111131
private def processOccurrence(occ: SymbolOccurrence)(using sb: StringBuilder, sourceFile: SourceFile): Unit =
112132
occ.range match
113133
case Some(range) =>

‎compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,8 @@ class SemanticdbTests:
142142
"-sourceroot", expectSrc.toString,
143143
"-classpath", target.toString,
144144
"-Xignore-scala2-macros",
145-
"-usejavacp"
145+
"-usejavacp",
146+
"-Wunused:all"
146147
) ++ inputFiles().map(_.toString)
147148
val exit = Main.process(args)
148149
assertFalse(s"dotc errors: ${exit.errorCount}", exit.hasErrors)

‎presentation-compiler/src/main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,7 @@ class SemanticdbTextDocumentProvider(
3232
SourceFile.virtual(filePath.toString, validCode)
3333
)
3434
val tree = driver.currentCtx.run.units.head.tpdTree
35-
val extract = ExtractSemanticDB()
36-
val extractor = extract.Extractor()
35+
val extractor = ExtractSemanticDB.Extractor()
3736
extractor.traverse(tree)(using driver.currentCtx)
3837
val path = workspace
3938
.flatMap { workspacePath =>

‎tests/semanticdb/expect/Deprecated.expect.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
object Deprecated/*<-_empty_::Deprecated.*/ {
2+
@deprecated/*->scala::deprecated#*/ def deprecatedMethod/*<-_empty_::Deprecated.deprecatedMethod().*/ = ???/*->scala::Predef.`???`().*/
3+
def main/*<-_empty_::Deprecated.main().*/ = deprecatedMethod/*->_empty_::Deprecated.deprecatedMethod().*/
4+
}

0 commit comments

Comments
 (0)