Merge pull request #10593 from dotty-staging/scala3doc/improve-stdlib-loading

Scala3doc: improve stdlib loading

Author: abgruszecki

.github/workflows/scala3doc.yaml

@@ -45,9 +45,6 @@ jobs:
       - name: Generate Scala 3 documentation
         run: ./project/scripts/sbt scala3doc/generateScala3Documentation
 
-      - name: Generate Scala 3 stdlib documentation
-        run: ./project/scripts/sbt scala3doc/generateScala3StdlibDocumentation
-
       - name: Generate documentation for example project using dotty-sbt
         run: ./project/scripts/sbt "sbt-dotty/scripted sbt-dotty/scala3doc"
 

library/src/scala/quoted/Quotes.scala

@@ -176,6 +176,8 @@ trait Quotes { self: runtime.QuoteUnpickler & runtime.QuoteMatching =>
    *  +- Flags
    *
    *  ```
+   *
+   * @syntax markdown
    */
   trait Reflection { self: reflect.type =>
 

project/Build.scala

@@ -841,6 +841,16 @@ object Build {
     settings(
       moduleName := "scala-library",
       javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
+      Compile/scalacOptions += "-Yerased-terms",
+      Compile/scalacOptions ++= {
+        Seq(
+          "-sourcepath",
+          Seq(
+            (Compile/sourceManaged).value / "scala-library-src",
+            (Compile/sourceManaged).value / "dotty-library-src",
+          ).mkString(File.pathSeparator),
+        )
+      },
       scalacOptions -= "-Xfatal-warnings",
       ivyConfigurations += SourceDeps.hide,
       transitiveClassifiers := Seq("sources"),
@@ -870,6 +880,30 @@ object Build {
           ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
         } (Set(scalaLibrarySourcesJar)).toSeq
       }.taskValue,
+      sourceGenerators in Compile += Def.task {
+        val s = streams.value
+        val cacheDir = s.cacheDirectory
+        val trgDir = (sourceManaged in Compile).value / "dotty-library-src"
+
+        // NOTE `sourceDirectory` is used for actual copying,
+        // but `sources` are used as cache keys
+        val dottyLibSourceDir = (`scala3-library-bootstrapped`/sourceDirectory).value
+        val dottyLibSources = (`scala3-library-bootstrapped`/Compile/sources).value
+
+        val cachedFun = FileFunction.cached(
+          cacheDir / s"copyDottyLibrarySrc",
+          FilesInfo.lastModified,
+          FilesInfo.exists,
+        ) { _ =>
+          s.log.info(s"Copying scala3-library sources from $dottyLibSourceDir to $trgDir...")
+          if (trgDir.exists) IO.delete(trgDir)
+          IO.copyDirectory(dottyLibSourceDir, trgDir)
+
+          ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
+        }
+
+        cachedFun(dottyLibSources.toSet).toSeq
+      }.taskValue,
       sources in Compile ~= (_.filterNot(file =>
         // sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
         file.getPath.endsWith("scala-library-src/scala/Any.scala") ||
@@ -1182,7 +1216,6 @@ object Build {
   val generateSelfDocumentation = taskKey[Unit]("Generate example documentation")
   // Note: the two tasks below should be one, but a bug in Tasty prevents that
   val generateScala3Documentation = inputKey[Unit]("Generate documentation for dotty lib")
-  val generateScala3StdlibDocumentation = taskKey[Unit]("Generate documentation for Scala3 standard library")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
   lazy val `scala3doc` = project.in(file("scala3doc")).asScala3doc
   lazy val `scala3doc-testcases` = project.in(file("scala3doc-testcases")).asScala3docTestcases
@@ -1489,7 +1522,7 @@ object Build {
       }
 
       def joinProducts(products: Seq[java.io.File]): String =
-        products.iterator.map(_.getAbsolutePath.toString).mkString(java.io.File.pathSeparator)
+        products.iterator.map(_.getAbsolutePath.toString).mkString(" ")
 
       val dokkaVersion = "1.4.10.2"
 
@@ -1532,12 +1565,12 @@ object Build {
             val majorVersion = (scalaBinaryVersion in LocalProject("scala3-library-bootstrapped")).value
 
             val dottyJars: Seq[java.io.File] = Seq(
+              (`stdlib-bootstrapped`/Compile/products).value,
               (`scala3-interfaces`/Compile/products).value,
               (`tasty-core-bootstrapped`/Compile/products).value,
-              (`scala3-library-bootstrapped`/Compile/products).value,
             ).flatten
 
-            val roots = dottyJars.mkString(" ")
+            val roots = joinProducts(dottyJars)
 
             if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") }
             else Def.task{
@@ -1547,24 +1580,9 @@ object Build {
               IO.write(dest / "CNAME", "dotty.epfl.ch")
             }.dependsOn(generateDocumentation(
               roots, "Scala 3", dest.getAbsolutePath, "master",
-              "-siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg"))
+              "-comment-syntax wiki -siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg"))
           }.evaluated,
 
-
-          generateScala3StdlibDocumentation:= Def.taskDyn {
-            val dottyJars: Seq[java.io.File] = Seq(
-              (`stdlib-bootstrapped`/Compile/products).value,
-            ).flatten
-
-            val roots = joinProducts(dottyJars)
-
-            if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") }
-            else generateDocumentation(
-              roots, "Scala 3", "scala3doc/output/scala3-stdlib", "maser",
-              "-siteroot scala3doc/scala3-docs -comment-syntax wiki -project-logo scala3doc/scala3-docs/logo.svg "
-            )
-          }.value,
-
           generateTestcasesDocumentation := Def.taskDyn {
             generateDocumentation(Build.testcasesOutputDir.in(Test).value, "Scala3doc testcases", "scala3doc/output/testcases", "master")
           }.value,

scala3doc/src/dotty/dokka/tasty/ClassLikeSupport.scala

@@ -69,11 +69,13 @@ trait ClassLikeSupport:
 
       val fullExtra =
         if (signatureOnly) baseExtra
-        else baseExtra.plus(CompositeMemberExtension(
-          classDef.extractMembers,
-          classDef.getParents.map(_.dokkaType.asSignature),
-          supertypes,
-          Nil))
+        else
+          baseExtra.plus(CompositeMemberExtension(
+            classDef.extractPatchedMembers,
+            classDef.getParents.map(_.dokkaType.asSignature),
+            supertypes,
+            Nil))
+        end if
 
       new DClass(
           dri,
@@ -203,6 +205,31 @@ trait ClassLikeSupport:
         inherited.flatMap(s => parseInheritedMember(s))
     }
 
+    /** Extracts members while taking Dotty logic for patching the stdlib into account. */
+    def extractPatchedMembers: Seq[Member] = {
+      val ownMembers = c.extractMembers
+      def extractPatchMembers(sym: Symbol) = {
+        // NOTE for some reason scala.language$.experimental$ class doesn't show up here, so we manually add the name
+        val ownMemberDRIs = ownMembers.iterator.map(_.name).toSet + "experimental$"
+        sym.tree.asInstanceOf[ClassDef]
+          .membersToDocument.filterNot(m => ownMemberDRIs.contains(m.symbol.name))
+          .flatMap(parseMember)
+      }
+      c.symbol.show match {
+        case "scala.Predef$" =>
+          ownMembers ++
+          extractPatchMembers(qctx.reflect.Symbol.requiredClass("scala.runtime.stdLibPatches.Predef$"))
+        case "scala.language$" =>
+          ownMembers ++
+          extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language").moduleClass)
+        case "scala.language$.experimental$" =>
+          ownMembers ++
+          extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language.experimental").moduleClass)
+        case _ => ownMembers
+      }
+
+    }
+
     def getParents: List[Tree] =
       for
         parentTree <- c.parents if isValidPos(parentTree.pos)  // We assume here that order is correct
@@ -417,4 +444,3 @@ trait ClassLikeSupport:
           valDef.symbol.source
       ))
     )
-

scala3doc/src/dotty/dokka/tasty/TastyParser.scala

@@ -30,17 +30,22 @@ case class DokkaTastyInspector(parser: Parser)(using ctx: DocContext) extends Do
   private val topLevels = Seq.newBuilder[Documentable]
 
   def processCompilationUnit(using q: Quotes)(root: q.reflect.Tree): Unit =
-    val parser = new TastyParser(q, this)
-
-    def driFor(link: String): Option[DRI] =
-      val symOps = new SymOps[q.type](q)
-      import symOps._
-      Try(QueryParser(link).readQuery()).toOption.flatMap(q =>
-        MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri}
-      )
+    // NOTE we avoid documenting definitions in the magical stdLibPatches directory;
+    // the symbols there are "patched" through dark Dotty magic onto other stdlib
+    // definitions, so if we documented their origin, we'd get defs with duplicate DRIs
+    if !root.symbol.show.startsWith("scala.runtime.stdLibPatches") then
+      val parser = new TastyParser(q, this)
+
+      def driFor(link: String): Option[DRI] =
+        val symOps = new SymOps[q.type](q)
+        import symOps._
+        Try(QueryParser(link).readQuery()).toOption.flatMap(q =>
+          MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri}
+        )
 
-    ctx.staticSiteContext.foreach(_.memberLinkResolver = driFor)
-    topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree])
+      ctx.staticSiteContext.foreach(_.memberLinkResolver = driFor)
+      topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree])
+  end processCompilationUnit
 
   def result(): List[DPackage] =
     topLevels.clear()

scala3doc/src/dotty/dokka/tasty/comments/wiki/Converter.scala

@@ -31,13 +31,16 @@ class Converter(val repr: Repr) extends BaseConverter {
     block match {
       case Title(text, level) =>
         val content = convertInline(text)
+        // NOTE: these aren't strictly necessary, but if you inline them, incremental compilation will break
+        val jContent = content.asJava : java.util.List[_ <: dkkd.DocTag]
+        val jAtt = kt.emptyMap[String, String]
         emit(level match {
-          case 1 => dkkd.H1(content.asJava, kt.emptyMap)
-          case 2 => dkkd.H2(content.asJava, kt.emptyMap)
-          case 3 => dkkd.H3(content.asJava, kt.emptyMap)
-          case 4 => dkkd.H4(content.asJava, kt.emptyMap)
-          case 5 => dkkd.H5(content.asJava, kt.emptyMap)
-          case 6 => dkkd.H6(content.asJava, kt.emptyMap)
+          case 1 => dkkd.H1(jContent, jAtt)
+          case 2 => dkkd.H2(jContent, jAtt)
+          case 3 => dkkd.H3(jContent, jAtt)
+          case 4 => dkkd.H4(jContent, jAtt)
+          case 5 => dkkd.H5(jContent, jAtt)
+          case 6 => dkkd.H6(jContent, jAtt)
         })
 
       case Paragraph(text) =>