Migrate most of dokka-site in

Create utils over Java Map, Set and List

Author: romanowski

project/Build.scala

@@ -1465,22 +1465,24 @@ object Build {
 
       def joinProducts(products: Seq[java.io.File]): String =
         products.iterator.map(_.getAbsolutePath.toString).mkString(java.io.File.pathSeparator)
+      
+      val dokkaVersion = "1.4.10.2"
 
       project.settings(commonBootstrappedSettings).
         dependsOn(`scala3-compiler-bootstrapped`).
         dependsOn(`scala3-tasty-inspector`).
         settings(
           // Needed to download dokka and its dependencies
           resolvers += Resolver.jcenterRepo,
-          // Needed to download dokka-site
-          resolvers += Resolver.bintrayRepo("virtuslab", "dokka"),
           libraryDependencies ++= Seq(
-            "com.virtuslab.dokka" % "dokka-site" % "0.1.9",
+            "org.jetbrains.dokka" % "dokka-core" % dokkaVersion,
+            "org.jetbrains.dokka" % "dokka-base" % dokkaVersion,
+            "org.jetbrains.kotlinx" % "kotlinx-html-jvm" % "0.7.2", // Needs update when dokka version changes
             "com.vladsch.flexmark" % "flexmark-all" % "0.42.12",
             "nl.big-o" % "liqp" % "0.6.7",
             "args4j" % "args4j" % "2.33",
 
-            "org.jetbrains.dokka" % "dokka-test-api" % "1.4.10.2" % "test",
+            "org.jetbrains.dokka" % "dokka-test-api" % dokkaVersion % "test",
             "com.novocode" % "junit-interface" % "0.11" % "test",
           ),
           Test / test := (Test / test).dependsOn(compile.in(Compile).in(`scala3doc-testcases`)).value,

sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala

@@ -358,10 +358,7 @@ object DottyPlugin extends AutoPlugin {
       }.value,
 
       // Configuration for the doctool
-      resolvers ++= (if(!useScala3doc.value) Nil else Seq(
-        Resolver.jcenterRepo,
-        Resolver.bintrayRepo("virtuslab", "dokka"),
-      )),
+      resolvers ++= (if(!useScala3doc.value) Nil else Seq(Resolver.jcenterRepo)),
       useScala3doc := false,
       scala3docOptions := Nil,
       Compile / doc / scalacOptions := {

scala3doc/src/dotty/dokka/DottyDokkaConfig.scala

@@ -3,17 +3,16 @@ package dotty.dokka
 import org.jetbrains.dokka._
 import org.jetbrains.dokka.DokkaSourceSetImpl
 import java.io.File
-import java.util.{ List => JList, Map => JMap}
 import collection.JavaConverters._
 
 case class DottyDokkaConfig(docConfiguration: DocConfiguration) extends DokkaConfiguration:
   override def getOutputDir: File = docConfiguration.args.output
   override def getCacheRoot: File = null
   override def getOfflineMode: Boolean = false
   override def getFailOnWarning: Boolean = false
-  override def getSourceSets: JList[DokkaConfiguration.DokkaSourceSet] = List(mkSourceSet).asJava
-  override def getModules: JList[DokkaConfiguration.DokkaModuleDescription] = List().asJava
-  override def getPluginsClasspath: JList[File] = Nil.asJava
+  override def getSourceSets: JList[DokkaSourceSet] = JList(mkSourceSet)
+  override def getModules: JList[DokkaConfiguration.DokkaModuleDescription] = JList()
+  override def getPluginsClasspath: JList[File] = JList()
   override def getModuleName(): String = "ModuleName"
   override def getModuleVersion(): String = ""
 
@@ -23,31 +22,31 @@ case class DottyDokkaConfig(docConfiguration: DocConfiguration) extends DokkaCon
       DokkaConfiguration$SerializationFormat.JSON.asInstanceOf[DokkaConfiguration$SerializationFormat]
     override def getValues: String = docConfiguration.args.docsRoot.getOrElse("")
 
-  override def getPluginsConfiguration: JList[DokkaConfiguration.PluginConfiguration] = List(OurConfig).asJava
+  override def getPluginsConfiguration: JList[DokkaConfiguration.PluginConfiguration] = JList(OurConfig)
 
-  def mkSourceSet: DokkaConfiguration.DokkaSourceSet =
+  def mkSourceSet: DokkaSourceSet =
     val sourceLinks:Set[SourceLinkDefinitionImpl] = docConfiguration.args.sourceLinks.map(SourceLinkDefinitionImpl.Companion.parseSourceLinkDefinition(_)).toSet
     new DokkaSourceSetImpl(
       /*displayName=*/ docConfiguration.args.name,
       /*sourceSetID=*/ new DokkaSourceSetID(docConfiguration.args.name, "main"),
-      /*classpath=*/ Nil.asJava,
-      /*sourceRoots=*/ Set().asJava,
-      /*dependentSourceSets=*/ Set().asJava,
-      /*samples=*/ Set().asJava,
-      /*includes=*/ Set().asJava,
+      /*classpath=*/ JList(),
+      /*sourceRoots=*/ JSet(),
+      /*dependentSourceSets=*/ JSet(),
+      /*samples=*/ JSet(),
+      /*includes=*/ JSet(),
       /*includeNonPublic=*/ true,
       /*reportUndocumented=*/ false, /* changed because of exception in reportUndocumentedTransformer - there's 'when' which doesnt match because it contains only KotlinVisbility cases */
       /*skipEmptyPackages=*/ false, // Now all our packages are empty from dokka perspective
       /*skipDeprecated=*/ true,
       /*jdkVersion=*/ 8,
       /*sourceLinks=*/ sourceLinks.asJava,
-      /*perPackageOptions=*/ Nil.asJava,
-      /*externalDocumentationLinks=*/ Set().asJava,
+      /*perPackageOptions=*/ JList(),
+      /*externalDocumentationLinks=*/ JSet(),
       /*languageVersion=*/ null,
       /*apiVersion=*/ null,
       /*noStdlibLink=*/ true,
       /*noJdkLink=*/  true,
-      /*suppressedFiles=*/  Set().asJava,
+      /*suppressedFiles=*/  JSet(),
       /*suppressedFiles=*/  Platform.jvm
-    ).asInstanceOf[DokkaConfiguration.DokkaSourceSet] // Why I do need to cast here? Kotlin magic?
+    ).asInstanceOf[DokkaSourceSet] // Why I do need to cast here? Kotlin magic?
 

scala3doc/src/dotty/dokka/DottyDokkaPlugin.scala

@@ -12,8 +12,6 @@ import org.jetbrains.dokka.links._
 import org.jetbrains.dokka.model.doc._
 import org.jetbrains.dokka.base.parsers._
 import org.jetbrains.dokka.plugability.DokkaContext
-import com.virtuslab.dokka.site.SourceSetWrapper
-import com.virtuslab.dokka.site.JavaSourceToDocumentableTranslator
 import collection.JavaConverters._
 import org.jetbrains.dokka.model.properties.PropertyContainer
 import dotty.dokka.tasty.{DokkaTastyInspector, SbtDokkaTastyInspector}
@@ -23,10 +21,7 @@ import org.jetbrains.dokka.base.signatures.SignatureProvider
 import org.jetbrains.dokka.pages._
 import dotty.dokka.model.api._
 import org.jetbrains.dokka.CoreExtensions
-import com.virtuslab.dokka.site.StaticSitePlugin
 import org.jetbrains.dokka.base.DokkaBase
-import com.virtuslab.dokka.site.ExtensionBuilderEx
-import java.util.{List => JList}
 
 /** Main Dokka plugin for the doctool.
   *
@@ -38,7 +33,6 @@ import java.util.{List => JList}
 class DottyDokkaPlugin extends DokkaJavaPlugin:
 
   lazy val dokkaBase = plugin(classOf[DokkaBase])
-  lazy val dokkaSitePlugin = plugin(classOf[StaticSitePlugin])
 
   val provideMembers = extend(
     _.extensionPoint(CoreExtensions.INSTANCE.getSourceToDocumentableTranslator)
@@ -116,7 +110,7 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
   val ourRenderer = extend(
     _.extensionPoint(CoreExtensions.INSTANCE.getRenderer)
       .fromRecipe(ScalaHtmlRenderer(_))
-      .overrideExtension(dokkaSitePlugin.getCustomRenderer)
+      .overrideExtension(dokkaBase.getHtmlRenderer)
   )
 
   val commentsToContentConverter = extend(
@@ -140,10 +134,23 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
     .name("muteDefaultSourceLinksTransformer")
   )
 
-// TODO remove once problem is fixed in Dokka
+// TODO (https://github.com/lampepfl/scala3doc/issues/232): remove once problem is fixed in Dokka
 extension [T]  (builder: ExtensionBuilder[T]):
-  def before(exts: Extension[_, _, _]*):  ExtensionBuilder[T] =
-    (new ExtensionBuilderEx).newOrdering(builder, exts.toArray, Array.empty)
-
-  def after(exts: Extension[_, _, _]*):  ExtensionBuilder[T] =
-    (new ExtensionBuilderEx).newOrdering(builder, Array.empty, exts.toArray)
+  def ordered(before: Seq[Extension[_, _, _]], after: Seq[Extension[_, _, _]]): ExtensionBuilder[T] =
+    val byDsl = new OrderingKind.ByDsl(dsl => {
+      dsl.after(after:_*)
+      dsl.before(before:_*)
+      kotlin.Unit.INSTANCE // TODO why U does not work here? 
+    })
+    // Does not compile but compiles in scala 2
+    // ExtensionBuilder.copy$default(builder, null, null, null, byDsl, null, null, 55, null)
+    val m = classOf[ExtensionBuilder[_]].getDeclaredMethods().find(_.getName == "copy$default").get
+    m.setAccessible(true)
+    // All nulls and 55 is taken from Kotlin bytecode and represent how defaut parameter are represented in Kotlin
+    // Defaut arguments are encoded by null and mapping that 55 represent whic arguments are actually provided
+    m.invoke(null, builder, null, null, null, byDsl, null, null, 55, null).asInstanceOf[ExtensionBuilder[T]]
+
+
+  def before(exts: Extension[_, _, _]*):  ExtensionBuilder[T] = ordered(exts, Nil)
+    
+  def after(exts: Extension[_, _, _]*):  ExtensionBuilder[T] = ordered(Nil, exts)

scala3doc/src/dotty/dokka/Main.scala

@@ -8,7 +8,6 @@ import java.io.File
 import java.util.jar._
 import collection.JavaConverters._
 import collection.immutable.ArraySeq
-import java.util.{List => JList}
 
 import scala.tasty.Reflection
 import scala.tasty.inspector.TastyInspector

scala3doc/src/dotty/dokka/ScalaModuleCreator.scala

@@ -1,23 +1,22 @@
 package dotty.dokka
 
 import org.jetbrains.dokka.{ DokkaConfiguration$DokkaSourceSet => DokkaSourceSet }
-import com.virtuslab.dokka.site.JavaSourceToDocumentableTranslator
-import com.virtuslab.dokka.site.SourceSetWrapper
 import org.jetbrains.dokka.plugability.DokkaContext
 import org.jetbrains.dokka.model.properties.PropertyContainer
+import org.jetbrains.dokka.model.DModule
+import org.jetbrains.dokka.transformers.sources.SourceToDocumentableTranslator
 
 import dotty.dokka.tasty.{DokkaTastyInspector, SbtDokkaTastyInspector}
 import org.jetbrains.dokka.pages._
 import dotty.dokka.model.api._
 import org.jetbrains.dokka.model._
 import org.jetbrains.dokka.links.DRI
-import java.util.{List => JList}
 import org.jetbrains.dokka.base.parsers.MarkdownParser
 import collection.JavaConverters._
+import kotlin.coroutines.Continuation
 
-object ScalaModuleProvider extends JavaSourceToDocumentableTranslator:
-   override def process(rawSourceSet: DokkaSourceSet, cxt: DokkaContext) =
-    val sourceSet = SourceSetWrapper(rawSourceSet)
+object ScalaModuleProvider extends SourceToDocumentableTranslator:
+   override def invoke(sourceSet: DokkaSourceSet, cxt: DokkaContext, unused: Continuation[? >: DModule]) =
     cxt.getConfiguration match
       case dottyConfig: DottyDokkaConfig =>
         val result = dottyConfig.docConfiguration match {
@@ -41,16 +40,16 @@ object ScalaModuleProvider extends JavaSourceToDocumentableTranslator:
         def flattenMember(m: Member): Seq[(DRI, Member)] = (m.dri -> m) +: m.allMembers.flatMap(flattenMember)
 
         new DModule(
-          sourceSet.getSourceSet.getDisplayName,
+          sourceSet.getDisplayName,
           result.asJava,
-          Map().asJava,
+          JMap(),
           null,
           sourceSet.toSet,
           PropertyContainer.Companion.empty() plus ModuleExtension(result.flatMap(flattenMember).toMap)
         )
       case _ =>
         ???
 
-object EmptyModuleProvider extends JavaSourceToDocumentableTranslator:
-  override def process(sourceSet: DokkaSourceSet, context: DokkaContext) =
-    DModule("", Nil.asJava, Map.empty.asJava, null, Set(sourceSet).asJava, PropertyContainer.Companion.empty())
+object EmptyModuleProvider extends SourceToDocumentableTranslator:
+   override def invoke(sourceSet: DokkaSourceSet, cxt: DokkaContext, unused: Continuation[? >: DModule]) =
+    DModule("", JList(), Map.empty.asJava, null, Set(sourceSet).asJava, PropertyContainer.Companion.empty())

scala3doc/src/dotty/dokka/compat.scala

@@ -0,0 +1,59 @@
+package dotty.dokka
+
+import org.jetbrains.dokka.links.{DRI, PointingToDeclaration}
+import org.jetbrains.dokka.DokkaConfiguration
+import org.jetbrains.dokka.DokkaConfiguration$DokkaSourceSet
+import collection.JavaConverters._
+import org.jetbrains.dokka.model.DisplaySourceSet
+import org.jetbrains.dokka.model.properties.WithExtraProperties
+// TODO reproduction! - comment line below to broke compiler!
+import org.jetbrains.dokka.model.properties.ExtraProperty
+// import java.util.Stream // TODO reproduction uncomment
+import java.util.stream.Stream // comment out - wrong error!
+import java.util.stream.Collectors
+
+def mkDRI(classNames: String = null, extra: String = null) = new DRI(null, classNames, null, PointingToDeclaration.INSTANCE, extra)
+
+val U: kotlin.Unit = kotlin.Unit.INSTANCE
+
+def JList[T](e: T*): JList[T] = e.asJava
+def JSet[T](e: T*): JSet[T] = e.toSet.asJava
+def JMap[K, V](e: (K, V)*): JMap[K, V] = e.toMap.asJava
+def JMap2[K, V](): JMap[K, V] = ??? // e.toMap.asJava
+
+type JList[T] = java.util.List[T]
+type JSet[T] = java.util.Set[T]
+type JMap[K, V] = java.util.Map[K, V]
+
+type SourceSetWrapper = DokkaConfiguration$DokkaSourceSet
+type DokkaSourceSet = DokkaConfiguration.DokkaSourceSet
+
+extension [T] (wrapper: SourceSetWrapper):
+    def toSet: JSet[DokkaConfiguration$DokkaSourceSet] = JSet(wrapper)
+    def toMap(value: T): JMap[DokkaConfiguration$DokkaSourceSet, T] = JMap(wrapper -> value)
+
+extension [T] (wrapper: DokkaSourceSet):
+    // when named `toSet` fails in runtime -- TODO: create a minimal!
+    // def toSet: JSet[DokkaConfiguration$DokkaSourceSet] = JSet(wrapper.asInstanceOf[SourceSetWrapper])
+    def asSet: JSet[DokkaConfiguration$DokkaSourceSet] = JSet(wrapper.asInstanceOf[SourceSetWrapper])
+    def asMap(value: T): JMap[DokkaConfiguration$DokkaSourceSet, T] = JMap(wrapper.asInstanceOf[SourceSetWrapper] -> value)    
+
+extension (sourceSets: JList[DokkaSourceSet]):
+  def asDokka: JSet[SourceSetWrapper] = sourceSets.asScala.toSet.asJava.asInstanceOf[JSet[SourceSetWrapper]]
+  def toDisplaySourceSet = sourceSets.asScala.map(ss => DisplaySourceSet(ss.asInstanceOf[SourceSetWrapper])).toSet.asJava 
+
+extension (sourceSets: Set[SourceSetWrapper]):
+  def toDisplay = sourceSets.map(DisplaySourceSet(_)).asJava    
+  
+extension [V] (a: WithExtraProperties[_]):
+  def get(key: ExtraProperty.Key[_, V]): V = a.getExtra().getMap().get(key).asInstanceOf[V]
+
+extension [E <: WithExtraProperties[E]] (a: E):
+  def put(value: ExtraProperty[_ >: E]): E = a.withNewExtras(a.getExtra plus value)
+
+extension [V] (map: JMap[SourceSetWrapper, V]):
+  def defaultValue: V = map.values.asScala.head
+
+extension [V](jlist: JList[V]):
+  def ++ (other: JList[V]): JList[V] = 
+    Stream.of(jlist, other).flatMap(_.stream).collect(Collectors.toList())  

scala3doc/src/dotty/dokka/model/api/api.scala

@@ -10,8 +10,6 @@ import org.jetbrains.dokka.links._
 import org.jetbrains.dokka.model.doc._
 import org.jetbrains.dokka.model.properties._
 import org.jetbrains.dokka.pages._
-import java.util.{List => JList, Set => JSet}
-
 
 enum Visibility(val name: String):
   case Unrestricted extends Visibility("")

scala3doc/src/dotty/dokka/model/api/internalExtensions.scala

@@ -21,9 +21,6 @@ import collection.JavaConverters._
 import org.jetbrains.dokka.links._
 import org.jetbrains.dokka.model.doc.DocumentationNode
 import org.jetbrains.dokka.model.properties._
-import java.util.{List => JList, Set => JSet}
-
-import com.virtuslab.dokka.site.SourceSetWrapper
 
 private [model] case class MemberExtension(
   visibility: Visibility,

scala3doc/src/dotty/dokka/model/extras.scala

@@ -9,7 +9,6 @@ import collection.JavaConverters._
 import org.jetbrains.dokka.links._
 import org.jetbrains.dokka.model.doc._
 import org.jetbrains.dokka.model.properties._
-import java.util.{List => JList, Set => JSet}
 import dotty.dokka.model.api._
 
 case class ModuleExtension(driMap: Map[DRI, Member]) extends ExtraProperty[DModule]: