Refactor static-site loading

Author: pikinier20

docs/docs/usage/scaladoc/index.md

@@ -3,6 +3,6 @@ layout: doc-page
 title: "Scaladoc"
 ---
 
-![scaladoc logo](images/scaladoc_logo.svg)
+![scaladoc logo](images/scaladoc-logo.png)
 
 scaladoc is a tool to generate the API documentation of your Scala 3 projects. It provides similar features to `javadoc` as well as `jekyll` or `docusaurus`.

docs/sidebar.yml

@@ -1,22 +1,35 @@
-sidebar:
-  - title: Blog
+rootIndex: docs/index.md
+pages:
+  # - title: Blog
   - title: Usage
     subsection:
       - page: docs/usage/sbt-projects.md
       - page: docs/usage/ide-support.md
       - page: docs/usage/cbt-projects.md
-      - page: docs/usage/scaladoc
+      - title: Scaladoc
+        index: docs/usage/scaladoc/index.md
+        subsection:
+          - page: docs/usage/scaladoc/docstrings.md
+          - page: docs/usage/scaladoc/linking.md
+          - page: docs/usage/scaladoc/search-engine.md
+          - page: docs/usage/scaladoc/settings.md
+          - page: docs/usage/scaladoc/site-versioning.md
+          - page: docs/usage/scaladoc/static-site.md
   - title: Reference
     subsection:
       - page: docs/reference/overview.md
       - title: New Types
         index: docs/reference/new-types/new-types.md
         subsection:
           - page: docs/reference/new-types/intersection-types.md
+          - page: docs/reference/new-types/intersection-types-spec.md
           - page: docs/reference/new-types/union-types.md
+          - page: docs/reference/new-types/union-types-spec.md
           - page: docs/reference/new-types/type-lambdas.md
+          - page: docs/reference/new-types/type-lambdas-spec.md
           - page: docs/reference/new-types/match-types.md
           - page: docs/reference/new-types/dependent-function-types.md
+          - page: docs/reference/new-types/dependent-function-types-spec.md
           - page: docs/reference/new-types/polymorphic-function-types.md
       - title: Enums
         index: docs/reference/enums/enums-index.md
@@ -26,16 +39,20 @@ sidebar:
           - page: docs/reference/enums/desugarEnums.md
       - title: Contextual Abstractions
         index: docs/reference/contextual/contextual.md
+        directory: contextual
         subsection:
           - page: docs/reference/contextual/givens.md
           - page: docs/reference/contextual/using-clauses.md
           - page: docs/reference/contextual/context-bounds.md
           - page: docs/reference/contextual/given-imports.md
           - page: docs/reference/contextual/extension-methods.md
+          - page: docs/reference/contextual/right-associative-extension-methods.md
           - page: docs/reference/contextual/type-classes.md
           - page: docs/reference/contextual/derivation.md
+          - page: docs/reference/contextual/derivation-macro.md
           - page: docs/reference/contextual/multiversal-equality.md
           - page: docs/reference/contextual/context-functions.md
+          - page: docs/reference/contextual/context-functions-spec.md
           - page: docs/reference/contextual/conversions.md
           - page: docs/reference/contextual/by-name-context-parameters.md
           - page: docs/reference/contextual/relationship-implicits.md
@@ -45,6 +62,7 @@ sidebar:
           - page: docs/reference/metaprogramming/inline.md
           - page: docs/reference/metaprogramming/compiletime-ops.md
           - page: docs/reference/metaprogramming/macros.md
+          - page: docs/reference/metaprogramming/macros-spec.md
           - page: docs/reference/metaprogramming/staging.md
           - page: docs/reference/metaprogramming/reflection.md
           - page: docs/reference/metaprogramming/tasty-inspect.md
@@ -56,8 +74,10 @@ sidebar:
           - page: docs/reference/other-new-features/creator-applications.md
           - page: docs/reference/other-new-features/export.md
           - page: docs/reference/other-new-features/opaques.md
+          - page: docs/reference/other-new-features/opaques-details.md
           - page: docs/reference/other-new-features/open-classes.md
           - page: docs/reference/other-new-features/parameter-untupling.md
+          - page: docs/reference/other-new-features/parameter-untupling-spec.md
           - page: docs/reference/other-new-features/kind-polymorphism.md
           - page: docs/reference/other-new-features/matchable.md
           - page: docs/reference/other-new-features/threadUnsafe-annotation.md
@@ -68,23 +88,27 @@ sidebar:
           - page: docs/reference/other-new-features/type-test.md
           - page: docs/reference/other-new-features/experimental-defs.md
       - title: Other Changed Features
+        directory: changed-features
         index: docs/reference/changed-features/changed-features.md
         subsection:
           - page: docs/reference/changed-features/numeric-literals.md
           - page: docs/reference/changed-features/structural-types.md
+          - page: docs/reference/changed-features/structural-types-spec.md
           - page: docs/reference/changed-features/operators.md
           - page: docs/reference/changed-features/wildcards.md
           - page: docs/reference/changed-features/imports.md
           - page: docs/reference/changed-features/type-checking.md
           - page: docs/reference/changed-features/type-inference.md
           - page: docs/reference/changed-features/implicit-resolution.md
           - page: docs/reference/changed-features/implicit-conversions.md
+          - page: docs/reference/changed-features/implicit-conversions-spec.md
           - page: docs/reference/changed-features/overload-resolution.md
           - page: docs/reference/changed-features/match-syntax.md
           - page: docs/reference/changed-features/vararg-splices.md
           - page: docs/reference/changed-features/pattern-bindings.md
           - page: docs/reference/changed-features/pattern-matching.md
           - page: docs/reference/changed-features/eta-expansion.md
+          - page: docs/reference/changed-features/eta-expansion-spec.md
           - page: docs/reference/changed-features/compiler-plugins.md
           - page: docs/reference/changed-features/lazy-vals-init.md
           - page: docs/reference/changed-features/main-functions.md
@@ -100,15 +124,18 @@ sidebar:
           - page: docs/reference/dropped-features/package-objects.md
           - page: docs/reference/dropped-features/early-initializers.md
           - page: docs/reference/dropped-features/class-shadowing.md
+          - page: docs/reference/dropped-features/class-shadowing-spec.md
           - page: docs/reference/dropped-features/limit22.md
           - page: docs/reference/dropped-features/xml.md
           - page: docs/reference/dropped-features/symlits.md
           - page: docs/reference/dropped-features/auto-apply.md
           - page: docs/reference/dropped-features/weak-conformance.md
+          - page: docs/reference/dropped-features/weak-conformance-spec.md
           - page: docs/reference/dropped-features/nonlocal-returns.md
           - page: docs/reference/dropped-features/this-qualifier.md
           - page: docs/reference/dropped-features/wildcard-init.md
       - title: Experimental Features
+        directory: experimental
         subsection:
           - page: docs/reference/experimental/overview.md
           - page: docs/reference/experimental/canthrow.md
@@ -123,6 +150,8 @@ sidebar:
         subsection:
           - page: docs/reference/language-versions/source-compatibility.md
           - page: docs/reference/language-versions/binary-compatibility.md
+      - page: docs/reference/soft-modifier.md
+      - page: docs/reference/features-classification.md
   - title: Contributing
     subsection:
       - page: docs/contributing/contribute-knowledge.md
@@ -131,6 +160,7 @@ sidebar:
       - page: docs/contributing/testing.md
       - page: docs/contributing/debugging.md
       - title: IDEs and Tools
+        directory: tools
         subsection:
           - page: docs/contributing/tools/ide.md
           - page: docs/contributing/tools/mill.md

scaladoc-testcases/docs/sidebar.yml

@@ -1,7 +1,6 @@
-sidebar:
-  - title: Docs
-    subsection:
-      - page: docs/docs/f1.md
-      - page: docs/docs/f2.md
-      - page: docs/docs/f3.md
-      - page: docs/docs/f4.md
+rootIndex: docs/index.md
+pages:
+  - page: docs/docs/f1.md
+  - page: docs/docs/f2.md
+  - page: docs/docs/f3.md
+  - page: docs/docs/f4.md

scaladoc/src/dotty/tools/scaladoc/renderers/Locations.scala

@@ -34,9 +34,9 @@ trait Locations(using ctx: DocContext):
     cache.get(dri) match
       case null =>
         val path = dri match
-          case `docsRootDRI` => List("docs", "index")
+          // case `docsRootDRI` => List("docs", "index")
           case `apiPageDRI` =>
-            if ctx.staticSiteContext.fold(false)(_.hasIndexFile)
+            if ctx.args.apiSubdirectory && ctx.staticSiteContext.nonEmpty
               then List("api", "index")
               else List("index")
           case dri if dri.isStaticFile =>
@@ -88,6 +88,6 @@ trait Locations(using ctx: DocContext):
 
   def pathToRoot(dri: DRI): String = rawLocation(dri).drop(1).map(_ => "..") match
     case Nil => ""
-    case seq => seq.mkString("", "/", "/")
+    case seq => seq.mkString("", "/" , "/")
 
   def driExists(dri: DRI) = effectiveMembers.get(dri).isDefined || dri.isStaticFile

scaladoc/src/dotty/tools/scaladoc/renderers/Renderer.scala

@@ -40,20 +40,12 @@ abstract class Renderer(rootPackage: Member, val members: Map[DRI, Member], prot
     staticSite match
       case None => rootPckPage.withTitle(args.name)
       case Some(siteContext) =>
-        val (indexes, templates) = siteContext.templates.partition(f =>
-          f.templateFile.isIndexPage() && f.file.toPath.getParent() == siteContext.docsPath)
-        if (indexes.size > 1)
-          val msg = s"ERROR: Multiple index pages for doc found ${indexes.map(_.file)}"
-          report.error(msg)
+        val rootTemplate = siteContext.staticSiteRoot.rootTemplate
 
         // Below code is for walking in order the tree and modifing its nodes basing on its neighbours
 
-        // First we flatten templates to get them sorted in-order
-        def flattenedTemplates(template: LoadedTemplate): Seq[LoadedTemplate] =
-          template +: template.children.flatMap(flattenedTemplates)
-
         // We add dummy guards
-        val allTemplates: Seq[Option[LoadedTemplate]] = None +: templates.flatMap(flattenedTemplates).map(Some(_)) :+ None
+        val allTemplates: Seq[Option[LoadedTemplate]] = None +: siteContext.allTemplates.map(Some(_)) :+ None
 
         // Let's gather the list of maps for each template with its in-order neighbours
         val newSettings: List[Map[String, Object]] = allTemplates.sliding(size = 3, step = 1).map {
@@ -82,34 +74,17 @@ abstract class Renderer(rootPackage: Member, val members: Map[DRI, Member], prot
             )
           updatedTemplates.result()
 
-        val newTemplates = updateSettings(templates, newSettings.to(ListBuffer))
+        val newTemplates = updateSettings(Seq(siteContext.staticSiteRoot.rootTemplate), newSettings.to(ListBuffer))
         val templatePages = newTemplates.map(templateToPage(_, siteContext))
 
-        indexes.headOption match
-          case None if templatePages.isEmpty=>
-            rootPckPage.withTitle(args.name)
-          case None =>
-            Page(Link(args.name, docsRootDRI),"", templatePages :+ rootPckPage.withTitle("API"))
-          case Some(indexPage) =>
-            val newChildren = templatePages :+ rootPckPage.withTitle("API")
-            templateToPage(indexPage, siteContext).withNewChildren(newChildren)
-
-  val hiddenPages: Seq[Page] =
-    staticSite match
-      case None =>
-        Seq(navigablePage.copy( // Add index page that is a copy of api/index.html
-          link = navigablePage.link.copy(dri = docsRootDRI),
-          children = Nil
-        ))
-      case Some(siteContext) =>
-        // In case that we do not have an index page and we do not have any API entries
-        // we want to create empty index page, so there is one
-        val actualIndexTemplate = siteContext.indexTemplate() match {
-            case None if effectiveMembers.isEmpty => Seq(siteContext.emptyIndexTemplate)
-            case templates => templates.toSeq
-          }
+        val newRoot = newTemplates.head
 
-          (siteContext.orphanedTemplates ++ actualIndexTemplate).map(templateToPage(_, siteContext))
+        if newRoot.children.size == 0 && newRoot.templateFile.rawCode == ""
+        then rootPckPage.withTitle(args.name)
+        else {
+          val newRootPage = templateToPage(newRoot, siteContext)
+          newRootPage.withNewChildren(Seq(rootPckPage.withTitle("API")))
+        }
 
   val redirectPages: Seq[Page] = staticSite.fold(Seq.empty)(siteContext => siteContext.redirectTemplates.map {
     case (template, driFrom, driTo) =>
@@ -121,22 +96,10 @@ abstract class Renderer(rootPackage: Member, val members: Map[DRI, Member], prot
    * Here we have to retrive index pages from hidden pages and replace fake index pages in navigable page tree.
    */
   val allPages: Seq[Page] =
-    def traversePages(page: Page): (Page, Seq[Page]) =
-      val (newChildren, newPagesToRemove): (Seq[Page], Seq[Page]) = page.children.map(traversePages(_)).foldLeft((Seq[Page](), Seq[Page]())) {
-        case ((pAcc, ptrAcc), (p, ptr)) => (pAcc :+ p, ptrAcc ++ ptr)
-      }
-      hiddenPages.find(_.link == page.link) match
-        case None =>
-          (page.copy(children = newChildren), newPagesToRemove)
-        case Some(newPage) =>
-          (newPage.copy(children = newChildren), newPagesToRemove :+ newPage)
-
-    val (newNavigablePage, pagesToRemove) = traversePages(navigablePage)
-
-    val all = newNavigablePage +: (hiddenPages.filterNot(pagesToRemove.contains) ++ redirectPages)
-    // We need to check for conflicts only if we have top-level member called blog or docs
+    val all = navigablePage +: redirectPages
+    // We need to check for conflicts only if we have top-level member called docs
     val hasPotentialConflict =
-      rootPackage.members.exists(m => m.name.startsWith("docs") || m.name.startsWith("blog"))
+      rootPackage.members.exists(m => m.name.startsWith("docs"))
 
     if hasPotentialConflict then
       def walk(page: Page): Unit =

scaladoc/src/dotty/tools/scaladoc/site/SidebarParser.scala

@@ -8,29 +8,46 @@ import collection.JavaConverters._
 import java.util.Optional
 
 enum Sidebar:
-  case Category(title: Option[String], indexPath: Option[String], nested: List[Sidebar])
+  case Root(index: Option[String], pages: List[Sidebar])
+  case Category(title: Option[String], indexPath: Option[String], nested: List[Sidebar], directory: Option[String])
   case Page(title: Option[String], pagePath: String)
 
 object Sidebar:
-  case class RawInput(var title: String, var page: String, var index: String, var subsection: JList[RawInput]):
-    def this() = this("", "", "", JList())
+  case class RawRoot(var rootIndex: String, var pages: JList[RawInput]):
+    def this() = this("", JList())
+
+    def setRootIndex(s: String) = rootIndex = s
+    def setPages(l: JList[RawInput]) = pages = l
+
+  case class RawInput(var title: String, var page: String, var index: String, var subsection: JList[RawInput], var directory: String):
+    def this() = this("", "", "", JList(), "")
 
     def setTitle(t: String) = this.title = t
     def setPage(p: String) = this.page = p
     def setIndex(i: String) = this.index = i
     def setSubsection(s: JList[RawInput]) = this.subsection = s
+    def setDirectory(d: String) = this.directory = d
 
-  type RawInnerTpe = JMap[String, JList[RawInput]]
-  private object RawTypeRef extends TypeReference[RawInnerTpe]
+  private object RootTypeRef extends TypeReference[RawRoot]
 
   private def toSidebar(r: RawInput): Sidebar = r match
-    case RawInput(title, page, index, subsection) if page.nonEmpty && index.isEmpty && subsection.isEmpty() || title == "Blog" =>
+    case RawInput(title, page, index, subsection, dir) if page.nonEmpty && index.isEmpty && subsection.isEmpty() || title == "Blog" =>
       Sidebar.Page(Option.when(title.nonEmpty)(title), page)
-    case RawInput(title, page, index, subsection) if page.isEmpty && !subsection.isEmpty() =>
-      Sidebar.Category(Option.when(title.nonEmpty)(title), Option.when(index.nonEmpty)(index), subsection.asScala.map(toSidebar).toList)
+    case RawInput(title, page, index, subsection, dir) if page.isEmpty && (!subsection.isEmpty() || !index.isEmpty()) =>
+      Sidebar.Category(Option.when(title.nonEmpty)(title), Option.when(index.nonEmpty)(index), subsection.asScala.map(toSidebar).toList, Option.when(dir.nonEmpty)(dir))
+
+  def load(content: String): Sidebar.Root =
+    val mapper = ObjectMapper(YAMLFactory())
+    val root: RawRoot = mapper.readValue(content, RootTypeRef)
+
+    val rootIndex: String = root.rootIndex
+    val pages: List[Sidebar] = root.pages.asScala.toList.map(toSidebar)
+    Sidebar.Root(Option.when(rootIndex.nonEmpty)(rootIndex), pages)
 
-  def load(content: String): Seq[Sidebar] =
+  def load(file: java.io.File): Sidebar.Root =
     val mapper = ObjectMapper(YAMLFactory())
-    val raw: RawInnerTpe = mapper.readValue(content, RawTypeRef)
+    val root: RawRoot = mapper.readValue(file, RootTypeRef)
 
-    raw.get("sidebar").asScala.toList.map(toSidebar)
+    val rootIndex: String = root.rootIndex
+    val pages: List[Sidebar] = root.pages.asScala.toList.map(toSidebar)
+    Sidebar.Root(Option.when(rootIndex.nonEmpty)(rootIndex), pages)