Add preprocessing of markdown files for docs.scala-lang

Author: BarkingBad

.gitignore

@@ -105,3 +105,7 @@ compiler/test-coursier/run/*.jar
 
 # Docs
 docs-for-dotty-page/*
+
+# docs.scala-lang deplyment temp dir
+docsScalaLang/
+docs/_site/

docs/docsScalaLangResources/scaladoc-assets.html

@@ -0,0 +1,5 @@
+<!--- This file is copied from dotty repo located at the github lampepefl/dotty inside docs/docsScalaLangResources/scaladoc-assets.html -->
+<script src="{{ site.baseurl }}/scripts/scaladoc-scalajs.js" type="text/javascript"></script>
+<link rel="stylesheet" href="{{ site.baseurl }}/resources/css/colors.css" type="text/css" />
+<link rel="stylesheet" href="{{ site.baseurl }}/resources/css/code-snippets.css" type="text/css" />
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.9.0/css/all.min.css">

project/Build.scala

@@ -24,6 +24,7 @@ import sbtbuildinfo.BuildInfoPlugin.autoImport._
 
 import scala.util.Properties.isJavaAtLeast
 import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._
+import org.scalajs.linker.interface.ModuleInitializer
 
 object DottyJSPlugin extends AutoPlugin {
   import Build._
@@ -1246,6 +1247,7 @@ object Build {
   // Note: the two tasks below should be one, but a bug in Tasty prevents that
   val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
+  val renderScaladocScalajsToFile = inputKey[Unit]("Copy the output of the scaladoc js files")
 
   lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")).
     dependsOn(`scala3-compiler-bootstrapped`).
@@ -1254,8 +1256,12 @@ object Build {
     enablePlugins(DottyJSPlugin).
     dependsOn(`scala3-library-bootstrappedJS`).
     settings(
+      Compile / scalaJSMainModuleInitializer := (sys.env.get("scaladoc.projectFormat") match {
+        case Some("md") => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "markdownMain"))
+        case _ => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "main"))
+      }),
       Test / fork := false,
-      scalaJSUseMainModuleInitializer := true,
+      Compile / scalaJSUseMainModuleInitializer := true,
       libraryDependencies += ("org.scala-js" %%% "scalajs-dom" % "1.1.0").cross(CrossVersion.for3Use2_13)
     )
 
@@ -1309,7 +1315,7 @@ object Build {
     ).
     settings(
       Compile / resourceGenerators += Def.task {
-        val jsDestinationFile = (Compile / resourceManaged).value / "dotty_res" / "scripts" / "searchbar.js"
+        val jsDestinationFile = (Compile / resourceManaged).value / "dotty_res" / "scripts" / "scaladoc-scalajs.js"
         sbt.IO.copyFile((`scaladoc-js` / Compile / fullOptJS).value.data, jsDestinationFile)
         Seq(jsDestinationFile)
       }.taskValue,
@@ -1439,6 +1445,23 @@ object Build {
         )
       }.value,
 
+      renderScaladocScalajsToFile := Def.inputTask {
+        val extraArgs = spaceDelimited("<arg>").parsed
+        val (destJS, destCSS, csses) = extraArgs match {
+          case js :: css :: tail => (js, css, tail)
+          case js :: Nil => (js, "", Nil)
+          case _ => throw new IllegalArgumentException("No js destination provided")
+        }
+        val jsDestinationFile: File = Paths.get(destJS).toFile
+        sbt.IO.copyFile((`scaladoc-js` / Compile / fullOptJS).value.data, jsDestinationFile)
+        csses.map { file =>
+          val cssDesitnationFile = Paths.get(destCSS).toFile / file
+          val cssSourceFile = (`scaladoc-js` / Compile / resourceDirectory).value / file
+          sbt.IO.copyFile(cssSourceFile, cssDesitnationFile)
+          cssDesitnationFile
+        }
+      }.evaluated,
+
       Test / buildInfoKeys := Seq[BuildInfoKey](
         (Test / Build.testcasesOutputDir),
         (Test / Build.testcasesSourceRoot),

project/scripts/genDocsScalaLang

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -e
+shopt -s extglob # needed for rm everything but x
+echo "Working directory: $PWD"
+
+GENDOC_EXTRA_ARGS=$@
+GIT_HEAD=$(git rev-parse HEAD) # save current head for commit message in gh-pages
+PREVIOUS_SNAPSHOTS_DIR="$PWD/../prev_snapshots"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)"
+SITE_OUT_DIR="$PWD/docs/_site"
+
+DOCS_SCALA_LANG_DIR="$PWD/docsScalaLang"
+
+rm -rf $DOCS_SCALA_LANG_DIR
+mkdir -pv $DOCS_SCALA_LANG_DIR
+git clone "https://github.com/scala/docs.scala-lang.git" $DOCS_SCALA_LANG_DIR
+
+SBT="$SCRIPT_DIR/sbt"
+mkdir -pv $SITE_OUT_DIR
+env "scaladoc.projectFormat=md" "$SBT" "scaladoc/renderScaladocScalajsToFile $DOCS_SCALA_LANG_DIR/scripts/scaladoc-scalajs.js $DOCS_SCALA_LANG_DIR/resources/css code-snippets.css"
+"dist/target/pack/bin/scaladoc" "-d" "$SITE_OUT_DIR" "-format" "md" "-siteroot" "docs" "/dev/null"
+
+if [ ! -d "$SITE_OUT_DIR" ]; then
+  echo "Output directory did not exist: $SITE_OUT_DIR" 1>&2
+  exit 1
+fi
+
+# Copy reference and scaladoc docs
+cp -rf "$SITE_OUT_DIR/docs/reference"/* "$DOCS_SCALA_LANG_DIR/_scala3-reference"
+cp -rf "$SITE_OUT_DIR/docs/usage/scaladoc"/* "$DOCS_SCALA_LANG_DIR/_overviews/scala3-scaladoc"
+
+cp -rf "$SITE_OUT_DIR/docs/reference/contextual/motivation.md" "$DOCS_SCALA_LANG_DIR/_scala3-reference/contextual.md"
+cp -rf "$SITE_OUT_DIR/docs/reference/metaprogramming/toc.md" "$DOCS_SCALA_LANG_DIR/_scala3-reference/metaprogramming.md"
+cp -rf "$SITE_OUT_DIR/docs/resources/talks.md" "$DOCS_SCALA_LANG_DIR/scala3/talks.md"
+cp -rf "$SITE_OUT_DIR/docs/usage/getting-started.md" "$DOCS_SCALA_LANG_DIR/scala3/getting-started.md"
+cp -rf "$SITE_OUT_DIR/docs/usage/language-versions.md" "$DOCS_SCALA_LANG_DIR/_scala3-reference/language-versions.md"
+cp -rf "$SITE_OUT_DIR/docs/usage/worksheet-mode.md" "$DOCS_SCALA_LANG_DIR/_overviews/scala3-book/tools-worksheets.md"
+
+
+# Copy csses and html importing these assets
+cp -f "$SITE_OUT_DIR/styles/colors.css" "$DOCS_SCALA_LANG_DIR/resources/css/colors.css"
+cp -f "$PWD/docs/docsScalaLangResources/scaladoc-assets.html" "$DOCS_SCALA_LANG_DIR/_includes/scaladoc-assets.html"
+
+# Hack inclusion of these assests by the docs.scala-lang jekyll builder
+echo "{% include scaladoc-assets.html %}" >> "$DOCS_SCALA_LANG_DIR/_layouts/inner-page-parent-dropdown.html"

scaladoc-js/src/Main.scala

@@ -1,9 +1,22 @@
 package dotty.tools.scaladoc
 
-object Main extends App {
-  Searchbar()
-  SocialLinks()
-  CodeSnippets()
-  DropdownHandler()
-  Ux()
-}
+object Main:
+
+  private def common(): Unit =
+    CodeSnippets()
+
+  def main(): Unit =
+    Searchbar()
+    SocialLinks()
+    DropdownHandler()
+    Ux()
+    common()
+
+  /**
+   * This main is conditionally enabled by system env variable `scaladoc.projectFormat=md`
+   * passed in ./projects/scripts/genDocsScalaLang
+   * The reason why we have to pass the condition by env variable is because js is build before scaladoc,
+   * so we cannot access its args
+   */
+  def markdownMain(): Unit =
+    common()

scaladoc/src/dotty/tools/scaladoc/Scaladoc.scala

@@ -59,7 +59,8 @@ object Scaladoc:
     versionsDictionaryUrl: Option[String] = None,
     generateInkuire : Boolean = false,
     apiSubdirectory : Boolean = false,
-    scastieConfiguration: String = ""
+    scastieConfiguration: String = "",
+    projectFormat: String = "html",
   )
 
   def run(args: Array[String], rootContext: CompilerContext): Reporter =
@@ -236,7 +237,8 @@ object Scaladoc:
         versionsDictionaryUrl.nonDefault,
         generateInkuire.get,
         apiSubdirectory.get,
-        scastieConfiguration.get
+        scastieConfiguration.get,
+        projectFormat.get,
       )
       (Some(docArgs), newContext)
     }
@@ -245,6 +247,10 @@ object Scaladoc:
     given docContext: DocContext = new DocContext(args, ctx)
     val module = ScalaModuleProvider.mkModule()
 
-    new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members).render()
+    val renderer = args.projectFormat match
+      case "html" => new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members)
+      case "md" => new dotty.tools.scaladoc.renderers.MarkdownRenderer(module.rootPackage, module.members)
+
+    renderer.render()
     report.inform("generation completed successfully")
     docContext

scaladoc/src/dotty/tools/scaladoc/ScaladocSettings.scala

@@ -128,6 +128,16 @@ class ScaladocSettings extends SettingGroup with AllScalaSettings:
 
   val scastieConfiguration: Setting[String] =
     StringSetting("-scastie-configuration", "Scastie configuration", "Additional configuration passed to Scastie in code snippets", "")
+    
+  val projectFormat: Setting[String] =
+    ChoiceSetting(
+      "-format",
+      "format of the static site output",
+      "Format of the static site output. The default value is html, which converts all static articles into a webpage. " +
+        "The md format only preprocess markdown files and should not be used as a direct output, but rather as a sources generator for an outer templating engine like Jekyll",
+      List("html", "md"),
+      "html"
+    )
 
   def scaladocSpecificSettings: Set[Setting[_]] =
     Set(sourceLinks, legacySourceLink, syntax, revision, externalDocumentationMappings, socialLinks, skipById, skipByRegex, deprecatedSkipPackages, docRootContent, snippetCompiler, generateInkuire, scastieConfiguration)

scaladoc/src/dotty/tools/scaladoc/renderers/HtmlRenderer.scala

@@ -14,120 +14,21 @@ import java.nio.file.Files
 import java.nio.file.FileVisitOption
 import java.io.File
 
-case class Page(link: Link, content: Member | ResolvedTemplate | String, children: Seq[Page]):
-  def withNewChildren(newChildren: Seq[Page]) = copy(children = children ++ newChildren)
+class HtmlRenderer(rootPackage: Member, members: Map[DRI, Member])(using ctx: DocContext)
+  extends Renderer(rootPackage, members, extension = "html"):
 
-  def withTitle(newTitle: String) = copy(link = link.copy(name = newTitle))
-
-  def hasFrame = content match
-    case t: ResolvedTemplate => t.hasFrame
-    case _ => true
-
-class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx: DocContext)
-  extends SiteRenderer, Resources, Locations, Writer:
-  private val args = summon[DocContext].args
-  val staticSite = summon[DocContext].staticSiteContext
-
-  val effectiveMembers = members
-
-  private def memberPage(member: Member): Page =
-    val childrenPages = member.members.filter(_.needsOwnPage)
-    Page(Link(member.name, member.dri), member, childrenPages.map(memberPage))
-
-  val navigablePage: Page =
-    val rootPckPage = memberPage(rootPackage)
-    staticSite match
-      case None => rootPckPage.withTitle(args.name)
-      case Some(siteContext) =>
-        val (indexes, templates) = siteContext.templates.partition(f =>
-          f.templateFile.isIndexPage() && f.file.toPath.getParent() == siteContext.docsPath)
-        if (indexes.size > 1)
-          val msg = s"ERROR: Multiple index pages for doc found ${indexes.map(_.file)}"
-          report.error(msg)
-
-        val templatePages = templates.map(templateToPage(_, siteContext))
-
-        indexes.headOption match
-          case None if templatePages.isEmpty=>
-            rootPckPage.withTitle(args.name)
-          case None =>
-            Page(Link(args.name, docsRootDRI),"", templatePages :+ rootPckPage.withTitle("API"))
-          case Some(indexPage) =>
-            val newChildren = templatePages :+ rootPckPage.withTitle("API")
-            templateToPage(indexPage, siteContext).withNewChildren(newChildren)
-
-  val hiddenPages: Seq[Page] =
-    staticSite match
-      case None =>
-        Seq(navigablePage.copy( // Add index page that is a copy of api/index.html
-          link = navigablePage.link.copy(dri = docsRootDRI),
-          children = Nil
-        ))
-      case Some(siteContext) =>
-        // In case that we do not have an index page and we do not have any API entries
-        // we want to create empty index page, so there is one
-        val actualIndexTemplate = siteContext.indexTemplate() match {
-            case None if effectiveMembers.isEmpty => Seq(siteContext.emptyIndexTemplate)
-            case templates => templates.toSeq
-          }
-
-          (siteContext.orphanedTemplates ++ actualIndexTemplate).map(templateToPage(_, siteContext))
-
-  /**
-   * Here we have to retrive index pages from hidden pages and replace fake index pages in navigable page tree.
-   */
-  val allPages: Seq[Page] =
-    def traversePages(page: Page): (Page, Seq[Page]) =
-      val (newChildren, newPagesToRemove): (Seq[Page], Seq[Page]) = page.children.map(traversePages(_)).foldLeft((Seq[Page](), Seq[Page]())) {
-        case ((pAcc, ptrAcc), (p, ptr)) => (pAcc :+ p, ptrAcc ++ ptr)
-      }
-      hiddenPages.find(_.link == page.link) match
-        case None =>
-          (page.copy(children = newChildren), newPagesToRemove)
-        case Some(newPage) =>
-          (newPage.copy(children = newChildren), newPagesToRemove :+ newPage)
-
-    val (newNavigablePage, pagesToRemove) = traversePages(navigablePage)
-
-    val all = newNavigablePage +: hiddenPages.filterNot(pagesToRemove.contains)
-    // We need to check for conflicts only if we have top-level member called blog or docs
-    val hasPotentialConflict =
-      rootPackage.members.exists(m => m.name.startsWith("docs") || m.name.startsWith("blog"))
-
-    if hasPotentialConflict then
-      def walk(page: Page): Unit =
-        if page.link.dri.isStaticFile then
-          val dest = absolutePath(page.link.dri)
-          if apiPaths.contains(dest) then
-            report.error(s"Conflict between static page and API member for $dest. $pathsConflictResoultionMsg")
-          page.children.foreach(walk)
-
-      all.foreach (walk)
-
-    all
-
-  def renderContent(page: Page) = page.content match
-    case m: Member =>
-      val signatureRenderer = new SignatureRenderer:
-        def currentDri: DRI = page.link.dri
-        def link(dri: DRI): Option[String] =
-          Some(pathToPage(currentDri, dri)).filter(_ != UnresolvedLocationLink)
-
-      MemberRenderer(signatureRenderer).fullMember(m)
-    case t: ResolvedTemplate => siteContent(page.link.dri, t)
-    case a: String =>  raw(a)
-
-
-  def renderPage(page: Page, parents: Vector[Link]): Seq[String] =
-    val newParents = parents :+ page.link
-    val content = html(
+  override def pageContent(page: Page, parents: Vector[Link]): AppliedTag =
+    html(
       mkHead(page),
       body(
         if !page.hasFrame then renderContent(page)
-        else mkFrame(page.link, newParents, renderContent(page))
+        else mkFrame(page.link, parents, renderContent(page))
       )
     )
-    write(page.link.dri, content) +: page.children.flatMap(renderPage(_, newParents))
+
+  override def render(): Unit =
+    val renderedResources = renderResources()
+    super.render()
 
   private def specificResources(page: Page): Set[String] =
     page.children.toSet.flatMap(specificResources) ++ (page.content match
@@ -157,10 +58,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
     val resources = siteResourcesPaths.toSeq.map(pathToResource) ++ allResources(allPages) ++ onlyRenderedResources
     resources.flatMap(renderResource)
 
-  def render(): Unit =
-    val renderedResources = renderResources()
-    val sites = allPages.map(renderPage(_, Vector.empty))
-
   def mkHead(page: Page): AppliedTag =
     val resources = page.content match
       case t: ResolvedTemplate =>
@@ -229,14 +126,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
 
     renderNested(navigablePage, toplevel = true)._2
 
-  private def canonicalUrl(l: String): AppliedTag | String =
-    val canon = args.docCanonicalBaseUrl
-    if !canon.isEmpty then
-      val canonicalUrl = if canon.endsWith("/") then canon else canon + "/"
-      link(rel := "canonical", href := canonicalUrl + l)
-    else
-      "" // return empty tag
-
   private def hasSocialLinks = !args.socialLinks.isEmpty
 
   private def socialLinks(whiteIcon: Boolean = true) =

scaladoc/src/dotty/tools/scaladoc/renderers/Locations.scala

@@ -21,7 +21,7 @@ trait Locations(using ctx: DocContext):
 
   // We generate this collection only if there may be a conflict with resources.
   // Potentially can be quite big.
-  lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath)
+  lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath(_))
 
   var cache = new JHashMap[DRI, Seq[String]]()
 
@@ -80,7 +80,7 @@ trait Locations(using ctx: DocContext):
     pathToRaw(from, to.split("/").toList)
 
   def resolveRoot(dri: DRI, path: String): String = resolveRoot(rawLocation(dri), path)
-  def absolutePath(dri: DRI): String = rawLocation(dri).mkString("", "/", ".html")
+  def absolutePath(dri: DRI, extension: String = "html"): String = rawLocation(dri).mkString("", "/", s".$extension")
 
   def resolveLink(dri: DRI, url: String): String =
     if URI(url).isAbsolute then url else resolveRoot(dri, url)