Merge pull request #8108 from dotty-staging/doc-snapshotting

Add documentation snapshotting capability

Author: anatoliykmetyuk

.drone.yml

@@ -124,6 +124,27 @@ steps:
     target:
     - nightly
 
+- name: nightly_documentation
+  pull: default
+  image: lampepfl/dotty:2020-01-22-2
+  depends_on:
+  - test
+  - test_bootstrapped
+  - community_build
+  - test_java11
+  - publish_nightly
+  commands:
+  - ./project/scripts/genDocs -doc-snapshot
+  environment:
+    BOT_TOKEN:
+      from_secret: bot_token
+    NIGHTLYBUILD: yes
+  when:
+    event:
+    - promote
+    target:
+    - nightly
+
 - name: publish_release
   pull: default
   image: lampepfl/dotty:2020-01-22-2
@@ -150,6 +171,26 @@ steps:
     event:
     - tag
 
+- name: release_documentation
+  pull: default
+  image: lampepfl/dotty:2020-01-22-2
+  depends_on:
+  - test
+  - test_bootstrapped
+  - community_build
+  - test_java11
+  - publish_release
+  - github_release
+  commands:
+  - ./project/scripts/genDocs -doc-snapshot
+  environment:
+    BOT_TOKEN:
+      from_secret: bot_token
+    RELEASEBUILD: yes
+  when:
+    event:
+    - tag
+
 - name: github_release
   pull: default
   image: plugins/github-release

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -215,5 +215,7 @@ class ScalaSettings extends Settings.SettingGroup {
     ""
   )
 
+  val docSnapshot: Setting[Boolean] = BooleanSetting("-doc-snapshot", "Generate a documentation snapshot for the current Dotty version")
+
   val wikiSyntax: Setting[Boolean] = BooleanSetting("-Xwiki-syntax", "Retains the Scala2 behavior of using Wiki Syntax in Scaladoc.")
 }

doc-tool/src/dotty/tools/dottydoc/DocDriver.scala

@@ -1,6 +1,8 @@
 package dotty.tools
 package dottydoc
 
+import java.io.File
+
 import dotty.tools.dottydoc.util.syntax._
 import core.ContextDottydoc
 import dotc.core.Contexts._
@@ -38,18 +40,26 @@ class DocDriver extends Driver {
     implicit val ctx: Context = ictx
     val reporter = doCompile(newCompiler, filesToDocument)
 
-    val siteRoot = new java.io.File(ctx.settings.siteRoot.value)
+    val siteRoot = File(ctx.settings.siteRoot.value)
     val projectName = ctx.settings.projectName.value
     val projectVersion = ctx.settings.projectVersion.value
     val projectUrl = Option(ctx.settings.projectUrl.value).filter(_.nonEmpty)
     val projectLogo = Option(ctx.settings.projectLogo.value).filter(_.nonEmpty)
+    val docSnapshot = ctx.settings.docSnapshot.value
+    var baseUrl = "/"
+
+    var outDir = File(siteRoot, "_site")
+    if docSnapshot then
+      val folderName = if projectVersion.endsWith("NIGHTLY") then "nightly" else projectVersion
+      outDir = File(outDir, folderName)
+      baseUrl = s"$baseUrl$folderName"
 
     if (projectName.isEmpty)
       ctx.error(s"Site project name not set. Use `-project <title>` to set the project name")
     else if (!siteRoot.exists || !siteRoot.isDirectory)
       ctx.error(s"Site root does not exist: $siteRoot")
     else {
-      Site(siteRoot, projectName, projectVersion, projectUrl, projectLogo, ctx.docbase.packages)
+      Site(siteRoot, outDir, projectName, projectVersion, projectUrl, projectLogo, ctx.docbase.packages, baseUrl)
         .generateApiDocs()
         .copyStaticFiles()
         .generateHtmlFiles()

doc-tool/src/dotty/tools/dottydoc/staticsite/Site.scala

@@ -31,11 +31,13 @@ import util.syntax._
 
 case class Site(
   root: JFile,
+  outDir: JFile,
   projectTitle: String,
   projectVersion: String,
   projectUrl: Option[String],
   projectLogo: Option[String],
-  documentation: Map[String, Package]
+  documentation: Map[String, Package],
+  baseUrl: String
 ) extends ResourceFinder {
   /** Documentation serialized to java maps */
   private val docs: JList[_] = {
@@ -101,7 +103,7 @@ case class Site(
         .flatMap { file =>
           val BlogPost.extract(year, month, day, name, ext) = file.getName
           val sourceFile = toSourceFile(file)
-          val params = defaultParams(file, 2).withUrl(s"/blog/$year/$month/$day/$name.html").toMap
+          val params = defaultParams(file).withUrl(s"/blog/$year/$month/$day/$name.html").toMap
           val page =
             if (ext == "md")
               new MarkdownPage(file.getPath, sourceFile, params, includes, documentation)
@@ -128,11 +130,8 @@ case class Site(
     new SourceFile(virtualFile, Codec.UTF8)
   }
 
-  /** Copy static files to `outDir` */
-  private[this] val defaultOutDir = new JFile(root.getAbsolutePath + JFile.separator + "_site")
-
-  def copyStaticFiles(outDir: JFile = defaultOutDir)(implicit ctx: Context): this.type =
-    createOutput (outDir) {
+  def copyStaticFiles()(implicit ctx: Context): this.type =
+    createOutput {
       // Copy user-defined static assets
       staticAssets.foreach { asset =>
         val target = mkdirs(fs.getPath(outDir.getAbsolutePath, stripRoot(asset)))
@@ -169,15 +168,8 @@ case class Site(
     }
 
   /** Generate default params included in each page */
-  private def defaultParams(pageLocation: JFile, additionalDepth: Int = 0): DefaultParams = {
+  private def defaultParams(pageLocation: JFile): DefaultParams =
     val pathFromRoot = stripRoot(pageLocation)
-    val baseUrl: String = {
-      val rootLen = root.toPath.toAbsolutePath.normalize.getNameCount
-      val assetLen = pageLocation.toPath.toAbsolutePath.normalize.getNameCount
-      "../" * (assetLen - rootLen - 1 + additionalDepth) + "."
-      // -1 because for root/index.html the root is the current directory (.) not its parent (../.)
-    }
-
     DefaultParams(
       docs, docsFlattened, documentation, PageInfo(pathFromRoot),
       SiteInfo(
@@ -186,19 +178,18 @@ case class Site(
       ),
       sidebar
     )
-  }
 
   /* Creates output directories if allowed */
-  private def createOutput(outDir: JFile)(op: => Unit)(implicit ctx: Context): this.type = {
+  private def createOutput(op: => Unit)(implicit ctx: Context): this.type = {
     if (!outDir.isDirectory) outDir.mkdirs()
     if (!outDir.isDirectory) ctx.docbase.error(s"couldn't create output folder: $outDir")
     else op
     this
   }
 
   /** Generate HTML for the API documentation */
-  def generateApiDocs(outDir: JFile = defaultOutDir)(implicit ctx: Context): this.type =
-    createOutput(outDir) {
+  def generateApiDocs()(implicit ctx: Context): this.type =
+    createOutput {
       def genDoc(e: model.Entity): Unit = {
         ctx.docbase.echo(s"Generating doc page for: ${e.path.mkString(".")}")
         // Suffix is index.html for packages and therefore the additional depth
@@ -212,7 +203,7 @@ case class Site(
         else
           e.path
         val target = mkdirs(fs.getPath(outDir.getAbsolutePath +  sep + "api" + sep + path.mkString(sep) + suffix))
-        val params = defaultParams(target.toFile, -1).withPosts(blogInfo).withEntity(Some(e)).toMap
+        val params = defaultParams(target.toFile).withPosts(blogInfo).withEntity(Some(e)).toMap
         val page = new HtmlPage("_layouts" + sep + "api-page.html", layouts("api-page").content, params, includes)
 
         render(page).foreach { rendered =>
@@ -231,7 +222,7 @@ case class Site(
 
       // generate search page:
       val target = mkdirs(fs.getPath(outDir.getAbsolutePath + sep + "api" + sep + "search.html"))
-      val searchPageParams = defaultParams(target.toFile, -1).withPosts(blogInfo).toMap
+      val searchPageParams = defaultParams(target.toFile).withPosts(blogInfo).toMap
       val searchPage = new HtmlPage("_layouts" + sep + "search.html", layouts("search").content, searchPageParams, includes)
       render(searchPage).foreach { rendered =>
         Files.copy(
@@ -243,8 +234,8 @@ case class Site(
     }
 
   /** Generate HTML files from markdown and .html sources */
-  def generateHtmlFiles(outDir: JFile = defaultOutDir)(implicit ctx: Context): this.type =
-    createOutput(outDir) {
+  def generateHtmlFiles()(implicit ctx: Context): this.type =
+    createOutput {
       compilableFiles.foreach { asset =>
         val pathFromRoot = stripRoot(asset)
         val sourceFile = toSourceFile(asset)
@@ -263,13 +254,13 @@ case class Site(
     }
 
   /** Generate blog from files in `blog/_posts` and output in `outDir` */
-  def generateBlog(outDir: JFile = defaultOutDir)(implicit ctx: Context): this.type =
-    createOutput(outDir) {
+  def generateBlog()(implicit ctx: Context): this.type =
+    createOutput {
       blogposts.foreach { file =>
         val BlogPost.extract(year, month, day, name, ext) = file.getName
         val sourceFile = toSourceFile(file)
         val date = s"$year-$month-$day 00:00:00"
-        val params = defaultParams(file, 2).withPosts(blogInfo).withDate(date).toMap
+        val params = defaultParams(file).withPosts(blogInfo).withDate(date).toMap
 
         // Output target
         val target = mkdirs(fs.getPath(outDir.getAbsolutePath, "blog", year, month, day, name + ".html"))

doc-tool/test/dotty/tools/dottydoc/staticsite/SourceFileOps.scala

@@ -11,9 +11,10 @@ import model.Package
 
 trait SourceFileOps {
   import scala.collection.JavaConverters._
+  val siteRoot = new java.io.File("doc-tool/resources/")
   val site = new Site(
-    new java.io.File("doc-tool/resources/"),
-    "test-site", "v0.1", Some("http://github.com/lampepfl/dotty"), None, Map.empty
+    siteRoot, new java.io.File(siteRoot, "_site"),
+    "test-site", "v0.1", Some("http://github.com/lampepfl/dotty"), None, Map.empty, "/"
   )
 
   def stringToSource(path: String, sourceCode: String): SourceFile = {

project/Build.scala

@@ -130,7 +130,7 @@ object Build {
   val dotr = inputKey[Unit]("run compiled binary using the correct classpath, or the user supplied classpath")
 
   // Compiles the documentation and static site
-  val genDocs = taskKey[Unit]("run dottydoc to generate static documentation site")
+  val genDocs = inputKey[Unit]("run dottydoc to generate static documentation site")
 
   // Shorthand for compiling a docs site
   val dottydoc = inputKey[Unit]("run dottydoc")
@@ -409,7 +409,9 @@ object Build {
 
     javaOptions ++= (javaOptions in `dotty-compiler`).value,
 
-    genDocs := Def.taskDyn {
+    genDocs := Def.inputTaskDyn {
+      val dottydocExtraArgs = spaceDelimited("<arg>").parsed
+
       // Make majorVersion available at dotty.epfl.ch/versions/latest-nightly-base
       // Used by sbt-dotty to resolve the latest nightly
       val majorVersion = baseVersion.take(baseVersion.lastIndexOf('.'))
@@ -427,11 +429,11 @@ object Build {
         "-project-logo", "dotty-logo.svg",
         "-classpath", dottydocClasspath.value,
         "-Yerased-terms"
-      )
+      ) ++ dottydocExtraArgs
       (runMain in Compile).toTask(
         s""" dotty.tools.dottydoc.Main ${args.mkString(" ")} ${sources.mkString(" ")}"""
       )
-    }.value,
+    }.evaluated,
 
     dottydoc := Def.inputTaskDyn {
       val args = spaceDelimited("<arg>").parsed

project/scripts/genDocs

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 
 # Usage:
-# BOT_TOKEN=<dotty-bot password> ./genDocs
+# BOT_TOKEN=<dotty-bot password> ./genDocs [-doc-snapshot]
 
 set -e
+GENDOC_EXTRA_ARGS=$@
 
 # set extended glob, needed for rm everything but x
 shopt -s extglob
@@ -18,7 +19,7 @@ echo "Working directory: $PWD"
 
 # this command will generate docs in $PWD/docs/_site
 SBT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/sbt"
-"$SBT" "genDocs"
+"$SBT" "genDocs $GENDOC_EXTRA_ARGS"
 
 # make sure that the previous command actually succeeded
 if [ ! -d "$PWD/docs/_site" ]; then
@@ -39,6 +40,9 @@ git config user.email "dotty-bot@d-d.me"
 git fetch doc-remote gh-pages
 git checkout gh-pages
 
+# save the old snapshots to the newly generated site
+mv 0.*/ $PWD/docs/_site
+
 # move newly generated _site dir to $PWD
 mv $PWD/docs/_site .