Tools refactoring

This commit substantially refactors Scala.js' linker. Notably:

Linker pipeline
---------------
The linker pipeline is divided into a frontend and a backend.

The frontend links and optimizes IR files into a `LinkingUnit`. It paves the path for linker plugins in the future.

The backend emits a linking unit into a JavaScript file. Currently there are two backends:
* BasicLinkerBackend (basic JavaScript generation, aka fastOptJS)
* ClosureBackend (optimization pass with the Google Closure compiler, aka fullOptJS)

As a side effect of this change, Rhino now runs optimized code (unless this is explicitly disabled). This further has the side effect, that the `PreLinkStage` was entirely removed.

Classpath traversal
-------------------
Classpath traversal and abstraction is fully pushed to the build system. As a consequence, the whole `org.scalajs.core.tools.classpath` package is removed. Functionally is mostly replaced by a couple of lines of sbt code.

This also fixes scala-js/scala-js#1635.

IR caching
----------
In the past, each linker cached its own IR. This caused a lot of IR to be in memory multiple times. This commit replaces caching with a global IR cache. There are two main advantages:

* IR for each class is only cached once in memory.
* Linking other projects or configurations (e.g. compile v.s. test) can reuse the already cached IR.

JSEnvs
------
Since classpaths have been removed, we need a way to re-enable lazy loading in Rhino. We therefore introduce the `LinkingUnitJSEnv` that can directly accept a `LinkingUnit`. Further, we enrich the `JSEnv`s with an API to load libraries. This allows for much nicer interfaces in a couple of spots, where you essentially just pass a `JSEnv` and then run your code in it, with all dependencies already present (e.g. FrameworkDetector).

There are a couple of advantages:
- Symbol requirements are made explicit
- No duplicate code for linking in RhinoJSEnv
- Rhino can now run with optimized code (and does so by default)

Author: gzm0

js-envs/src/main/scala/org/scalajs/jsenv/AsyncJSEnv.scala

@@ -9,11 +9,22 @@
 
 package org.scalajs.jsenv
 
-import org.scalajs.core.tools.io._
-import org.scalajs.core.tools.classpath._
-import org.scalajs.core.tools.logging._
+import org.scalajs.core.tools.io.VirtualJSFile
+import org.scalajs.core.tools.jsdep.ResolvedJSDependency
 
 trait AsyncJSEnv extends JSEnv {
-  def asyncRunner(classpath: CompleteClasspath, code: VirtualJSFile,
-      logger: Logger, console: JSConsole): AsyncJSRunner
+  def asyncRunner(libs: Seq[ResolvedJSDependency], code: VirtualJSFile): AsyncJSRunner
+
+  final def asyncRunner(code: VirtualJSFile): AsyncJSRunner =
+    asyncRunner(Nil, code)
+
+  override def loadLibs(libs: Seq[ResolvedJSDependency]): AsyncJSEnv =
+    new AsyncLoadedLibs { val loadedLibs = libs }
+
+  private[jsenv] trait AsyncLoadedLibs extends LoadedLibs with AsyncJSEnv {
+    def asyncRunner(libs: Seq[ResolvedJSDependency],
+        code: VirtualJSFile): AsyncJSRunner = {
+      AsyncJSEnv.this.asyncRunner(loadedLibs ++ libs, code)
+    }
+  }
 }

js-envs/src/main/scala/org/scalajs/jsenv/AsyncJSRunner.scala

@@ -3,6 +3,8 @@ package org.scalajs.jsenv
 import scala.concurrent.{Future, Await}
 import scala.concurrent.duration.Duration
 
+import org.scalajs.core.tools.logging.Logger
+
 trait AsyncJSRunner {
 
   /** A future that completes when the associated run has terminated. */
@@ -13,7 +15,7 @@ trait AsyncJSRunner {
    *  when the run terminates. The returned Future is equivalent to
    *  the one returned by [[future]].
    */
-  def start(): Future[Unit]
+  def start(logger: Logger, console: JSConsole): Future[Unit]
 
   /** Aborts the associated run.
    *

js-envs/src/main/scala/org/scalajs/jsenv/ComJSEnv.scala

@@ -9,9 +9,8 @@
 
 package org.scalajs.jsenv
 
-import org.scalajs.core.tools.io._
-import org.scalajs.core.tools.classpath._
-import org.scalajs.core.tools.logging._
+import org.scalajs.core.tools.io.VirtualJSFile
+import org.scalajs.core.tools.jsdep.ResolvedJSDependency
 
 /** An [[AsyncJSEnv]] that provides communication to and from the JS VM.
  *
@@ -29,8 +28,19 @@ import org.scalajs.core.tools.logging._
  *  }}}
  */
 trait ComJSEnv extends AsyncJSEnv {
-  def comRunner(classpath: CompleteClasspath, code: VirtualJSFile,
-      logger: Logger, console: JSConsole): ComJSRunner
+  def comRunner(libs: Seq[ResolvedJSDependency], code: VirtualJSFile): ComJSRunner
+
+  final def comRunner(code: VirtualJSFile): ComJSRunner = comRunner(Nil, code)
+
+  override def loadLibs(libs: Seq[ResolvedJSDependency]): ComJSEnv =
+    new ComLoadedLibs { val loadedLibs = libs }
+
+  private[jsenv] trait ComLoadedLibs extends AsyncLoadedLibs with ComJSEnv {
+    def comRunner(libs: Seq[ResolvedJSDependency],
+        code: VirtualJSFile): ComJSRunner = {
+      ComJSEnv.this.comRunner(loadedLibs ++ libs, code)
+    }
+  }
 }
 
 object ComJSEnv {

js-envs/src/main/scala/org/scalajs/jsenv/ExternalJSEnv.scala

@@ -1,8 +1,8 @@
 package org.scalajs.jsenv
 
 import org.scalajs.core.tools.io._
-import org.scalajs.core.tools.classpath._
-import org.scalajs.core.tools.logging._
+import org.scalajs.core.tools.logging.Logger
+import org.scalajs.core.tools.jsdep.ResolvedJSDependency
 
 import java.io.{ Console => _, _ }
 import scala.io.Source
@@ -16,6 +16,8 @@ abstract class ExternalJSEnv(
 
   import ExternalJSEnv._
 
+  def name: String = s"ExternalJSEnv for $vmName"
+
   /** Printable name of this VM */
   protected def vmName: String
 
@@ -25,9 +27,21 @@ abstract class ExternalJSEnv(
   /** Custom initialization scripts. */
   protected def customInitFiles(): Seq[VirtualJSFile] = Nil
 
-  protected class AbstractExtRunner(protected val classpath: CompleteClasspath,
-      protected val code: VirtualJSFile, protected val logger: Logger,
-      protected val console: JSConsole) {
+  protected class AbstractExtRunner(
+      protected val libs: Seq[ResolvedJSDependency],
+      protected val code: VirtualJSFile) {
+
+    private[this] var _logger: Logger = _
+    private[this] var _console: JSConsole = _
+
+    protected def logger: Logger = _logger
+    protected def console: JSConsole = _console
+
+    protected def setupLoggerAndConsole(logger: Logger, console: JSConsole) = {
+      require(_logger == null && _console == null)
+      _logger = logger
+      _console = console
+    }
 
     /** JS files used to setup VM */
     protected def initFiles(): Seq[VirtualJSFile] = Nil
@@ -53,7 +67,7 @@ abstract class ExternalJSEnv(
 
     /** Get files that are a library (i.e. that do not run anything) */
     protected def getLibJSFiles(): Seq[VirtualJSFile] =
-      initFiles() ++ customInitFiles() ++ classpath.allCode
+      initFiles() ++ customInitFiles() ++ libs.map(_.lib)
 
     /** Get all files that are passed to VM (libraries and code) */
     protected def getJSFiles(): Seq[VirtualJSFile] =
@@ -133,23 +147,21 @@ abstract class ExternalJSEnv(
 
   }
 
-  protected class ExtRunner(classpath: CompleteClasspath, code: VirtualJSFile,
-      logger: Logger, console: JSConsole
-  ) extends AbstractExtRunner(classpath, code, logger, console)
-       with JSRunner {
+  protected class ExtRunner(libs: Seq[ResolvedJSDependency], code: VirtualJSFile)
+      extends AbstractExtRunner(libs, code) with JSRunner {
+
+    def run(logger: Logger, console: JSConsole): Unit = {
+      setupLoggerAndConsole(logger, console)
 
-    def run(): Unit = {
       val vmInst = startVM()
 
       pipeVMData(vmInst)
       waitForVM(vmInst)
     }
   }
 
-  protected class AsyncExtRunner(classpath: CompleteClasspath,
-      code: VirtualJSFile, logger: Logger, console: JSConsole
-  ) extends AbstractExtRunner(classpath, code, logger, console)
-       with AsyncJSRunner {
+  protected class AsyncExtRunner(libs: Seq[ResolvedJSDependency], code: VirtualJSFile)
+      extends AbstractExtRunner(libs, code) with AsyncJSRunner {
 
     private[this] var vmInst: Process = null
     private[this] var ioThreadEx: Throwable = null
@@ -173,11 +185,22 @@ abstract class ExternalJSEnv(
 
     def future: Future[Unit] = promise.future
 
-    def start(): Future[Unit] = {
+    def start(logger: Logger, console: JSConsole): Future[Unit] = {
+      setupLoggerAndConsole(logger, console)
+      startExternalJSEnv()
+      future
+    }
+
+    /** Core functionality of [[start]].
+     *
+     *  Same as [[start]] but without a call to [[setupLoggerAndConsole]] and
+     *  not returning [[future]].
+     *  Useful to be called in overrides of [[start]].
+     */
+    protected def startExternalJSEnv(): Unit = {
       require(vmInst == null, "start() may only be called once")
       vmInst = startVM()
       thread.start()
-      future
     }
 
     def stop(): Unit = {

js-envs/src/main/scala/org/scalajs/jsenv/JSEnv.scala

@@ -9,12 +9,42 @@
 
 package org.scalajs.jsenv
 
-import org.scalajs.core.tools.io._
-import org.scalajs.core.tools.classpath._
-import org.scalajs.core.tools.logging._
+import org.scalajs.core.tools.io.VirtualJSFile
+import org.scalajs.core.tools.jsdep.ResolvedJSDependency
 
 trait JSEnv {
+  /** Human-readable name for this [[JSEnv]] */
+  def name: String
+
   /** Prepare a runner for the code in the virtual file. */
-  def jsRunner(classpath: CompleteClasspath, code: VirtualJSFile,
-      logger: Logger, console: JSConsole): JSRunner
+  def jsRunner(libs: Seq[ResolvedJSDependency], code: VirtualJSFile): JSRunner
+
+  /** Prepare a runner without any libraries.
+   *
+   *  Strictly equivalent to:
+   *  {{{
+   *  this.jsRunner(Nil, code)
+   *  }}}
+   */
+  final def jsRunner(code: VirtualJSFile): JSRunner = jsRunner(Nil, code)
+
+  /** Return this [[JSEnv]] with the given libraries already loaded.
+   *
+   *  The following two are equivalent:
+   *  {{{
+   *  jsEnv.loadLibs(a).jsRunner(b, c)
+   *  jsEnv.jsRunner(a ++ b, c)
+   *  }}}
+   */
+  def loadLibs(libs: Seq[ResolvedJSDependency]): JSEnv =
+    new LoadedLibs { val loadedLibs = libs }
+
+  private[jsenv] trait LoadedLibs extends JSEnv {
+    val loadedLibs: Seq[ResolvedJSDependency]
+
+    def name: String = JSEnv.this.name
+
+    def jsRunner(libs: Seq[ResolvedJSDependency], code: VirtualJSFile): JSRunner =
+      JSEnv.this.jsRunner(loadedLibs ++ libs, code)
+  }
 }

js-envs/src/main/scala/org/scalajs/jsenv/JSRunner.scala

@@ -9,7 +9,9 @@
 
 package org.scalajs.jsenv
 
+import org.scalajs.core.tools.logging.Logger
+
 trait JSRunner {
   /** Run the associated JS code. Throw if an error occurs. */
-  def run(): Unit
+  def run(logger: Logger, console: JSConsole): Unit
 }

js-envs/src/main/scala/org/scalajs/jsenv/LinkingUnitAsyncJSEnv.scala

@@ -0,0 +1,42 @@
+/*                     __                                               *\
+**     ________ ___   / /  ___      __ ____  Scala.js sbt plugin        **
+**    / __/ __// _ | / /  / _ | __ / // __/  (c) 2013, LAMP/EPFL        **
+**  __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \    http://scala-js.org/       **
+** /____/\___/_/ |_/____/_/ | |__/ /____/                               **
+**                          |/____/                                     **
+\*                                                                      */
+
+
+package org.scalajs.jsenv
+
+import org.scalajs.core.tools.io.VirtualJSFile
+import org.scalajs.core.tools.jsdep.ResolvedJSDependency
+import org.scalajs.core.tools.linker.LinkingUnit
+
+trait LinkingUnitAsyncJSEnv extends LinkingUnitJSEnv with AsyncJSEnv {
+  def asyncRunner(preLibs: Seq[ResolvedJSDependency], linkingUnit: LinkingUnit,
+      postLibs: Seq[ResolvedJSDependency], code: VirtualJSFile): AsyncJSRunner
+
+  override def loadLibs(libs: Seq[ResolvedJSDependency]): LinkingUnitAsyncJSEnv =
+    new LinkingUnitAsyncLoadedLibs { val loadedLibs = libs }
+
+  override def loadLinkingUnit(linkingUnit: LinkingUnit): AsyncJSEnv =
+    new AsyncLoadedUnit { val loadedUnit = linkingUnit }
+
+  private[jsenv] trait LinkingUnitAsyncLoadedLibs extends LinkingUnitLoadedLibs
+      with AsyncLoadedLibs with LinkingUnitAsyncJSEnv {
+    def asyncRunner(preLibs: Seq[ResolvedJSDependency], linkingUnit: LinkingUnit,
+        postLibs: Seq[ResolvedJSDependency],
+        code: VirtualJSFile): AsyncJSRunner = {
+      LinkingUnitAsyncJSEnv.this.asyncRunner(loadedLibs ++ preLibs, linkingUnit,
+          postLibs, code)
+    }
+  }
+
+  private[jsenv] trait AsyncLoadedUnit extends LoadedUnit with AsyncJSEnv {
+    def asyncRunner(libs: Seq[ResolvedJSDependency],
+        code: VirtualJSFile): AsyncJSRunner = {
+      LinkingUnitAsyncJSEnv.this.asyncRunner(Nil, loadedUnit, libs, code)
+    }
+  }
+}

js-envs/src/main/scala/org/scalajs/jsenv/LinkingUnitComJSEnv.scala

@@ -0,0 +1,43 @@
+/*                     __                                               *\
+**     ________ ___   / /  ___      __ ____  Scala.js sbt plugin        **
+**    / __/ __// _ | / /  / _ | __ / // __/  (c) 2013, LAMP/EPFL        **
+**  __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \    http://scala-js.org/       **
+** /____/\___/_/ |_/____/_/ | |__/ /____/                               **
+**                          |/____/                                     **
+\*                                                                      */
+
+
+package org.scalajs.jsenv
+
+import org.scalajs.core.tools.io.VirtualJSFile
+import org.scalajs.core.tools.jsdep.ResolvedJSDependency
+import org.scalajs.core.tools.linker.LinkingUnit
+
+trait LinkingUnitComJSEnv extends LinkingUnitAsyncJSEnv with ComJSEnv {
+  def comRunner(preLibs: Seq[ResolvedJSDependency], linkingUnit: LinkingUnit,
+      postLibs: Seq[ResolvedJSDependency], code: VirtualJSFile): ComJSRunner
+
+  override def loadLibs(libs: Seq[ResolvedJSDependency]): LinkingUnitComJSEnv =
+    new LinkingUnitComLoadedLibs { val loadedLibs = libs }
+
+  override def loadLinkingUnit(linkingUnit: LinkingUnit): ComJSEnv =
+    new ComLoadedUnit { val loadedUnit = linkingUnit }
+
+  private[jsenv] trait LinkingUnitComLoadedLibs
+      extends LinkingUnitAsyncLoadedLibs with ComLoadedLibs
+      with LinkingUnitComJSEnv {
+    def comRunner(preLibs: Seq[ResolvedJSDependency], linkingUnit: LinkingUnit,
+        postLibs: Seq[ResolvedJSDependency],
+        code: VirtualJSFile): ComJSRunner = {
+      LinkingUnitComJSEnv.this.comRunner(loadedLibs ++ preLibs, linkingUnit,
+          postLibs, code)
+    }
+  }
+
+  private[jsenv] trait ComLoadedUnit extends AsyncLoadedUnit with ComJSEnv {
+    def comRunner(libs: Seq[ResolvedJSDependency],
+        code: VirtualJSFile): ComJSRunner = {
+      LinkingUnitComJSEnv.this.comRunner(Nil, loadedUnit, libs, code)
+    }
+  }
+}