chore: extract repl to be its own artifact

Author: hamzaremmal

build.sbt

@@ -7,6 +7,7 @@ val `scala3-compiler` = Build.`scala3-compiler`
 val `scala3-compiler-nonbootstrapped` = Build.`scala3-compiler-nonbootstrapped`
 val `scala3-compiler-bootstrapped-new` = Build.`scala3-compiler-bootstrapped-new`
 val `scala3-compiler-bootstrapped` = Build.`scala3-compiler-bootstrapped`
+val `scala3-repl` = Build.`scala3-repl`
 val `scala-library-sjs` = Build.`scala-library-sjs`
 val `scala3-library-sjs` = Build.`scala3-library-sjs`
 val `scala-library-nonbootstrapped` = Build.`scala-library-nonbootstrapped`

compiler/src/dotty/tools/dotc/quoted/Interpreter.scala

@@ -25,7 +25,7 @@ import dotty.tools.dotc.quoted.*
 import dotty.tools.dotc.typer.ImportInfo.withRootImports
 import dotty.tools.dotc.util.SrcPos
 import dotty.tools.dotc.reporting.Message
-import dotty.tools.repl.AbstractFileClassLoader
+import dotty.tools.io.AbstractFileClassLoader
 import dotty.tools.dotc.core.CyclicReference
 
 /** Tree interpreter for metaprogramming constructs */
@@ -35,7 +35,7 @@ class Interpreter(pos: SrcPos, classLoader0: ClassLoader)(using Context):
 
   val classLoader =
     if ctx.owner.topLevelClass.name.startsWith(str.REPL_SESSION_LINE) then
-        new AbstractFileClassLoader(ctx.settings.outputDir.value, classLoader0, "false")
+        new AbstractFileClassLoader(ctx.settings.outputDir.value, classLoader0)
     else classLoader0
 
   /** Local variable environment */
@@ -204,11 +204,7 @@ class Interpreter(pos: SrcPos, classLoader0: ClassLoader)(using Context):
     }
 
   private def loadReplLineClass(moduleClass: Symbol): Class[?] = {
-    val lineClassloader = new AbstractFileClassLoader(
-      ctx.settings.outputDir.value,
-      classLoader,
-      "false"
-    )
+    val lineClassloader = new AbstractFileClassLoader(ctx.settings.outputDir.value, classLoader)
     lineClassloader.loadClass(moduleClass.name.firstPart.toString)
   }
 

compiler/src/dotty/tools/dotc/transform/Splicer.scala

@@ -22,7 +22,7 @@ import dotty.tools.dotc.quoted.Interpreter
 
 import scala.util.control.NonFatal
 import dotty.tools.dotc.util.SrcPos
-import dotty.tools.repl.AbstractFileClassLoader
+import dotty.tools.io.AbstractFileClassLoader
 
 import scala.reflect.ClassTag
 

compiler/src/dotty/tools/dotc/util/ClasspathFromClassloader.scala

@@ -5,7 +5,7 @@ import scala.language.unsafeNulls
 import java.net.URLClassLoader
 import java.nio.file.Paths
 
-import dotty.tools.repl.AbstractFileClassLoader
+import dotty.tools.io.AbstractFileClassLoader
 
 object ClasspathFromClassloader {
 

compiler/src/dotty/tools/io/AbstractFileClassLoader.scala

@@ -0,0 +1,61 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package dotty.tools
+package io
+
+import scala.language.unsafeNulls
+
+import dotty.tools.io.AbstractFile
+
+import java.net.{URL, URLConnection, URLStreamHandler}
+import java.util.Collections
+
+class AbstractFileClassLoader(val root: AbstractFile, parent: ClassLoader) extends ClassLoader(parent):
+  private def findAbstractFile(name: String) = root.lookupPath(name.split('/').toIndexedSeq, directory = false)
+
+  // on JDK 20 the URL constructor we're using is deprecated,
+  // but the recommended replacement, URL.of, doesn't exist on JDK 8
+  @annotation.nowarn("cat=deprecation")
+  override protected def findResource(name: String): URL | Null =
+    findAbstractFile(name) match
+      case null => null
+      case file => new URL(null, s"memory:${file.path}", new URLStreamHandler {
+        override def openConnection(url: URL): URLConnection = new URLConnection(url) {
+          override def connect() = ()
+          override def getInputStream = file.input
+        }
+      })
+  override protected def findResources(name: String): java.util.Enumeration[URL] =
+    findResource(name) match
+      case null => Collections.enumeration(Collections.emptyList[URL])  //Collections.emptyEnumeration[URL]
+      case url  => Collections.enumeration(Collections.singleton(url))
+
+  override def findClass(name: String): Class[?] = {
+    var file: AbstractFile | Null = root
+    val pathParts = name.split("[./]").toList
+    for (dirPart <- pathParts.init) {
+      file = file.lookupName(dirPart, true)
+      if (file == null) {
+        throw new ClassNotFoundException(name)
+      }
+    }
+    file = file.lookupName(pathParts.last+".class", false)
+    if (file == null) {
+      throw new ClassNotFoundException(name)
+    }
+    val bytes = file.toByteArray
+    defineClass(name, bytes, 0, bytes.length)
+  }
+
+  override def loadClass(name: String): Class[?] = try findClass(name) catch case _: ClassNotFoundException => super.loadClass(name)
+end AbstractFileClassLoader

compiler/src/dotty/tools/repl/package.scala

@@ -724,9 +724,6 @@ object Build {
       libraryDependencies ++= Seq(
         "org.scala-lang.modules" % "scala-asm" % "9.8.0-scala-1", // used by the backend
         Dependencies.compilerInterface,
-        "org.jline" % "jline-reader" % "3.29.0",   // used by the REPL
-        "org.jline" % "jline-terminal" % "3.29.0",
-        "org.jline" % "jline-terminal-jni" % "3.29.0", // needed for Windows
         ("io.get-coursier" %% "coursier" % "2.0.16" % Test).cross(CrossVersion.for3Use2_13),
       ),
 
@@ -1479,7 +1476,8 @@ object Build {
     .enablePlugins(ScriptedPlugin)
     .aggregate(`scala3-interfaces`, `scala3-library-bootstrapped-new` , `scala-library-bootstrapped`,
       `tasty-core-bootstrapped-new`, `scala3-compiler-bootstrapped-new`, `scala3-sbt-bridge-bootstrapped`,
-      `scala3-staging-new`, `scala3-tasty-inspector-new`, `scala-library-sjs`, `scala3-library-sjs`, `scaladoc-new`)
+      `scala3-staging-new`, `scala3-tasty-inspector-new`, `scala-library-sjs`, `scala3-library-sjs`, 
+      `scaladoc-new`, `scala3-repl`)
     .settings(
       name          := "scala3-bootstrapped",
       moduleName    := "scala3-bootstrapped",
@@ -1594,7 +1592,7 @@ object Build {
 
   /* Configuration of the org.scala-lang:scala3-sbt-bridge:*.**.**-bootstrapped project */
   lazy val `scala3-sbt-bridge-bootstrapped` = project.in(file("sbt-bridge"))
-    .dependsOn(`scala3-compiler-bootstrapped-new`) // TODO: Would this actually evict the reference compiler in scala-tool?
+    .dependsOn(`scala3-compiler-bootstrapped-new`, `scala3-repl`) // TODO: Would this actually evict the reference compiler in scala-tool?
     .settings(publishSettings)
     .settings(
       name          := "scala3-sbt-bridge-bootstrapped",
@@ -1607,6 +1605,7 @@ object Build {
       // Add the source directories for the sbt-bridge (boostrapped)
       Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"),
       Test    / unmanagedSourceDirectories := Seq(baseDirectory.value / "test"),
+      Compile / unmanagedSourceDirectories += baseDirectory.value / "src-bootstrapped",
       Compile / resourceDirectory := baseDirectory.value / "resources",
       // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now
       Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"),
@@ -1773,6 +1772,90 @@ object Build {
       },
     )
 
+  lazy val `scala3-repl` = project.in(file("repl"))
+    .dependsOn(`scala3-compiler-bootstrapped-new` % "compile->compile;test->test")
+    .settings(publishSettings)
+    .settings(
+      name          := "scala3-repl",
+      moduleName    := "scala3-repl",
+      version       := dottyVersion,
+      versionScheme := Some("semver-spec"),
+      scalaVersion  := referenceVersion,
+      crossPaths    := true, // org.scala-lang:scala3-tasty-inspector has a crosspath
+      autoScalaLibrary := false, // do not add a dependency to stdlib, we depend transitively on the stdlib from `scala3-compiler-bootstrapped`
+      // Add the source directories for the sbt-bridge (boostrapped)
+      Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"),
+      Test    / unmanagedSourceDirectories := Seq(baseDirectory.value / "test"),
+      // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now
+      Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"),
+      // Make sure that the produced artifacts have the minimum JVM version in the bytecode
+      Compile / javacOptions  ++= Seq("--release", Versions.minimumJVMVersion),
+      Compile / scalacOptions ++= Seq("--java-output-version", Versions.minimumJVMVersion),
+      // Packaging configuration of `scala3-staging`
+      Compile / packageBin / publishArtifact := true,
+      Compile / packageDoc / publishArtifact := false,
+      Compile / packageSrc / publishArtifact := true,
+      // Only publish compilation artifacts, no test artifacts
+      Test    / publishArtifact := false,
+      publish / skip := false,
+      libraryDependencies ++= Seq(
+        "com.github.sbt" % "junit-interface" % "0.13.3" % Test,
+      ),
+      // Configure to use the non-bootstrapped compiler
+      scalaInstance := {
+        val externalCompilerDeps = (`scala3-compiler-nonbootstrapped` / Compile / externalDependencyClasspath).value.map(_.data).toSet
+
+        // IMPORTANT: We need to use actual jars to form the ScalaInstance and not
+        // just directories containing classfiles because sbt maintains a cache of
+        // compiler instances. This cache is invalidated based on timestamps
+        // however this is only implemented on jars, directories are never
+        // invalidated.
+        val tastyCore = (`tasty-core-nonbootstrapped` / Compile / packageBin).value
+        val scalaLibrary = (`scala-library-nonbootstrapped` / Compile / packageBin).value
+        val scala3Interfaces = (`scala3-interfaces` / Compile / packageBin).value
+        val scala3Compiler = (`scala3-compiler-nonbootstrapped` / Compile / packageBin).value
+
+        Defaults.makeScalaInstance(
+          dottyNonBootstrappedVersion,
+          libraryJars     = Array(scalaLibrary),
+          allCompilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ externalCompilerDeps,
+          allDocJars      = Seq.empty,
+          state.value,
+          scalaInstanceTopLoader.value
+        )
+      },
+      scalaCompilerBridgeBinaryJar := {
+        Some((`scala3-sbt-bridge-nonbootstrapped` / Compile / packageBin).value)
+      },
+      Test / javaOptions ++= {
+        val log = streams.value.log
+        val managedSrcDir = {
+          // Populate the directory
+          (Compile / managedSources).value
+
+          (Compile / sourceManaged).value
+        }
+        val externalDeps = (ThisProject / Runtime / externalDependencyClasspath).value
+        Seq(
+          s"-Ddotty.tests.dottyCompilerManagedSources=${managedSrcDir}",
+          s"-Ddotty.tests.classes.dottyInterfaces=${(`scala3-interfaces` / Compile / packageBin).value}",
+          s"-Ddotty.tests.classes.dottyCompiler=${(ThisProject / Compile / packageBin).value}",
+          s"-Ddotty.tests.classes.tastyCore=${(`tasty-core-bootstrapped-new` / Compile / packageBin).value}",
+          s"-Ddotty.tests.classes.compilerInterface=${findArtifactPath(externalDeps, "compiler-interface")}",
+          s"-Ddotty.tests.classes.scalaLibrary=${(`scala-library-bootstrapped` / Compile / packageBin).value}",
+          s"-Ddotty.tests.classes.scalaJSScalalib=${(`scala-library-sjs` / Compile / packageBin).value}",
+          s"-Ddotty.tests.classes.scalaAsm=${findArtifactPath(externalDeps, "scala-asm")}",
+          s"-Ddotty.tests.classes.jlineTerminal=${findArtifactPath(externalDeps, "jline-terminal")}",
+          s"-Ddotty.tests.classes.jlineReader=${findArtifactPath(externalDeps, "jline-reader")}",
+          s"-Ddotty.tests.classes.dottyStaging=${(LocalProject("scala3-staging-new") / Compile / packageBin).value}",
+          s"-Ddotty.tests.classes.dottyTastyInspector=${(LocalProject("scala3-tasty-inspector-new") / Compile / packageBin).value}",
+          s"-Ddotty.tools.dotc.semanticdb.test=${(ThisBuild / baseDirectory).value/"tests"/"semanticdb"}",
+        )
+      },
+      Test / forkOptions := 
+        (Test / forkOptions).value.withWorkingDirectory((ThisBuild / baseDirectory).value),
+    )
+
   // ==============================================================================================
   // =================================== SCALA STANDARD LIBRARY ===================================
   // ==============================================================================================

project/Build.scala

@@ -0,0 +1 @@
+scala.tools.repl.ScriptEngine$Factory

repl/resources/META-INF/services/javax.script.ScriptEngineFactory

@@ -16,7 +16,6 @@ package repl
 import scala.language.unsafeNulls
 
 import io.AbstractFile
-import dotty.tools.repl.ReplBytecodeInstrumentation
 
 import java.net.{URL, URLConnection, URLStreamHandler}
 import java.util.Collections

compiler/src/dotty/tools/repl/AbstractFileClassLoader.scala renamed to repl/src/dotty/tools/repl/AbstractFileClassLoader.scala

@@ -1,8 +1,9 @@
-package dotty.tools.repl
+package dotty.tools
+package repl
 
-import dotty.tools.dotc.ast.tpd
-import dotty.tools.dotc.core.Contexts.*
-import dotty.tools.dotc.core.Phases.Phase
+import dotc.ast.tpd
+import dotc.core.Contexts.*
+import dotc.core.Phases.Phase
 
 import scala.compiletime.uninitialized