Backport "Use pathing jars in cli commands" to 3.5.0 (#21172)

Backports #21121 to 3.5.0-RC4

Author: WojciechMazur

compiler/src/dotty/tools/dotc/classpath/ClassPathFactory.scala

@@ -7,6 +7,7 @@ import dotty.tools.io.{AbstractFile, VirtualDirectory}
 import FileUtils.*
 import dotty.tools.io.ClassPath
 import dotty.tools.dotc.core.Contexts.*
+import java.nio.file.Files
 
 /**
  * Provides factory methods for classpath. When creating classpath instances for a given path,
@@ -52,14 +53,30 @@ class ClassPathFactory {
 
   // Internal
   protected def classesInPathImpl(path: String, expand: Boolean)(using Context): List[ClassPath] =
-    for {
+    val files = for {
       file <- expandPath(path, expand)
       dir <- {
         def asImage = if (file.endsWith(".jimage")) Some(AbstractFile.getFile(file)) else None
         Option(AbstractFile.getDirectory(file)).orElse(asImage)
       }
     }
-    yield newClassPath(dir)
+    yield dir
+
+    val expanded =
+      if scala.util.Properties.propOrFalse("scala.expandjavacp") then
+        for
+          file <- files
+          a <- ClassPath.expandManifestPath(file.absolutePath)
+          path = java.nio.file.Paths.get(a.toURI()).nn
+          if Files.exists(path)
+        yield
+          newClassPath(AbstractFile.getFile(path))
+      else
+        Seq.empty
+
+    files.map(newClassPath) ++ expanded
+
+  end classesInPathImpl
 
   private def createSourcePath(file: AbstractFile)(using Context): ClassPath =
     if (file.isJarOrZip)

compiler/src/dotty/tools/io/ClassPath.scala

@@ -152,13 +152,18 @@ object ClassPath {
 
     val baseDir = file.parent
     new Jar(file).classPathElements map (elem =>
-      specToURL(elem) getOrElse (baseDir / elem).toURL
+      specToURL(elem, baseDir) getOrElse (baseDir / elem).toURL
     )
   }
 
-  def specToURL(spec: String): Option[URL] =
-    try Some(new URI(spec).toURL)
-    catch case _: MalformedURLException | _: URISyntaxException => None
+  def specToURL(spec: String, basedir: Directory): Option[URL] =
+    try
+      val uri = new URI(spec)
+      if uri.isAbsolute() then Some(uri.toURL())
+      else
+        Some(basedir.resolve(Path(spec)).toURL)
+    catch
+      case _: MalformedURLException | _: URISyntaxException => None
 
   def manifests: List[java.net.URL] = {
     import scala.jdk.CollectionConverters.EnumerationHasAsScala

dist/bin/common

@@ -6,25 +6,9 @@ source "$PROG_HOME/bin/common-shared"
 # * The code below is for Dotty
 # *-------------------------------------------------*/
 
-load_classpath () {
-  command="$1"
-  psep_pattern="$2"
-  __CLASS_PATH=""
-  while IFS= read -r line || [ -n "$line" ]; do
-    # jna-5 only appropriate for some combinations
-    if ! [[ ( -n ${conemu-} || -n ${msys-}) && "$line" == "*jna-5*" ]]; then
-      if [ -n "$__CLASS_PATH" ]; then
-        __CLASS_PATH+="$psep_pattern"
-      fi
-      __CLASS_PATH+="$PROG_HOME/maven2/$line"
-    fi
-  done < "$PROG_HOME/etc/$command.classpath"
-  echo "$__CLASS_PATH"
-}
-
 compilerJavaClasspathArgs () {
-  toolchain="$(load_classpath "scala" "$PSEP")"
-  toolchain_extra="$(load_classpath "with_compiler" "$PSEP")"
+  toolchain="$PROG_HOME/lib/scala.jar"
+  toolchain_extra="$PROG_HOME/lib/with_compiler.jar"
 
   if [ -n "$toolchain_extra" ]; then
     toolchain+="$PSEP$toolchain_extra"

dist/bin/common.bat

@@ -38,6 +38,6 @@ if not defined _PROG_HOME (
    set _EXITCODE=1
    goto :eof
 )
-set "_ETC_DIR=%_PROG_HOME%\etc"
+set "_LIB_DIR=%_PROG_HOME%\lib"
 
 set _PSEP=;

dist/bin/scalac

@@ -86,6 +86,7 @@ eval "\"$JAVACMD\"" \
    ${JAVA_OPTS:-$default_java_opts} \
    "${java_args[@]}" \
    "-classpath \"$jvm_cp_args\"" \
+   "-Dscala.expandjavacp=true" \
    "-Dscala.usejavacp=true" \
    "-Dscala.home=\"$PROG_HOME\"" \
    "dotty.tools.MainGenericCompiler" \

dist/bin/scalac.bat

@@ -24,7 +24,7 @@ call :compilerJavaClasspathArgs
 @rem we need to escape % in the java command path, for some reason this doesnt work in common.bat
 set "_JAVACMD=!_JAVACMD:%%=%%%%!"
 
-call "%_JAVACMD%" %_JAVA_ARGS% -classpath "%_JVM_CP_ARGS%" "-Dscala.usejavacp=true" "-Dscala.home=%_PROG_HOME%" dotty.tools.MainGenericCompiler %_SCALA_ARGS%
+call "%_JAVACMD%" %_JAVA_ARGS% -classpath "%_JVM_CP_ARGS%" "-Dscala.usejavacp=true" "-Dscala.expandjavacp=true" "-Dscala.home=%_PROG_HOME%" dotty.tools.MainGenericCompiler %_SCALA_ARGS%
 if not %ERRORLEVEL%==0 (
     set _EXITCODE=1
     goto end
@@ -88,17 +88,8 @@ goto :eof
 
 @rem output parameter: _JVM_CP_ARGS
 :compilerJavaClasspathArgs
-
-set "CP_FILE=%_ETC_DIR%\scala.classpath"
-call :loadClasspathFromFile %CP_FILE%
-set "__TOOLCHAIN=%_CLASS_PATH_RESULT%"
-
-set "CP_FILE=%_ETC_DIR%\with_compiler.classpath"
-call :loadClasspathFromFile %CP_FILE%
-
-if defined _CLASS_PATH_RESULT (
-    set "__TOOLCHAIN=%__TOOLCHAIN%%_PSEP%%_CLASS_PATH_RESULT%"
-)
+set "__TOOLCHAIN=%_LIB_DIR%\scala.jar"
+set "__TOOLCHAIN=%__TOOLCHAIN%%_PSEP%%_LIB_DIR%\with_compiler.jar%"
 
 if defined _SCALA_CPATH (
     set "_JVM_CP_ARGS=%__TOOLCHAIN%%_SCALA_CPATH%"
@@ -107,24 +98,6 @@ if defined _SCALA_CPATH (
 )
 goto :eof
 
-@REM concatentate every line in "%_ARG_FILE%" with _PSEP
-@REM arg 1 - file to read
-:loadClasspathFromFile
-set _ARG_FILE=%1
-set _CLASS_PATH_RESULT=
-if exist "%_ARG_FILE%" (
-    for /f "usebackq delims=" %%i in ("%_ARG_FILE%") do (
-        set "_LIB=%_PROG_HOME%\maven2\%%i"
-        set "_LIB=!_LIB:/=\!"
-        if not defined _CLASS_PATH_RESULT (
-            set "_CLASS_PATH_RESULT=!_LIB!"
-        ) else (
-            set "_CLASS_PATH_RESULT=!_CLASS_PATH_RESULT!%_PSEP%!_LIB!"
-        )
-    )
-)
-goto :eof
-
 @rem #########################################################################
 @rem ## Cleanups
 

dist/bin/scaladoc

@@ -36,6 +36,7 @@ CompilerMain=dotty.tools.dotc.Main
 DecompilerMain=dotty.tools.dotc.decompiler.Main
 ReplMain=dotty.tools.repl.Main
 ScriptingMain=dotty.tools.scripting.Main
+JVM_CP_ARGS="$PROG_HOME/lib/scaladoc.jar"
 
 PROG_NAME=$CompilerMain
 
@@ -52,12 +53,6 @@ addScrip() {
   script_args+=("'$1'")
 }
 
-classpathArgs () {
-  CLASS_PATH="$(load_classpath "scaladoc" "$PSEP")"
-
-  jvm_cp_args="-classpath \"$CLASS_PATH\""
-}
-
 #for A in "$@" ; do echo "A[$A]" ; done ; exit 2
 
 while [[ $# -gt 0 ]]; do
@@ -79,12 +74,11 @@ case "$1" in
   esac
 done
 
-classpathArgs
-
 eval "\"$JAVACMD\"" \
      ${JAVA_OPTS:-$default_java_opts} \
      "${java_args[@]}" \
-     "${jvm_cp_args-}" \
+     -classpath "${JVM_CP_ARGS}" \
+      -Dscala.expandjavacp=true \
       -Dscala.usejavacp=true \
      "dotty.tools.scaladoc.Main"  \
      "${scala_args[@]}" \

dist/bin/scaladoc.bat

@@ -21,8 +21,6 @@ call :args %*
 @rem #########################################################################
 @rem ## Main
 
-call :classpathArgs
-
 if defined JAVA_OPTS ( set _JAVA_OPTS=%JAVA_OPTS%
 ) else ( set _JAVA_OPTS=%_DEFAULT_JAVA_OPTS%
 )
@@ -31,7 +29,8 @@ if defined JAVA_OPTS ( set _JAVA_OPTS=%JAVA_OPTS%
 set "_JAVACMD=!_JAVACMD:%%=%%%%!"
 
 call "%_JAVACMD%" %_JAVA_OPTS% %_JAVA_DEBUG% %_JAVA_ARGS% ^
--classpath "%_CLASS_PATH%" ^
+-classpath "%_LIB_DIR%\scaladoc.jar" ^
+-Dscala.expandjavacp=true ^
 -Dscala.usejavacp=true ^
 dotty.tools.scaladoc.Main %_SCALA_ARGS% %_RESIDUAL_ARGS%
 if not %ERRORLEVEL%==0 (
@@ -103,29 +102,6 @@ goto :eof
 set _RESIDUAL_ARGS=%_RESIDUAL_ARGS% %~1
 goto :eof
 
-@rem output parameter: _CLASS_PATH
-:classpathArgs
-set "_ETC_DIR=%_PROG_HOME%\etc"
-@rem keep list in sync with bash script `bin\scaladoc` !
-call :loadClasspathFromFile
-goto :eof
-
-@REM concatentate every line in "%_ETC_DIR%\scaladoc.classpath" with _PSEP
-:loadClasspathFromFile
-set _CLASS_PATH=
-if exist "%_ETC_DIR%\scaladoc.classpath" (
-    for /f "usebackq delims=" %%i in ("%_ETC_DIR%\scaladoc.classpath") do (
-        set "_LIB=%_PROG_HOME%\maven2\%%i"
-        set "_LIB=!_LIB:/=\!"
-        if not defined _CLASS_PATH (
-            set "_CLASS_PATH=!_LIB!"
-        ) else (
-            set "_CLASS_PATH=!_CLASS_PATH!%_PSEP%!_LIB!"
-        )
-    )
-)
-goto :eof
-
 @rem #########################################################################
 @rem ## Cleanups
 

project/Build.scala

@@ -2131,7 +2131,7 @@ object Build {
     republishRepo := target.value / "republish",
     packResourceDir += (republishRepo.value / "bin" -> "bin"),
     packResourceDir += (republishRepo.value / "maven2" -> "maven2"),
-    packResourceDir += (republishRepo.value / "etc" -> "etc"),
+    packResourceDir += (republishRepo.value / "lib" -> "lib"),
     republishCommandLibs +=
       ("scala" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core")),
     republishCommandLibs +=

project/RepublishPlugin.scala

@@ -213,16 +213,24 @@ object RepublishPlugin extends AutoPlugin {
     val classpaths = coursierFetch(coursierJar, log, csrCacheDir, localRepo, resolvedLocal.map(_.id.toString))
 
     if (commandLibs.nonEmpty) {
-      IO.createDirectory(republishDir / "etc")
+      IO.createDirectory(republishDir / "lib")
       for ((command, libs) <- commandLibs) {
         val (negated, actual) = libs.partition(_.startsWith("^!"))
         val subtractions = negated.map(_.stripPrefix("^!"))
 
         def compose(libs: List[String]): List[String] =
           libs.map(fuzzyFind(classpaths, _)).reduceOption(_ ++ _).map(_.distinct).getOrElse(Nil)
 
+        // Compute the classpath entries
         val entries = compose(actual).diff(compose(subtractions))
-        IO.write(republishDir / "etc" / s"$command.classpath", entries.mkString("\n"))
+        // Generate the MANIFEST for the pathing jar
+        val manifest = new java.util.jar.Manifest();
+        manifest.getMainAttributes().put(java.util.jar.Attributes.Name.MANIFEST_VERSION, "1.0");
+        manifest.getMainAttributes().put(java.util.jar.Attributes.Name.CLASS_PATH, entries.map(e => s"../maven2/$e").mkString(" "))
+        // Write the pathing jar to the Disk
+        val file = republishDir / "lib" / s"$command.jar"
+        val jar = new java.util.jar.JarOutputStream(new java.io.FileOutputStream(file), manifest)
+        jar.close()
       }
     }