diff --git a/build.mill.scala b/build.mill.scala index 12a56613db..3d77168989 100644 --- a/build.mill.scala +++ b/build.mill.scala @@ -39,6 +39,7 @@ import mill._ import mill.api.Loose import scalalib.{publish => _, _} import mill.contrib.bloop.Bloop +import mill.define.Task.Simple import mill.testrunner.TestResult import _root_.scala.util.{Properties, Using} @@ -109,15 +110,6 @@ object `test-runner` extends Cross[TestRunner](Scala.runnerScalaVersions) with CrossScalaDefaultToRunner object `tasty-lib` extends Cross[TastyLib](Scala.scala3MainVersions) with CrossScalaDefaultToInternal -// Runtime classes used within native image on Scala 3 replacing runtime from Scala -object `scala3-runtime` extends Cross[Scala3Runtime](Scala.scala3MainVersions) - with CrossScalaDefaultToInternal -// Logic to process classes that is shared between build and the scala-cli itself -object `scala3-graal` extends Cross[Scala3Graal](Scala.scala3MainVersions) - with CrossScalaDefaultToInternal -// Main app used to process classpath within build itself -object `scala3-graal-processor` extends Cross[Scala3GraalProcessor](Scala.scala3MainVersions) - with CrossScalaDefaultToInternal object `scala-cli-bsp` extends JavaModule with ScalaCliPublishModule { override def ivyDeps: T[Agg[Dep]] = super.ivyDeps() ++ Seq( @@ -685,38 +677,6 @@ trait Options extends ScalaCliCrossSbtModule with ScalaCliPublishModule with Has } } -trait Scala3Runtime extends CrossSbtModule with ScalaCliPublishModule { - override def crossScalaVersion: String = crossValue - override def ivyDeps: T[Agg[Dep]] = super.ivyDeps() -} - -trait Scala3Graal extends ScalaCliCrossSbtModule - with ScalaCliPublishModule with ScalaCliScalafixModule { - override def crossScalaVersion: String = crossValue - override def ivyDeps: T[Agg[Dep]] = super.ivyDeps() ++ Agg( - Deps.asm, - Deps.osLib - ) - - override def resources: T[Seq[PathRef]] = Task.Sources { - val extraResourceDir = Task.dest / "extra" - // scala3RuntimeFixes.jar is also used within - // resource-config.json and BytecodeProcessor.scala - os.copy.over( - `scala3-runtime`(crossScalaVersion).jar().path, - extraResourceDir / "scala3RuntimeFixes.jar", - createFolders = true - ) - super.resources() ++ Seq(mill.PathRef(extraResourceDir)) - } -} - -trait Scala3GraalProcessor extends CrossScalaModule with ScalaCliPublishModule { - override def moduleDeps: Seq[SonatypeCentralPublishModule] = - Seq(`scala3-graal`(crossScalaVersion)) - override def finalMainClass: T[String] = "scala.cli.graal.CoursierCacheProcessor" -} - trait Build extends ScalaCliCrossSbtModule with ScalaCliPublishModule with HasTests @@ -926,7 +886,6 @@ trait Cli extends CrossSbtModule with ProtoBuildModule with CliLaunchers def moduleDeps: Seq[SonatypeCentralPublishModule] = Seq( `build-module`(crossScalaVersion), config(crossScalaVersion), - `scala3-graal`(crossScalaVersion), `specification-level`(crossScalaVersion) ) @@ -948,7 +907,9 @@ trait Cli extends CrossSbtModule with ProtoBuildModule with CliLaunchers Deps.signingCli.exclude((organization, "config_2.13")), Deps.slf4jNop, // to silence jgit Deps.sttp, - Deps.scalafixInterfaces + Deps.scalafixInterfaces, + Deps.scala3Graal, // TODO: drop this if we ever bump internal JDK to 24+ + Deps.scala3GraalProcessor // TODO: drop this if we ever bump internal JDK to 24+ ) override def compileIvyDeps: T[Agg[Dep]] = super.compileIvyDeps() ++ Agg( Deps.jsoniterMacros, @@ -956,13 +917,21 @@ trait Cli extends CrossSbtModule with ProtoBuildModule with CliLaunchers ) override def mainClass: T[Option[String]] = Some("scala.cli.ScalaCli") + private def scala3GraalProcessorClassPath: T[Agg[PathRef]] = T { + resolveDeps(T { + val bind = bindDependency() + Agg(Deps.scala3GraalProcessor).map(bind) + })() + } + override def nativeImageClassPath: T[Seq[PathRef]] = Task { val classpath = super.nativeImageClassPath().map(_.path).mkString(File.pathSeparator) val cache = Task.dest / "native-cp" - // `scala3-graal-processor`.run() do not give me output and I cannot pass dynamically computed values like classpath + // `scala3-graal-processor`.run() does not give me output and I cannot pass dynamically computed values like classpath + // TODO: drop this if we ever bump internal JDK to 24+ val res = mill.util.Jvm.callProcess( - mainClass = `scala3-graal-processor`(crossScalaVersion).finalMainClass(), - classPath = `scala3-graal-processor`(crossScalaVersion).runClasspath().map(_.path), + mainClass = "scala.cli.graal.CoursierCacheProcessor", + classPath = scala3GraalProcessorClassPath().map(_.path).toList, mainArgs = Seq(cache.toNIO.toString, classpath) ) val cp = res.out.trim() diff --git a/modules/cli/src/main/scala/scala/cli/packaging/NativeImage.scala b/modules/cli/src/main/scala/scala/cli/packaging/NativeImage.scala index 646b1bdee3..d168a69485 100644 --- a/modules/cli/src/main/scala/scala/cli/packaging/NativeImage.scala +++ b/modules/cli/src/main/scala/scala/cli/packaging/NativeImage.scala @@ -4,8 +4,9 @@ import java.io.File import scala.annotation.tailrec import scala.build.internal.{ManifestJar, Runner} +import scala.build.internals.ConsoleUtils.ScalaCliConsole.warnPrefix import scala.build.internals.EnvVar -import scala.build.{Build, Logger, Positioned} +import scala.build.{Build, Logger, Positioned, coursierVersion} import scala.cli.errors.GraalVMNativeImageError import scala.cli.graal.{BytecodeProcessor, TempCache} import scala.cli.internal.CachedBinary @@ -206,7 +207,15 @@ object NativeImage { // seems native-image doesn't correctly parse paths in manifests - this is especially a problem on Windows wrongSimplePathsInManifest = true ) { processedClassPath => - val needsProcessing = builds.head.scalaParams.exists(_.scalaVersion.startsWith("3.")) + val needsProcessing = + builds.head.scalaParams.map(_.scalaVersion.coursierVersion) + .exists(sv => (sv >= "3.0.0".coursierVersion) && (sv < "3.3.0".coursierVersion)) + if needsProcessing then + logger.message( + s"""$warnPrefix building native images with Scala 3 older than 3.3.0 is deprecated. + |$warnPrefix support will be dropped in a future Scala CLI version. + |$warnPrefix it is advised to upgrade to a more recent Scala version.""".stripMargin + ) val (classPath, toClean, scala3extraOptions) = if needsProcessing then { val cpString = processedClassPath.mkString(File.pathSeparator) diff --git a/modules/integration/src/test/scala/scala/cli/integration/TestNativeImageOnScala3.scala b/modules/integration/src/test/scala/scala/cli/integration/TestNativeImageOnScala3.scala index 503d04ac29..371137f9f8 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/TestNativeImageOnScala3.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/TestNativeImageOnScala3.scala @@ -40,14 +40,21 @@ class TestNativeImageOnScala3 extends ScalaCliSuite { } } - test("lazy vals") { - runTest("1")("2") { - """//> using scala 3.1.1 - |class A(a: String) { lazy val b = a.toInt + 1 } - |@main def add1(i: String) = println(A(i).b) - |""".stripMargin - } + for { + scalaVersion <- TestUtil.legacyScalaVersionsOnePerMinor.sorted ++ Seq( + Constants.scala3Lts, + Constants.scala3Next, + Constants.scala3NextRc + ) } + test(s"lazy vals ($scalaVersion)") { + runTest("1")("2") { + s"""//> using scala $scalaVersion + |class A(a: String) { lazy val b = a.toInt + 1 } + |@main def add1(i: String) = println(A(i).b) + |""".stripMargin + } + } test("lazy vals and enums with default scala version") { runTest("1")("2", "A") { diff --git a/modules/scala3-graal-processor/src/scala/cli/graal/CoursierCacheProcessor.scala b/modules/scala3-graal-processor/src/scala/cli/graal/CoursierCacheProcessor.scala deleted file mode 100644 index cd82bb6dca..0000000000 --- a/modules/scala3-graal-processor/src/scala/cli/graal/CoursierCacheProcessor.scala +++ /dev/null @@ -1,15 +0,0 @@ -package scala.cli.graal - -import java.io.File -import java.nio.channels.FileChannel - -object CoursierCacheProcessor { - def main(args: Array[String]) = { - val List(cacheDir, classpath) = args.toList - val cache = DirCache(os.Path(cacheDir, os.pwd)) - - val newCp = BytecodeProcessor.processClassPath(classpath, cache).map(_.nioPath) - - println(newCp.mkString(File.pathSeparator)) - } -} diff --git a/modules/scala3-graal/src/main/scala/scala/cli/graal/BytecodeProcessor.scala b/modules/scala3-graal/src/main/scala/scala/cli/graal/BytecodeProcessor.scala deleted file mode 100644 index 0f0f1fb673..0000000000 --- a/modules/scala3-graal/src/main/scala/scala/cli/graal/BytecodeProcessor.scala +++ /dev/null @@ -1,220 +0,0 @@ -package scala.cli.graal - -import org.objectweb.asm.* - -import java.io.{File, InputStream} -import java.nio.file.{Files, StandardOpenOption} -import java.util.jar.{Attributes, JarEntry, JarFile, JarOutputStream, Manifest} - -import scala.jdk.CollectionConverters.* - -object BytecodeProcessor { - - def toClean(classpath: Seq[ClassPathEntry]): Seq[os.Path] = classpath.flatMap { - case Processed(path, _, TempCache) => Seq(path) - case PathingJar(path, entries) => toClean(path +: entries) - case _ => Nil - } - - def processPathingJar(pathingJar: os.Path, cache: JarCache): Seq[ClassPathEntry] = { - val jarFile = new JarFile(pathingJar.toIO) - try { - val cp = jarFile.getManifest().getMainAttributes().getValue(Attributes.Name.CLASS_PATH) - if (cp != null && cp.nonEmpty) { - // paths in pathing jars are separated by spaces - val entries = cp.split(" +").toSeq.map { rawEntry => - // In manifest JARs, class path entries are supposed to be encoded as URL paths. - // This especially matters on Windows, where we end up with paths like "/C:/…". - // Theoretically, we should decode those paths with - // os.Path(java.nio.file.Paths.get(new java.net.URI("file://" + rawEntry)), os.pwd) - // but native-image doesn't follow this, and decodes them with just Paths.get(…). - // As the JARs we are handed are supposed to be passed to native-image, we follow - // the native-image convention here. - os.Path(rawEntry, os.pwd) - } - val processedCp = processClassPathEntries(entries, cache) - val dest = os.temp(suffix = ".jar") - val outStream = Files.newOutputStream(dest.toNIO, StandardOpenOption.CREATE) - try { - val stringCp = processedCp.map(_.path.toNIO).mkString(" ") - val manifest = new Manifest(jarFile.getManifest()) - manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, stringCp) - val outjar = new JarOutputStream(outStream, manifest) - outjar.close() - dest.toNIO.toString() - Seq(PathingJar(Processed(dest, pathingJar, TempCache), processedCp)) - } - finally outStream.close() - } - else processClassPathEntries(Seq(pathingJar), cache) - } - finally jarFile.close() - } - - def processClassPath(classPath: String, cache: JarCache = TempCache): Seq[ClassPathEntry] = - classPath.split(File.pathSeparator) match { - case Array(maybePathingJar) if maybePathingJar.endsWith(".jar") => - processPathingJar(os.Path(maybePathingJar, os.pwd), cache) - case cp => - val cp0 = cp.toSeq.map(os.Path(_, os.pwd)) - processClassPathEntries(cp0, cache) - } - - def processClassPathEntries(entries: Seq[os.Path], cache: JarCache): Seq[ClassPathEntry] = { - val cp = entries.map { path => - cache.cache(path) { dest => - if (path.ext == "jar" && os.isFile(path)) processJar(path, dest, cache) - else if (os.isDir(path)) processDir(path, dest, cache) - else Unmodified(dest) - } - } - if (cp.exists(_.modified)) { - // jar with runtime deps is added as a resource - // scala3RuntimeFixes.jar is also used within - // resource-config.json and BytecodeProcessor.scala - val jarName = "scala3RuntimeFixes.jar" - val runtimeJarIs = getClass().getClassLoader.getResourceAsStream(jarName) - if (runtimeJarIs == null) throw new NoSuchElementException( - "Unable to find scala3RuntimeFixes.jar on classpath, did you add scala3-graal jar on classpath?" - ) - val created = cache.put(os.RelPath(jarName), runtimeJarIs.readAllBytes()) - created +: cp - } - else cp // No need to add processed jar - } - - def processDir(dir: os.Path, dest: os.Path, cache: JarCache): ClassPathEntry = { - val paths = os.walk(dir).filter(os.isFile) - val (skipped, processed) = paths.partitionMap { - case p if p.ext != "class" => - Left(p) - case clazzFile => - val original = os.read.bytes(clazzFile) - processClassFile(original) match { - case Some(content) => - val relPath = clazzFile.relativeTo(dir) - val destPath = dest / relPath - os.makeDir.all(destPath / os.up) - assert(content != original) - os.write(destPath, content) - Right(clazzFile) - case _ => - Left(clazzFile) - } - } - if (processed.nonEmpty) { - skipped.foreach(file => - os.copy.over(file, dest / (file.relativeTo(dir)), createFolders = true) - ) - Processed(dest, dir, cache) - } - else Unmodified(dir) - } - - def processJar(path: os.Path, dest: os.Path, cache: JarCache): ClassPathEntry = { - val jarFile = new JarFile(path.toIO) - try { - var processedBytecode: Option[Array[Byte]] = None - val endMarker = "///" // not a valid path - var processed: String = endMarker - def processEntry(entry: JarEntry) = { - val newBytecode = processClassFile(jarFile.getInputStream(entry)) - processed = entry.getName() - processedBytecode = newBytecode - newBytecode.fold(entry.getName())(_ => endMarker) // empty string is an end marker - } - - val classFilesIterator = - jarFile.entries().asIterator().asScala.filter(_.getName().endsWith(".class")) - val cachedEntries = classFilesIterator.map(processEntry).takeWhile(_ != endMarker).toSet - - if (processedBytecode.isEmpty) Unmodified(path) - else { - os.makeDir.all(dest / os.up) - val outStream = Files.newOutputStream(dest.toNIO, StandardOpenOption.CREATE) - val outjar = new JarOutputStream(outStream) - jarFile.entries().asIterator().asScala.foreach { entry => - val content: Array[Byte] = jarFile.getInputStream(entry).readAllBytes() - val name = entry.getName() - val destBytes = - if (cachedEntries.contains(name) || !name.endsWith(".class")) content - else if (name == processed) processedBytecode.get - else processClassFile(content).getOrElse(content) - - val newEntry = new JarEntry(entry.getName()) - - outjar.putNextEntry(newEntry) - outjar.write(destBytes) - outjar.closeEntry() - } - outjar.close() - Processed(dest, path, cache) - } - } - finally jarFile.close() - } - - def processClassReader(reader: ClassReader): Option[Array[Byte]] = { - val writer = new ClassWriter(reader, ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS) - val visitor = new LazyValVisitor(writer) - val res = util.Try(reader.accept(visitor, 0)) - if (visitor.changed && res.isSuccess) Some(writer.toByteArray) else None - } - - def processClassFile(content: => InputStream): Option[Array[Byte]] = { - val is = content - try processClassReader(new ClassReader(is)) - finally is.close() - } - - def processClassFile(content: Array[Byte]): Option[Array[Byte]] = - processClassReader(new ClassReader(content)) - - class LazyValVisitor(writer: ClassWriter) extends ClassVisitor(Opcodes.ASM9, writer) { - - var changed: Boolean = false - - class StaticInitVistor(parent: MethodVisitor) - extends MethodVisitor(Opcodes.ASM9, parent) { - - override def visitMethodInsn( - opcode: Int, - owner: String, - name: String, - descr: String, - isInterface: Boolean - ): Unit = - if (owner == "scala/runtime/LazyVals$" && name == "getOffset") { - changed = true - super.visitMethodInsn( - Opcodes.INVOKEVIRTUAL, - "java/lang/Class", - "getDeclaredField", - "(Ljava/lang/String;)Ljava/lang/reflect/Field;", - false - ) - super.visitMethodInsn( - Opcodes.INVOKESTATIC, - "scala/cli/runtime/SafeLazyVals", - "getOffset", - "(Ljava/lang/Object;Ljava/lang/reflect/Field;)J", - false - ) - } - else - super.visitMethodInsn(opcode, owner, name, descr, isInterface) - } - - override def visitMethod( - access: Int, - name: String, - desc: String, - sig: String, - exceptions: Array[String] - ): MethodVisitor = - if (name == "") - new StaticInitVistor(super.visitMethod(access, name, desc, sig, exceptions)) - else - super.visitMethod(access, name, desc, sig, exceptions) - } -} diff --git a/modules/scala3-graal/src/main/scala/scala/cli/graal/ClassPathEntry.scala b/modules/scala3-graal/src/main/scala/scala/cli/graal/ClassPathEntry.scala deleted file mode 100644 index 77bd7f5e83..0000000000 --- a/modules/scala3-graal/src/main/scala/scala/cli/graal/ClassPathEntry.scala +++ /dev/null @@ -1,19 +0,0 @@ -package scala.cli.graal - -import java.nio.file.Path - -sealed trait ClassPathEntry { - def nioPath: Path = path.toNIO - def path: os.Path - def modified = true -} - -case class Unmodified(path: os.Path) extends ClassPathEntry { - override def modified: Boolean = false -} -case class Processed(path: os.Path, original: os.Path, cache: JarCache) extends ClassPathEntry -case class CreatedEntry(path: os.Path) extends ClassPathEntry - -case class PathingJar(jar: ClassPathEntry, entries: Seq[ClassPathEntry]) extends ClassPathEntry { - override def path: os.Path = jar.path -} diff --git a/modules/scala3-graal/src/main/scala/scala/cli/graal/CoursierCache.scala b/modules/scala3-graal/src/main/scala/scala/cli/graal/CoursierCache.scala deleted file mode 100644 index 60acc9439d..0000000000 --- a/modules/scala3-graal/src/main/scala/scala/cli/graal/CoursierCache.scala +++ /dev/null @@ -1,30 +0,0 @@ -package scala.cli.graal - -case class CoursierCache(root: os.Path) extends JarCache { - val hasher: os.Path => String = p => os.size(p).toString() // replace with proper md5 based caches - val prefix = "v2" - val cache = root / ".graal_processor" - - override def put(entry: os.RelPath, content: Array[Byte]): ClassPathEntry = { - // TODO better hashing - val name = s"${entry.baseName}_${content.length}_$prefix.${entry.ext}" - val dest = cache / entry / os.up / name - if (!os.exists(dest)) { - os.makeDir.all(dest / os.up) - os.write(dest, content) - } - CreatedEntry(dest) - } - - override def cache(path: os.Path)(processPath: os.Path => ClassPathEntry): ClassPathEntry = { - def ignore = TempCache.cache(path)(processPath) - if (!path.startsWith(root) || os.isDir(path)) ignore - else { - val relPath = path.relativeTo(root) - val name = s"${path.baseName}_${hasher(path)}_$prefix.${path.ext}" - val dest = cache / relPath / os.up / name - if (os.exists(dest)) Processed(dest, path, this) - else processPath(dest) - } - } -} diff --git a/modules/scala3-graal/src/main/scala/scala/cli/graal/JarCache.scala b/modules/scala3-graal/src/main/scala/scala/cli/graal/JarCache.scala deleted file mode 100644 index acbfae447b..0000000000 --- a/modules/scala3-graal/src/main/scala/scala/cli/graal/JarCache.scala +++ /dev/null @@ -1,35 +0,0 @@ -package scala.cli.graal - -trait JarCache { - def cache(path: os.Path)(processPath: os.Path => ClassPathEntry): ClassPathEntry - def put(entry: os.RelPath, bytes: Array[Byte]): ClassPathEntry -} - -case class DirCache(dir: os.Path) extends JarCache { - private def dest(original: os.Path) = - dir / s"${original.toNIO.toString.hashCode()}-${original.last}" - override def cache(path: os.Path)(processPath: os.Path => ClassPathEntry): ClassPathEntry = - processPath(dest(path)) - - override def put(entry: os.RelPath, content: Array[Byte]): ClassPathEntry = { - val path = dir / entry - os.write.over(path, content, createFolders = true) - CreatedEntry(path) - } -} - -object TempCache extends JarCache { - - override def cache(path: os.Path)(processPath: os.Path => ClassPathEntry): ClassPathEntry = - processPath( - if (os.isDir(path)) os.temp.dir(prefix = path.last) - else os.temp(prefix = path.baseName, suffix = "." + path.ext) - ) - - override def put(entry: os.RelPath, content: Array[Byte]): ClassPathEntry = { - val path = os.temp(prefix = entry.baseName, suffix = entry.ext) - os.write.over(path, content, createFolders = true) - CreatedEntry(path) - } - -} diff --git a/modules/scala3-runtime/src/main/scala/scala/cli/runtime/SafeLazyVals.scala b/modules/scala3-runtime/src/main/scala/scala/cli/runtime/SafeLazyVals.scala deleted file mode 100644 index df74dff3bd..0000000000 --- a/modules/scala3-runtime/src/main/scala/scala/cli/runtime/SafeLazyVals.scala +++ /dev/null @@ -1,28 +0,0 @@ -package scala.cli.runtime - -import java.lang.reflect.Field - -object SafeLazyVals { - private val debug = false - - private val unsafe: sun.misc.Unsafe = - classOf[sun.misc.Unsafe].getDeclaredFields.nn.find { field => - field.nn.getType == classOf[sun.misc.Unsafe] && { - field.nn.setAccessible(true) - true - } - } - .map(_.nn.get(null).asInstanceOf[sun.misc.Unsafe]) - .getOrElse { - throw new ExceptionInInitializerError { - new IllegalStateException("Can't find instance of sun.misc.Unsafe") - } - } - - def getOffset(unused: Any, field: Field): Long = { - val r = unsafe.objectFieldOffset(field) - if (debug) - println(s"getOffset(${field.getDeclaringClass}}, ${field.getName}) = $r") - r - } -} diff --git a/project/deps/package.mill.scala b/project/deps/package.mill.scala index c75cb82e8f..0b1f7d5eca 100644 --- a/project/deps/package.mill.scala +++ b/project/deps/package.mill.scala @@ -6,6 +6,7 @@ object Cli { def runnerScala30LegacyVersion = "1.7.1" // last runner version to support pre-LTS Scala 3 versions def runnerScala2LegacyVersion = "1.9.1" // last runner version to support pre-Scala-3 versions + def scala3GraalLegacyVersion = "1.9.1" // last version to support pre-3.3 processing for Graal } object Scala { @@ -214,7 +215,10 @@ object Deps { def pythonInterface = mvn"io.github.alexarchambault.python:interface:0.1.0" def pythonNativeLibs = mvn"ai.kien::python-native-libs:0.2.4" def scalac(sv: String) = mvn"org.scala-lang:scala-compiler:$sv" - def scalafmtCli = mvn"org.scalameta:scalafmt-cli_2.13:${Versions.scalafmt}" + def scala3Graal = mvn"org.virtuslab.scala-cli::scala3-graal:${Cli.scala3GraalLegacyVersion}" + def scala3GraalProcessor = + mvn"org.virtuslab.scala-cli::scala3-graal-processor:${Cli.scala3GraalLegacyVersion}" + def scalafmtCli = mvn"org.scalameta:scalafmt-cli_2.13:${Versions.scalafmt}" // Force using of 2.13 - is there a better way? def scalaJsEnvJsdomNodejs = mvn"org.scala-js:scalajs-env-jsdom-nodejs_2.13:1.1.0"