From 6f6e0ac28fc8681e95603b8b43329cafa4430897 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Sat, 27 Sep 2025 09:28:43 +0200 Subject: [PATCH 01/21] copy in PrecompileTools --- .../src/PrecompileTools/PrecompileTools.jl | 18 +++ .../src/PrecompileTools/invalidations.jl | 93 +++++++++++++ .../src/PrecompileTools/workloads.jl | 125 ++++++++++++++++++ .../src/QuartoNotebookWorker.jl | 3 + src/QuartoNotebookWorker/src/precompile.jl | 3 + 5 files changed, 242 insertions(+) create mode 100644 src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl create mode 100644 src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl create mode 100644 src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl create mode 100644 src/QuartoNotebookWorker/src/precompile.jl diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl new file mode 100644 index 00000000..3b6a8e5d --- /dev/null +++ b/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl @@ -0,0 +1,18 @@ +module PrecompileTools + +using Preferences + +export @setup_workload, @compile_workload, @recompile_invalidations + +const verbose = Ref(false) # if true, prints all the precompiles + +function precompile_mi(mi::Core.MethodInstance) + precompile(mi) + verbose[] && println(mi) + return +end + +include("workloads.jl") +include("invalidations.jl") + +end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl b/src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl new file mode 100644 index 00000000..f90a1c8a --- /dev/null +++ b/src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl @@ -0,0 +1,93 @@ +""" + @recompile_invalidations begin + using PkgA + ⋮ + end + +Recompile any invalidations that occur within the given expression. This is generally intended to be used +by users in creating "Startup" packages to ensure that the code compiled by package authors is not invalidated. +""" +macro recompile_invalidations(expr) + # use QuoteNode instead of esc(Expr(:quote)) so that $ is not permitted as usual (instead of having this macro work like `@eval`) + return :(recompile_invalidations($__module__, $(QuoteNode(expr)))) +end + +const ReinferUtils = isdefined(Base, :ReinferUtils) ? Base.ReinferUtils : Base.StaticData + +function recompile_invalidations(__module__::Module, @nospecialize expr) + listi = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) + liste = ReinferUtils.debug_method_invalidation(true) + try + Core.eval(__module__, expr) + finally + ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) + ReinferUtils.debug_method_invalidation(false) + end + if ccall(:jl_generating_output, Cint, ()) == 1 + foreach(precompile_mi, invalidation_leaves(listi, liste)) + end + nothing +end + +function invalidation_leaves(listi, liste) + umis = Set{Core.MethodInstance}() + # `queued` is a queue of length 0 or 1 of invalidated MethodInstances. + # We wait to read the `depth` to find out if it's a leaf. + queued, depth = nothing, 0 + function cachequeued(item, nextdepth) + if queued !== nothing && nextdepth <= depth + push!(umis, queued) + end + queued, depth = item, nextdepth + end + + # Process method insertion/deletion events + i, ilast = firstindex(listi), lastindex(listi) + while i <= ilast + item = listi[i] + if isa(item, Core.MethodInstance) + if i < lastindex(listi) + nextitem = listi[i+1] + if nextitem == "invalidate_mt_cache" + cachequeued(nothing, 0) + i += 2 + continue + end + if nextitem ∈ ("jl_method_table_disable", "jl_method_table_insert") + cachequeued(nothing, 0) + push!(umis, item) + end + if isa(nextitem, Integer) + cachequeued(item, nextitem) + i += 2 + continue + end + end + end + if (isa(item, Method) || isa(item, Type)) && queued !== nothing + push!(umis, queued) + queued, depth = nothing, 0 + end + i += 1 + end + + # Process edge-validation events + i, ilast = firstindex(liste), lastindex(liste) + while i <= ilast + tag = liste[i + 1] # the tag is always second + if tag == "method_globalref" + push!(umis, Core.Compiler.get_ci_mi(liste[i + 2])) + i += 4 + elseif tag == "insert_backedges_callee" + push!(umis, Core.Compiler.get_ci_mi(liste[i + 2])) + i += 4 + elseif tag == "verify_methods" + push!(umis, Core.Compiler.get_ci_mi(liste[i])) + i += 3 + else + error("Unknown tag found in invalidation list: ", tag) + end + end + + return umis +end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl b/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl new file mode 100644 index 00000000..32d4e644 --- /dev/null +++ b/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl @@ -0,0 +1,125 @@ +const newly_inferred = Core.CodeInstance[] # only used to support verbose[] + +function workload_enabled(mod::Module) + try + if load_preference(@__MODULE__, "precompile_workloads", true) + return load_preference(mod, "precompile_workload", true) + else + return false + end + catch + true + end +end + +@noinline is_generating_output() = ccall(:jl_generating_output, Cint, ()) == 1 + +macro latestworld_if_toplevel() + Expr(Symbol("latestworld-if-toplevel")) +end + +function tag_newly_inferred_enable() + ccall(:jl_tag_newly_inferred_enable, Cvoid, ()) + if !Base.generating_output() # for verbose[] + ccall(:jl_set_newly_inferred, Cvoid, (Any,), newly_inferred) + end +end +function tag_newly_inferred_disable() + ccall(:jl_tag_newly_inferred_disable, Cvoid, ()) + if !Base.generating_output() # for verbose[] + ccall(:jl_set_newly_inferred, Cvoid, (Any,), nothing) + end + if verbose[] + for ci in newly_inferred + println(ci.def) + end + end + return nothing +end + +""" + @compile_workload f(args...) + +`precompile` (and save in the `compile_workload` file) any method-calls that occur inside the expression. All calls (direct or indirect) inside a +`@compile_workload` block will be cached. + +`@compile_workload` has three key features: + +1. code inside runs only when the package is being precompiled (i.e., a `*.ji` + precompile `compile_workload` file is being written) +2. the interpreter is disabled, ensuring your calls will be compiled +3. both direct and indirect callees will be precompiled, even for methods defined in other packages + and even for runtime-dispatched callees (requires Julia 1.8 and above). + +!!! note + For comprehensive precompilation, ensure the first usage of a given method/argument-type combination + occurs inside `@compile_workload`. + + In detail: runtime-dispatched callees are captured only when type-inference is executed, and they + are inferred only on first usage. Inferrable calls that trace back to a method defined in your package, + and their *inferrable* callees, will be precompiled regardless of "ownership" of the callees + (Julia 1.8 and higher). + + Consequently, this recommendation matters only for: + + - direct calls to methods defined in Base or other packages OR + - indirect runtime-dispatched calls to such methods. +""" +macro compile_workload(ex::Expr) + local iscompiling = :($PrecompileTools.is_generating_output() && $PrecompileTools.workload_enabled(@__MODULE__)) + ex = quote + begin + $PrecompileTools.@latestworld_if_toplevel # block inference from proceeding beyond this point (xref https://github.com/JuliaLang/julia/issues/57957) + $(esc(ex)) + end + end + ex = quote + $PrecompileTools.tag_newly_inferred_enable() + try + $ex + finally + $PrecompileTools.tag_newly_inferred_disable() + end + end + return quote + if $iscompiling || $PrecompileTools.verbose[] + $ex + end + end +end + +""" + @setup_workload begin + vars = ... + ⋮ + end + +Run the code block only during package precompilation. `@setup_workload` is often used in combination +with [`@compile_workload`](@ref), for example: + + @setup_workload begin + vars = ... + @compile_workload begin + y = f(vars...) + g(y) + ⋮ + end + end + +`@setup_workload` does not force compilation (though it may happen anyway) nor intentionally capture +runtime dispatches (though they will be precompiled anyway if the runtime-callee is for a method belonging +to your package). +""" +macro setup_workload(ex::Expr) + local iscompiling = :((ccall(:jl_generating_output, Cint, ()) == 1 && $PrecompileTools.workload_enabled(@__MODULE__))) + # Ideally we'd like a `let` around this to prevent namespace pollution, but that seem to + # trigger inference & codegen in undesirable ways (see #16). + return quote + if $iscompiling || $PrecompileTools.verbose[] + let + $PrecompileTools.@latestworld_if_toplevel # block inference from proceeding beyond this point (xref https://github.com/JuliaLang/julia/issues/57957) + $(esc(ex)) + end + end + end +end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl b/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl index 5868834b..4968d1b1 100644 --- a/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl +++ b/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl @@ -101,4 +101,7 @@ include("notebook_metadata.jl") include("manifest_validation.jl") include("python.jl") +include("PrecompileTools/PrecompileTools.jl") +include("precompile.jl") + end diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl new file mode 100644 index 00000000..12307b07 --- /dev/null +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -0,0 +1,3 @@ +PrecompileTools.@compile_workload begin + +end \ No newline at end of file From 3744c3c8b72bde2d85aa045268c4a97e7fcc33cc Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 11:27:00 +0200 Subject: [PATCH 02/21] disable preferences --- .../src/PrecompileTools/PrecompileTools.jl | 2 -- .../src/PrecompileTools/workloads.jl | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl index 3b6a8e5d..fefd5789 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl @@ -1,7 +1,5 @@ module PrecompileTools -using Preferences - export @setup_workload, @compile_workload, @recompile_invalidations const verbose = Ref(false) # if true, prints all the precompiles diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl b/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl index 32d4e644..8839fb04 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl @@ -1,15 +1,16 @@ const newly_inferred = Core.CodeInstance[] # only used to support verbose[] function workload_enabled(mod::Module) - try - if load_preference(@__MODULE__, "precompile_workloads", true) - return load_preference(mod, "precompile_workload", true) - else - return false - end - catch - true - end + # try + # if load_preference(@__MODULE__, "precompile_workloads", true) + # return load_preference(mod, "precompile_workload", true) + # else + # return false + # end + # catch + # true + # end + true # not using Preferences end @noinline is_generating_output() = ccall(:jl_generating_output, Cint, ()) == 1 From 93eb126f7a8d27969b12be38be4811af5710c630 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 12:26:07 +0200 Subject: [PATCH 03/21] add timeroutput measurements --- src/Malt.jl | 8 ++++++-- src/server.jl | 16 +++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Malt.jl b/src/Malt.jl index 03d966ef..22d31e9f 100644 --- a/src/Malt.jl +++ b/src/Malt.jl @@ -103,8 +103,11 @@ mutable struct Worker <: AbstractWorker env = vcat("MALT_WORKER_TEMP_DIR=$temp_dir", env) + Main.@timeit " run(exe)" run(`$exe -e '1 + 1'`) + # Spawn process cmd = _get_worker_cmd(; exe, env, exeflags) + Main.@timeit " worker cmd startup till port read" begin proc = open(Cmd(cmd; detach = true, windows_hide = true), "w+") # Keep internal list @@ -113,6 +116,7 @@ mutable struct Worker <: AbstractWorker # Block until reading the port number of the process (from its stdout) port_str = readline(proc) + end port = tryparse(UInt16, port_str) # Generate an error message for Julia version mismatches. This does @@ -120,7 +124,7 @@ mutable struct Worker <: AbstractWorker # the worker is connected to the socket and if there is a mismatch # we call `stop` to gracefully close the worker. We cannot # gracefully close it until that point. - manifest_file, manifest_error = + Main.@timeit "validate manifest" manifest_file, manifest_error = _validate_worker_process_manifest(metadata_toml_file, errors_log_file) if port === nothing @@ -196,7 +200,7 @@ mutable struct Worker <: AbstractWorker throw(UserError(manifest_error)) end - _manifest_in_sync_check(w) + Main.@timeit "manifest sync check" _manifest_in_sync_check(w) return w end diff --git a/src/server.jl b/src/server.jl index 3ed5c11f..acb135ec 100644 --- a/src/server.jl +++ b/src/server.jl @@ -28,7 +28,7 @@ mutable struct File timeout = _extract_timeout(merged_options) exe, _exeflags = _julia_exe(exeflags) - worker = cd( + Main.@timeit "Worker(...)" worker = cd( () -> Malt.Worker(; exe, exeflags = _exeflags, @@ -51,7 +51,7 @@ mutable struct File nothing, Channel{Symbol}(32), # we don't want an unbuffered channel because we might want to `put!` to it without blocking ) - init!(file, merged_options) + Main.@timeit "init!" init!(file, merged_options) return file else throw( @@ -1559,7 +1559,7 @@ function run!( result_task = Threads.@spawn begin try - evaluate!( + Main.@timeit "evaluate!" evaluate!( file, output; showprogress, @@ -1650,7 +1650,7 @@ function borrow_file!( if optionally_create # it's not ideal to create the `File` under server.lock but it takes a second or # so on my machine to init it, so for practical purposes it should be ok - file = server.workers[apath] = File(apath, options) + file = server.workers[apath] = Main.@timeit "File(...)" File(apath, options) lock(file.lock) # don't let anything get to the fresh file before us on_change(server) return true, file @@ -1713,9 +1713,11 @@ function render( output::Union{AbstractString,IO,Nothing} = nothing, showprogress::Bool = true, ) - server = Server() - run!(server, file; output, showprogress) - close!(server, file) + Main.reset_timer!() + Main.@timeit "Server()" server = Server() + Main.@timeit "run!" run!(server, file; output, showprogress) + Main.@timeit "close!" close!(server, file) + Main.print_timer(allocations = false) end function close!(server::Server) From b459c1cc8d21487e4b191903478865a355675df6 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:12:25 +0200 Subject: [PATCH 04/21] allow to inject a different module in precompilation --- src/QuartoNotebookWorker/src/NotebookState.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/QuartoNotebookWorker/src/NotebookState.jl b/src/QuartoNotebookWorker/src/NotebookState.jl index 24cc8fa3..d58587d1 100644 --- a/src/QuartoNotebookWorker/src/NotebookState.jl +++ b/src/QuartoNotebookWorker/src/NotebookState.jl @@ -47,7 +47,9 @@ function define_notebook_module!(root = Main) return mod end +const NotebookModuleForPrecompile = Base.RefValue{Union{Nothing,Module}}(nothing) + # `getfield` ends up throwing a segfault here, `getproperty` works fine though. -notebook_module() = Base.getproperty(Main, :Notebook)::Module +notebook_module() = NotebookModuleForPrecompile[] === nothing ? Base.getproperty(Main, :Notebook)::Module : NotebookModuleForPrecompile[] end From ec62fd9ea6331a0f6df360734f1b46c454bc1bc4 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:12:47 +0200 Subject: [PATCH 05/21] add precompile statements --- src/QuartoNotebookWorker/src/precompile.jl | 40 ++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl index 12307b07..394d4358 100644 --- a/src/QuartoNotebookWorker/src/precompile.jl +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -1,3 +1,39 @@ +precompile(Tuple{typeof(QuartoNotebookWorker.Malt.main)}) +precompile(Tuple{typeof(QuartoNotebookWorker.Malt._bson_deserialize), QuartoNotebookWorker.Malt.Sockets.TCPSocket}) + +for type in [Int,Float64,String,Nothing,Missing] + precompile( + Tuple{ + typeof(Core.kwcall), + NamedTuple{(:inline,), Tuple{Bool}}, + typeof(QuartoNotebookWorker.render_mimetypes), + type, + Base.Dict{String, Any} + } + ) +end + +precompile(Tuple{typeof(Base.Filesystem.mkpath), String}) +precompile(Tuple{typeof(QuartoNotebookWorker.refresh!), Base.Dict{String, Any}}) +precompile(Tuple{typeof(QuartoNotebookWorker.refresh!), Base.Dict{String, Any}, Base.Dict{String, Any}}) +precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:file, :line, :cell_options), Tuple{String, Int64, Base.Dict{String, Any}}}, typeof(QuartoNotebookWorker.include_str), Module, String}) + +module __PrecompilationModule end + +QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = __PrecompilationModule + PrecompileTools.@compile_workload begin - -end \ No newline at end of file + result = QuartoNotebookWorker.render( + "1 + 1", + "some_file", + 1, + Dict{String,Any}(); + inline = false, + ) + io = IOBuffer() + bson = QuartoNotebookWorker.Packages.BSON.bson(io, Dict{Symbol,Any}(:data => result)) + seekstart(io) + QuartoNotebookWorker.Packages.BSON.load(io)[:data] +end + +QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = nothing \ No newline at end of file From d2694bf4a0788ad9b9ea41442a6c0e30c05231af Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:13:01 +0200 Subject: [PATCH 06/21] don't need qnw env anymore --- src/Malt.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Malt.jl b/src/Malt.jl index 22d31e9f..9a735c24 100644 --- a/src/Malt.jl +++ b/src/Malt.jl @@ -394,7 +394,6 @@ const worker_package = RelocatableFolders.@path joinpath(@__DIR__, "QuartoNotebo function _get_worker_cmd(; exe, env, exeflags, file = String(startup_file)) defaults = Dict( "OPENBLAS_NUM_THREADS" => "1", - "QUARTONOTEBOOKWORKER_PACKAGE" => String(worker_package), ) env = vcat(Base.byteenv(defaults), Base.byteenv(env)) return addenv(`$exe --startup-file=no $exeflags $file`, env) From 03620af68a246d23b86eb30573b02121e7fddb9c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:13:19 +0200 Subject: [PATCH 07/21] add Scratch as dep --- Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Project.toml b/Project.toml index 1cd67639..9d825772 100644 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,7 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Scratch = "6c6a2e73-6563-6170-7368-637461726353" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" @@ -45,6 +46,7 @@ REPL = "1.6" Random = "1.6" RelocatableFolders = "1" SHA = "0.7, 1.6" +Scratch = "1" Sockets = "1.6" TOML = "1" YAML = "0.4" From 52ea287c5a8550c4f0dff577081141bcb1ec6906 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:13:41 +0200 Subject: [PATCH 08/21] use scratch env to store resolved env for worker --- src/QuartoNotebookRunner.jl | 1 + src/server.jl | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/QuartoNotebookRunner.jl b/src/QuartoNotebookRunner.jl index 2f7ac955..1ff8d260 100644 --- a/src/QuartoNotebookRunner.jl +++ b/src/QuartoNotebookRunner.jl @@ -32,6 +32,7 @@ import Preferences import ProgressLogging import REPL import Random +import Scratch import Sockets import SHA import TOML diff --git a/src/server.jl b/src/server.jl index acb135ec..4178c85f 100644 --- a/src/server.jl +++ b/src/server.jl @@ -28,11 +28,34 @@ mutable struct File timeout = _extract_timeout(merged_options) exe, _exeflags = _julia_exe(exeflags) + + qnw_env_dir = Scratch.@get_scratch!("quartonotebookworker-env") + + script = """ + qnw_env_dir::String = $(repr(qnw_env_dir)) + qnw_package_dir::String = $(repr(Malt.worker_package)) + + env_path = joinpath(qnw_env_dir, string(VERSION)) + env_proj = joinpath(env_path, "Project.toml") + env_mani = joinpath(env_path, "Manifest.toml") + + if !(isfile(env_proj) && isfile(env_mani)) + import Pkg + + Pkg.activate(env_path) + Pkg.develop(path = qnw_package_dir) + end + + println(env_path) + """ + + env_path = readchomp(`$exe --startup-file=no -e $script`) + Main.@timeit "Worker(...)" worker = cd( () -> Malt.Worker(; exe, exeflags = _exeflags, - env = vcat(env, quarto_env), + env = vcat(env, quarto_env, "QUARTONOTEBOOKWORKER_ENV=$env_path"), ), dirname(path), ) From ccbd7344e2f36f45bcae513cc5cfd99cb0676026 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:15:37 +0200 Subject: [PATCH 09/21] refactor startup file to make use of prepopulated scratch env --- src/startup.jl | 52 +++++++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/src/startup.jl b/src/startup.jl index 27b50f21..47deec62 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -13,11 +13,15 @@ function capture(func) try return func() catch err + bt = catch_backtrace() errors_log_file = joinpath(ENV["MALT_WORKER_TEMP_DIR"], "errors.log") - open(errors_log_file, "w") do io + io = open(errors_log_file, "w") + try showerror(io, err) - Base.show_backtrace(io, catch_backtrace()) + Base.inferencebarrier(Base.show_backtrace)(io, bt) flush(io) + finally + close(io) end exit() end @@ -45,9 +49,10 @@ pushfirst!(LOAD_PATH, "@stdlib") # and if a user was to perform `Pkg` operations they may affect that # environment. Instead we provide a temporary sandbox environment that gets # discarded when the notebook process exits. + let temp = mktempdir() sandbox = joinpath(temp, "QuartoSandbox") - mkpath(sandbox) + mkdir(sandbox) # The empty project file is key to making this the active environment if # noting else is available if the rest of the `LOAD_PATH`. touch(joinpath(sandbox, "Project.toml")) @@ -56,31 +61,8 @@ let temp = mktempdir() # Step 2b: # # We also need to ensure that the `QuartoNotebookWorker` package is - # available on the `LOAD_PATH`. This is done by creating another - # environment alongside the sandbox environment. We `Pkg.develop` the - # "local" `QuartoNotebookWorker` package into this environment. `Pkg` - # operations are logged to the `pkg.log` file that the server process can - # read to provide feedback to the user if needed. - # - # `Pkg` is loaded outside of this closure otherwise the methods required do - # not exist in a new enough world age to be callable. - Pkg = Base.require(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) - capture() do - worker = joinpath(temp, "QuartoNotebookWorker") - mkpath(worker) - push!(LOAD_PATH, worker) - open(joinpath(ENV["MALT_WORKER_TEMP_DIR"], "pkg.log"), "w") do io - ap = Base.active_project() - try - Pkg.activate(worker; io) - Pkg.develop(; path = ENV["QUARTONOTEBOOKWORKER_PACKAGE"], io) - finally - # Ensure that we switch the active project back afterwards. - Pkg.activate(ap; io) - end - flush(io) - end - end + # available on the `LOAD_PATH`. + push!(LOAD_PATH, ENV["QUARTONOTEBOOKWORKER_ENV"]) end # Step 3: @@ -94,9 +76,10 @@ end # instead manually write the strings so that this can happen prior to any # stdlib loading, which could trigger errors that we would then want this # metadata to be able to properly inform the user about. -capture() do +function save_metadata() metadata_toml_file = joinpath(ENV["MALT_WORKER_TEMP_DIR"], "metadata.toml") - open(metadata_toml_file, "w") do io + io = open(metadata_toml_file, "w") + try project_toml_file = Base.active_project() if !isnothing(project_toml_file) && isfile(project_toml_file) println(io, "project = $(repr(project_toml_file))") @@ -107,8 +90,11 @@ capture() do end println(io, "julia_version = $(repr(string(VERSION)))") flush(io) + finally + close(io) end end +capture(save_metadata) # Step: 4 # @@ -119,18 +105,19 @@ end # should do a manual `import Revise` in their notebook if they need `Revise` # support. const QUARTO_ENABLE_REVISE = get(ENV, "QUARTO_ENABLE_REVISE", "false") == "true" -capture() do +function require_revise() if QUARTO_ENABLE_REVISE pkgid = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise") Base.require(pkgid) end end +capture(require_revise) # Step 5: # # Now load in the worker package. This may trigger package precompilation on # first load, hence it is run under a `capture` should it fail to run. -const QuartoNotebookWorker = capture() do +function require_qnw() Base.require( Base.PkgId( Base.UUID("38328d9c-a911-4051-bc06-3f7f556ffeda"), @@ -138,6 +125,7 @@ const QuartoNotebookWorker = capture() do ), ) end +const QuartoNotebookWorker = capture(require_qnw) # Step 6: # From a31c53498387350685de7a4f8a560172e9261ef6 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:19:24 +0200 Subject: [PATCH 10/21] include worker package location in scratch hash so we don't use stale locations for some reason --- src/server.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.jl b/src/server.jl index 4178c85f..19c02381 100644 --- a/src/server.jl +++ b/src/server.jl @@ -29,7 +29,7 @@ mutable struct File exe, _exeflags = _julia_exe(exeflags) - qnw_env_dir = Scratch.@get_scratch!("quartonotebookworker-env") + qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") script = """ qnw_env_dir::String = $(repr(qnw_env_dir)) From 2e17043a19ee331637c4f20a5f359ab43522e02b Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:28:40 +0200 Subject: [PATCH 11/21] remove precompiletools again for now, failed when switching notebook to 1.11 again --- .../src/PrecompileTools/PrecompileTools.jl | 1 - .../src/PrecompileTools/invalidations.jl | 93 ------------------- src/QuartoNotebookWorker/src/precompile.jl | 4 +- 3 files changed, 3 insertions(+), 95 deletions(-) delete mode 100644 src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl index fefd5789..9cc66de0 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl @@ -11,6 +11,5 @@ function precompile_mi(mi::Core.MethodInstance) end include("workloads.jl") -include("invalidations.jl") end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl b/src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl deleted file mode 100644 index f90a1c8a..00000000 --- a/src/QuartoNotebookWorker/src/PrecompileTools/invalidations.jl +++ /dev/null @@ -1,93 +0,0 @@ -""" - @recompile_invalidations begin - using PkgA - ⋮ - end - -Recompile any invalidations that occur within the given expression. This is generally intended to be used -by users in creating "Startup" packages to ensure that the code compiled by package authors is not invalidated. -""" -macro recompile_invalidations(expr) - # use QuoteNode instead of esc(Expr(:quote)) so that $ is not permitted as usual (instead of having this macro work like `@eval`) - return :(recompile_invalidations($__module__, $(QuoteNode(expr)))) -end - -const ReinferUtils = isdefined(Base, :ReinferUtils) ? Base.ReinferUtils : Base.StaticData - -function recompile_invalidations(__module__::Module, @nospecialize expr) - listi = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) - liste = ReinferUtils.debug_method_invalidation(true) - try - Core.eval(__module__, expr) - finally - ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) - ReinferUtils.debug_method_invalidation(false) - end - if ccall(:jl_generating_output, Cint, ()) == 1 - foreach(precompile_mi, invalidation_leaves(listi, liste)) - end - nothing -end - -function invalidation_leaves(listi, liste) - umis = Set{Core.MethodInstance}() - # `queued` is a queue of length 0 or 1 of invalidated MethodInstances. - # We wait to read the `depth` to find out if it's a leaf. - queued, depth = nothing, 0 - function cachequeued(item, nextdepth) - if queued !== nothing && nextdepth <= depth - push!(umis, queued) - end - queued, depth = item, nextdepth - end - - # Process method insertion/deletion events - i, ilast = firstindex(listi), lastindex(listi) - while i <= ilast - item = listi[i] - if isa(item, Core.MethodInstance) - if i < lastindex(listi) - nextitem = listi[i+1] - if nextitem == "invalidate_mt_cache" - cachequeued(nothing, 0) - i += 2 - continue - end - if nextitem ∈ ("jl_method_table_disable", "jl_method_table_insert") - cachequeued(nothing, 0) - push!(umis, item) - end - if isa(nextitem, Integer) - cachequeued(item, nextitem) - i += 2 - continue - end - end - end - if (isa(item, Method) || isa(item, Type)) && queued !== nothing - push!(umis, queued) - queued, depth = nothing, 0 - end - i += 1 - end - - # Process edge-validation events - i, ilast = firstindex(liste), lastindex(liste) - while i <= ilast - tag = liste[i + 1] # the tag is always second - if tag == "method_globalref" - push!(umis, Core.Compiler.get_ci_mi(liste[i + 2])) - i += 4 - elseif tag == "insert_backedges_callee" - push!(umis, Core.Compiler.get_ci_mi(liste[i + 2])) - i += 4 - elseif tag == "verify_methods" - push!(umis, Core.Compiler.get_ci_mi(liste[i])) - i += 3 - else - error("Unknown tag found in invalidation list: ", tag) - end - end - - return umis -end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl index 394d4358..59539559 100644 --- a/src/QuartoNotebookWorker/src/precompile.jl +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -22,7 +22,8 @@ module __PrecompilationModule end QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = __PrecompilationModule -PrecompileTools.@compile_workload begin +# PrecompileTools.@compile_workload begin +let # the copying of PrecompileTools source did not just work on 1.11 result = QuartoNotebookWorker.render( "1 + 1", "some_file", @@ -35,5 +36,6 @@ PrecompileTools.@compile_workload begin seekstart(io) QuartoNotebookWorker.Packages.BSON.load(io)[:data] end +# end QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = nothing \ No newline at end of file From 3fa3eb10e6d9e63b51e38a83252b69f618f977c9 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:31:34 +0200 Subject: [PATCH 12/21] remove timeroutput statements --- src/Malt.jl | 8 ++------ src/server.jl | 16 +++++++--------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/Malt.jl b/src/Malt.jl index 9a735c24..fc899439 100644 --- a/src/Malt.jl +++ b/src/Malt.jl @@ -103,11 +103,8 @@ mutable struct Worker <: AbstractWorker env = vcat("MALT_WORKER_TEMP_DIR=$temp_dir", env) - Main.@timeit " run(exe)" run(`$exe -e '1 + 1'`) - # Spawn process cmd = _get_worker_cmd(; exe, env, exeflags) - Main.@timeit " worker cmd startup till port read" begin proc = open(Cmd(cmd; detach = true, windows_hide = true), "w+") # Keep internal list @@ -116,7 +113,6 @@ mutable struct Worker <: AbstractWorker # Block until reading the port number of the process (from its stdout) port_str = readline(proc) - end port = tryparse(UInt16, port_str) # Generate an error message for Julia version mismatches. This does @@ -124,7 +120,7 @@ mutable struct Worker <: AbstractWorker # the worker is connected to the socket and if there is a mismatch # we call `stop` to gracefully close the worker. We cannot # gracefully close it until that point. - Main.@timeit "validate manifest" manifest_file, manifest_error = + manifest_file, manifest_error = _validate_worker_process_manifest(metadata_toml_file, errors_log_file) if port === nothing @@ -200,7 +196,7 @@ mutable struct Worker <: AbstractWorker throw(UserError(manifest_error)) end - Main.@timeit "manifest sync check" _manifest_in_sync_check(w) + _manifest_in_sync_check(w) return w end diff --git a/src/server.jl b/src/server.jl index 19c02381..465153ce 100644 --- a/src/server.jl +++ b/src/server.jl @@ -51,7 +51,7 @@ mutable struct File env_path = readchomp(`$exe --startup-file=no -e $script`) - Main.@timeit "Worker(...)" worker = cd( + worker = cd( () -> Malt.Worker(; exe, exeflags = _exeflags, @@ -74,7 +74,7 @@ mutable struct File nothing, Channel{Symbol}(32), # we don't want an unbuffered channel because we might want to `put!` to it without blocking ) - Main.@timeit "init!" init!(file, merged_options) + init!(file, merged_options) return file else throw( @@ -1582,7 +1582,7 @@ function run!( result_task = Threads.@spawn begin try - Main.@timeit "evaluate!" evaluate!( + evaluate!( file, output; showprogress, @@ -1673,7 +1673,7 @@ function borrow_file!( if optionally_create # it's not ideal to create the `File` under server.lock but it takes a second or # so on my machine to init it, so for practical purposes it should be ok - file = server.workers[apath] = Main.@timeit "File(...)" File(apath, options) + file = server.workers[apath] = File(apath, options) lock(file.lock) # don't let anything get to the fresh file before us on_change(server) return true, file @@ -1736,11 +1736,9 @@ function render( output::Union{AbstractString,IO,Nothing} = nothing, showprogress::Bool = true, ) - Main.reset_timer!() - Main.@timeit "Server()" server = Server() - Main.@timeit "run!" run!(server, file; output, showprogress) - Main.@timeit "close!" close!(server, file) - Main.print_timer(allocations = false) + server = Server() + run!(server, file; output, showprogress) + close!(server, file) end function close!(server::Server) From 350d82a95d25c431e9fb7e26229c29007779ced9 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:35:54 +0200 Subject: [PATCH 13/21] remove type assertions that are now useless and break 1.6 --- src/server.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.jl b/src/server.jl index 465153ce..29d07971 100644 --- a/src/server.jl +++ b/src/server.jl @@ -32,8 +32,8 @@ mutable struct File qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") script = """ - qnw_env_dir::String = $(repr(qnw_env_dir)) - qnw_package_dir::String = $(repr(Malt.worker_package)) + qnw_env_dir = $(repr(qnw_env_dir)) + qnw_package_dir = $(repr(Malt.worker_package)) env_path = joinpath(qnw_env_dir, string(VERSION)) env_proj = joinpath(env_path, "Project.toml") From e30c5e54965219c3c0a267d276673f2bd0f61d55 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 14:38:42 +0200 Subject: [PATCH 14/21] kwcall is a 1.9 method --- src/QuartoNotebookWorker/src/precompile.jl | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl index 59539559..3be8e129 100644 --- a/src/QuartoNotebookWorker/src/precompile.jl +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -1,22 +1,24 @@ precompile(Tuple{typeof(QuartoNotebookWorker.Malt.main)}) precompile(Tuple{typeof(QuartoNotebookWorker.Malt._bson_deserialize), QuartoNotebookWorker.Malt.Sockets.TCPSocket}) -for type in [Int,Float64,String,Nothing,Missing] - precompile( - Tuple{ - typeof(Core.kwcall), - NamedTuple{(:inline,), Tuple{Bool}}, - typeof(QuartoNotebookWorker.render_mimetypes), - type, - Base.Dict{String, Any} - } - ) +if VERSION >= v"1.9" + for type in [Int,Float64,String,Nothing,Missing] + precompile( + Tuple{ + typeof(Core.kwcall), + NamedTuple{(:inline,), Tuple{Bool}}, + typeof(QuartoNotebookWorker.render_mimetypes), + type, + Base.Dict{String, Any} + } + ) + end + precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:file, :line, :cell_options), Tuple{String, Int64, Base.Dict{String, Any}}}, typeof(QuartoNotebookWorker.include_str), Module, String}) end precompile(Tuple{typeof(Base.Filesystem.mkpath), String}) precompile(Tuple{typeof(QuartoNotebookWorker.refresh!), Base.Dict{String, Any}}) precompile(Tuple{typeof(QuartoNotebookWorker.refresh!), Base.Dict{String, Any}, Base.Dict{String, Any}}) -precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:file, :line, :cell_options), Tuple{String, Int64, Base.Dict{String, Any}}}, typeof(QuartoNotebookWorker.include_str), Module, String}) module __PrecompilationModule end From 3596b84da6f4993fe4ac61a0836f0f37af72ebb2 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 29 Sep 2025 15:26:54 +0200 Subject: [PATCH 15/21] maybe need to specifically add stdlib? --- src/server.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server.jl b/src/server.jl index 29d07971..001a281c 100644 --- a/src/server.jl +++ b/src/server.jl @@ -32,6 +32,8 @@ mutable struct File qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") script = """ + pushfirst!(LOAD_PATH, "@stdlib") + qnw_env_dir = $(repr(qnw_env_dir)) qnw_package_dir = $(repr(Malt.worker_package)) From 219cea2ab3ad2db7e6fe424492c2528bca5f7c5c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 30 Sep 2025 10:25:35 +0200 Subject: [PATCH 16/21] remove unnecessary second julia invocation --- src/server.jl | 24 +----------------------- src/startup.jl | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/server.jl b/src/server.jl index 001a281c..a6c482ce 100644 --- a/src/server.jl +++ b/src/server.jl @@ -31,33 +31,11 @@ mutable struct File qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") - script = """ - pushfirst!(LOAD_PATH, "@stdlib") - - qnw_env_dir = $(repr(qnw_env_dir)) - qnw_package_dir = $(repr(Malt.worker_package)) - - env_path = joinpath(qnw_env_dir, string(VERSION)) - env_proj = joinpath(env_path, "Project.toml") - env_mani = joinpath(env_path, "Manifest.toml") - - if !(isfile(env_proj) && isfile(env_mani)) - import Pkg - - Pkg.activate(env_path) - Pkg.develop(path = qnw_package_dir) - end - - println(env_path) - """ - - env_path = readchomp(`$exe --startup-file=no -e $script`) - worker = cd( () -> Malt.Worker(; exe, exeflags = _exeflags, - env = vcat(env, quarto_env, "QUARTONOTEBOOKWORKER_ENV=$env_path"), + env = vcat(env, quarto_env, "QUARTONOTEBOOKWORKER_ENV_DIR=$qnw_env_dir", "QUARTONOTEBOOKWORKER_PACKAGE_DIR=$(Malt.worker_package)"), ), dirname(path), ) diff --git a/src/startup.jl b/src/startup.jl index 47deec62..f00fbfd5 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -57,14 +57,41 @@ let temp = mktempdir() # noting else is available if the rest of the `LOAD_PATH`. touch(joinpath(sandbox, "Project.toml")) push!(LOAD_PATH, sandbox) +end + +# Step 2b: +# +# We also need to ensure that the `QuartoNotebookWorker` package is +# available on the `LOAD_PATH`. + +qnw_env_dir = ENV["QUARTONOTEBOOKWORKER_ENV_DIR"] +qnw_package_dir = ENV["QUARTONOTEBOOKWORKER_PACKAGE_DIR"] + +env_path = joinpath(qnw_env_dir, string(VERSION)) +env_proj = joinpath(env_path, "Project.toml") +env_mani = joinpath(env_path, "Manifest.toml") - # Step 2b: - # - # We also need to ensure that the `QuartoNotebookWorker` package is - # available on the `LOAD_PATH`. - push!(LOAD_PATH, ENV["QUARTONOTEBOOKWORKER_ENV"]) +if !(isfile(env_proj) && isfile(env_mani)) + # `Pkg` is loaded outside of this closure otherwise the methods required do + # not exist in a new enough world age to be callable. + Pkg = Base.require(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) + capture() do + open(joinpath(ENV["MALT_WORKER_TEMP_DIR"], "pkg.log"), "w") do io + ap = Base.active_project() + try + Pkg.activate(env_path; io) + Pkg.develop(; path = qnw_package_dir, io) + finally + # Ensure that we switch the active project back afterwards. + Pkg.activate(ap; io) + end + flush(io) + end + end end +push!(LOAD_PATH, env_path) + # Step 3: # # The parent process needs some additional metadata about this `julia` process to From ddf0b710742f0629f92084664a01d227a79d8468 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 30 Sep 2025 10:47:46 +0200 Subject: [PATCH 17/21] add env variables on refresh, too --- src/server.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/server.jl b/src/server.jl index a6c482ce..62abdc98 100644 --- a/src/server.jl +++ b/src/server.jl @@ -29,13 +29,11 @@ mutable struct File exe, _exeflags = _julia_exe(exeflags) - qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") - worker = cd( () -> Malt.Worker(; exe, exeflags = _exeflags, - env = vcat(env, quarto_env, "QUARTONOTEBOOKWORKER_ENV_DIR=$qnw_env_dir", "QUARTONOTEBOOKWORKER_PACKAGE_DIR=$(Malt.worker_package)"), + env = vcat(env, quarto_env, startup_env_variables()), ), dirname(path), ) @@ -69,6 +67,11 @@ mutable struct File end end +function startup_env_variables() + qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") + return ["QUARTONOTEBOOKWORKER_ENV_DIR=$qnw_env_dir", "QUARTONOTEBOOKWORKER_PACKAGE_DIR=$(Malt.worker_package)"] +end + struct SourceRange file::Union{String,Nothing} lines::UnitRange{Int} @@ -240,7 +243,7 @@ function refresh!(file::File, options::Dict) Malt.stop(file.worker) exe, _exeflags = _julia_exe(exeflags) file.worker = cd( - () -> Malt.Worker(; exe, exeflags = _exeflags, env = vcat(env, quarto_env)), + () -> Malt.Worker(; exe, exeflags = _exeflags, env = vcat(env, quarto_env, startup_env_variables())), dirname(file.path), ) file.exe = exe @@ -1578,7 +1581,6 @@ function run!( # block until a decision is reached decision = take!(file.run_decision_channel) - # :forceclose might have been set from another task if decision === :forceclose close!(server, file.path) # this is in the same task, so reentrant lock allows access From c069346e594be5ba918bfbb35a7e6c5d5f846dbd Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 30 Sep 2025 11:10:02 +0200 Subject: [PATCH 18/21] add precompiletools in two versions --- .../PrecompileTools.jl | 0 .../workloads.jl | 0 .../PrecompileTools.jl | 17 +++ .../src/PrecompileTools-pre1.12/workloads.jl | 144 ++++++++++++++++++ .../src/QuartoNotebookWorker.jl | 6 +- src/QuartoNotebookWorker/src/precompile.jl | 28 ++-- 6 files changed, 180 insertions(+), 15 deletions(-) rename src/QuartoNotebookWorker/src/{PrecompileTools => PrecompileTools-post1.12}/PrecompileTools.jl (100%) rename src/QuartoNotebookWorker/src/{PrecompileTools => PrecompileTools-post1.12}/workloads.jl (100%) create mode 100644 src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl create mode 100644 src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/PrecompileTools.jl similarity index 100% rename from src/QuartoNotebookWorker/src/PrecompileTools/PrecompileTools.jl rename to src/QuartoNotebookWorker/src/PrecompileTools-post1.12/PrecompileTools.jl diff --git a/src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl b/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/workloads.jl similarity index 100% rename from src/QuartoNotebookWorker/src/PrecompileTools/workloads.jl rename to src/QuartoNotebookWorker/src/PrecompileTools-post1.12/workloads.jl diff --git a/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl new file mode 100644 index 00000000..dfa0b7da --- /dev/null +++ b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl @@ -0,0 +1,17 @@ +module PrecompileTools + +export @setup_workload, @compile_workload, @recompile_invalidations + +const verbose = Ref(false) # if true, prints all the precompiles +const have_inference_tracking = isdefined(Core.Compiler, :__set_measure_typeinf) +const have_force_compile = isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("#@force_compile")) + +function precompile_mi(mi) + precompile(mi.specTypes) # TODO: Julia should allow one to pass `mi` directly (would handle `invoke` properly) + verbose[] && println(mi) + return +end + +include("workloads.jl") + +end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl new file mode 100644 index 00000000..3d5a29e7 --- /dev/null +++ b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl @@ -0,0 +1,144 @@ + +function workload_enabled(mod::Module) + # try + # if load_preference(@__MODULE__, "precompile_workloads", true) + # return load_preference(mod, "precompile_workload", true) + # else + # return false + # end + # catch + # true + # end + return true +end + +""" + check_edges(node) + +Recursively ensure that all callees of `node` are precompiled. This is (rarely) necessary +because sometimes there is no backedge from callee to caller (xref https://github.com/JuliaLang/julia/issues/49617), +and `staticdata.c` relies on the backedge to trace back to a MethodInstance that is tagged `mi.precompiled`. +""" +function check_edges(node) + parentmi = node.mi_info.mi + for child in node.children + childmi = child.mi_info.mi + if !(isdefined(childmi, :backedges) && parentmi ∈ childmi.backedges) + precompile_mi(childmi) + end + check_edges(child) + end +end + +function precompile_roots(roots) + @assert have_inference_tracking + for child in roots + precompile_mi(child.mi_info.mi) + check_edges(child) + end +end + +""" + @compile_workload f(args...) + +`precompile` (and save in the compile_workload file) any method-calls that occur inside the expression. All calls (direct or indirect) inside a +`@compile_workload` block will be cached. + +`@compile_workload` has three key features: + +1. code inside runs only when the package is being precompiled (i.e., a `*.ji` + precompile compile_workload file is being written) +2. the interpreter is disabled, ensuring your calls will be compiled +3. both direct and indirect callees will be precompiled, even for methods defined in other packages + and even for runtime-dispatched callees (requires Julia 1.8 and above). + +!!! note + For comprehensive precompilation, ensure the first usage of a given method/argument-type combination + occurs inside `@compile_workload`. + + In detail: runtime-dispatched callees are captured only when type-inference is executed, and they + are inferred only on first usage. Inferrable calls that trace back to a method defined in your package, + and their *inferrable* callees, will be precompiled regardless of "ownership" of the callees + (Julia 1.8 and higher). + + Consequently, this recommendation matters only for: + + - direct calls to methods defined in Base or other packages OR + - indirect runtime-dispatched calls to such methods. +""" +macro compile_workload(ex::Expr) + local iscompiling = if Base.VERSION < v"1.6" + :(ccall(:jl_generating_output, Cint, ()) == 1) + else + :((ccall(:jl_generating_output, Cint, ()) == 1 && $PrecompileTools.workload_enabled(@__MODULE__))) + end + if have_force_compile + ex = quote + begin + Base.Experimental.@force_compile + $(esc(ex)) + end + end + else + # Use the hack on earlier Julia versions that blocks the interpreter + ex = quote + while false end + $(esc(ex)) + end + end + if have_inference_tracking + ex = quote + Core.Compiler.Timings.reset_timings() + Core.Compiler.__set_measure_typeinf(true) + try + $ex + finally + Core.Compiler.__set_measure_typeinf(false) + Core.Compiler.Timings.close_current_timer() + end + $PrecompileTools.precompile_roots(Core.Compiler.Timings._timings[1].children) + end + end + return quote + if $iscompiling || $PrecompileTools.verbose[] + $ex + end + end +end + +""" + @setup_workload begin + vars = ... + ⋮ + end + +Run the code block only during package precompilation. `@setup_workload` is often used in combination +with [`@compile_workload`](@ref), for example: + + @setup_workload begin + vars = ... + @compile_workload begin + y = f(vars...) + g(y) + ⋮ + end + end + +`@setup_workload` does not force compilation (though it may happen anyway) nor intentionally capture +runtime dispatches (though they will be precompiled anyway if the runtime-callee is for a method belonging +to your package). +""" +macro setup_workload(ex::Expr) + local iscompiling = if Base.VERSION < v"1.6" + :(ccall(:jl_generating_output, Cint, ()) == 1) + else + :((ccall(:jl_generating_output, Cint, ()) == 1 && $PrecompileTools.workload_enabled(@__MODULE__))) + end + # Ideally we'd like a `let` around this to prevent namespace pollution, but that seem to + # trigger inference & codegen in undesirable ways (see #16). + return quote + if $iscompiling || $PrecompileTools.verbose[] + $(esc(ex)) + end + end +end \ No newline at end of file diff --git a/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl b/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl index 4968d1b1..f35594bf 100644 --- a/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl +++ b/src/QuartoNotebookWorker/src/QuartoNotebookWorker.jl @@ -101,7 +101,11 @@ include("notebook_metadata.jl") include("manifest_validation.jl") include("python.jl") -include("PrecompileTools/PrecompileTools.jl") +if VERSION >= v"1.12-rc1" + include("PrecompileTools-post1.12/PrecompileTools.jl") +else + include("PrecompileTools-pre1.12/PrecompileTools.jl") +end include("precompile.jl") end diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl index 3be8e129..d228c6c2 100644 --- a/src/QuartoNotebookWorker/src/precompile.jl +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -24,20 +24,20 @@ module __PrecompilationModule end QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = __PrecompilationModule -# PrecompileTools.@compile_workload begin -let # the copying of PrecompileTools source did not just work on 1.11 - result = QuartoNotebookWorker.render( - "1 + 1", - "some_file", - 1, - Dict{String,Any}(); - inline = false, - ) - io = IOBuffer() - bson = QuartoNotebookWorker.Packages.BSON.bson(io, Dict{Symbol,Any}(:data => result)) - seekstart(io) - QuartoNotebookWorker.Packages.BSON.load(io)[:data] +PrecompileTools.@compile_workload begin + for code in ["1 + 1", "println(\"abc\")", "error()"] + result = QuartoNotebookWorker.render( + code, + "some_file", + 1, + Dict{String,Any}("error" => "true"); + inline = false, + ) + io = IOBuffer() + bson = QuartoNotebookWorker.Packages.BSON.bson(io, Dict{Symbol,Any}(:data => result)) + seekstart(io) + QuartoNotebookWorker.Packages.BSON.load(io)[:data] + end end -# end QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = nothing \ No newline at end of file From 25c934badccdb4a670ed326e0bd6e024968f24bc Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 30 Sep 2025 11:13:34 +0200 Subject: [PATCH 19/21] one more precompile --- src/QuartoNotebookWorker/src/precompile.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl index d228c6c2..e17e844c 100644 --- a/src/QuartoNotebookWorker/src/precompile.jl +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -25,7 +25,7 @@ module __PrecompilationModule end QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = __PrecompilationModule PrecompileTools.@compile_workload begin - for code in ["1 + 1", "println(\"abc\")", "error()"] + for code in ["1 + 1", "println(\"abc\")", "error()", "@info \"info text\" value=1"] result = QuartoNotebookWorker.render( code, "some_file", From 4662c66a66b638879d255616e4ac9aac3c76204f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 1 Oct 2025 10:54:53 +0200 Subject: [PATCH 20/21] formatting --- src/Malt.jl | 4 +- src/QuartoNotebookWorker/src/NotebookState.jl | 4 +- .../PrecompileTools.jl | 2 +- .../src/PrecompileTools-post1.12/workloads.jl | 12 +++-- .../PrecompileTools.jl | 6 ++- .../src/PrecompileTools-pre1.12/workloads.jl | 15 +++++-- src/QuartoNotebookWorker/src/precompile.jl | 45 ++++++++++++++----- src/server.jl | 11 ++++- 8 files changed, 72 insertions(+), 27 deletions(-) diff --git a/src/Malt.jl b/src/Malt.jl index fc899439..b762cc67 100644 --- a/src/Malt.jl +++ b/src/Malt.jl @@ -388,9 +388,7 @@ const empty_file = RelocatableFolders.@path joinpath(@__DIR__, "empty.jl") const worker_package = RelocatableFolders.@path joinpath(@__DIR__, "QuartoNotebookWorker") function _get_worker_cmd(; exe, env, exeflags, file = String(startup_file)) - defaults = Dict( - "OPENBLAS_NUM_THREADS" => "1", - ) + defaults = Dict("OPENBLAS_NUM_THREADS" => "1") env = vcat(Base.byteenv(defaults), Base.byteenv(env)) return addenv(`$exe --startup-file=no $exeflags $file`, env) end diff --git a/src/QuartoNotebookWorker/src/NotebookState.jl b/src/QuartoNotebookWorker/src/NotebookState.jl index d58587d1..a0589275 100644 --- a/src/QuartoNotebookWorker/src/NotebookState.jl +++ b/src/QuartoNotebookWorker/src/NotebookState.jl @@ -50,6 +50,8 @@ end const NotebookModuleForPrecompile = Base.RefValue{Union{Nothing,Module}}(nothing) # `getfield` ends up throwing a segfault here, `getproperty` works fine though. -notebook_module() = NotebookModuleForPrecompile[] === nothing ? Base.getproperty(Main, :Notebook)::Module : NotebookModuleForPrecompile[] +notebook_module() = + NotebookModuleForPrecompile[] === nothing ? Base.getproperty(Main, :Notebook)::Module : + NotebookModuleForPrecompile[] end diff --git a/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/PrecompileTools.jl index 9cc66de0..6ea696bc 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/PrecompileTools.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/PrecompileTools.jl @@ -12,4 +12,4 @@ end include("workloads.jl") -end \ No newline at end of file +end diff --git a/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/workloads.jl b/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/workloads.jl index 8839fb04..047e3e0c 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/workloads.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools-post1.12/workloads.jl @@ -67,7 +67,10 @@ end - indirect runtime-dispatched calls to such methods. """ macro compile_workload(ex::Expr) - local iscompiling = :($PrecompileTools.is_generating_output() && $PrecompileTools.workload_enabled(@__MODULE__)) + local iscompiling = :( + $PrecompileTools.is_generating_output() && + $PrecompileTools.workload_enabled(@__MODULE__) + ) ex = quote begin $PrecompileTools.@latestworld_if_toplevel # block inference from proceeding beyond this point (xref https://github.com/JuliaLang/julia/issues/57957) @@ -112,7 +115,10 @@ runtime dispatches (though they will be precompiled anyway if the runtime-callee to your package). """ macro setup_workload(ex::Expr) - local iscompiling = :((ccall(:jl_generating_output, Cint, ()) == 1 && $PrecompileTools.workload_enabled(@__MODULE__))) + local iscompiling = :(( + ccall(:jl_generating_output, Cint, ()) == 1 && + $PrecompileTools.workload_enabled(@__MODULE__) + )) # Ideally we'd like a `let` around this to prevent namespace pollution, but that seem to # trigger inference & codegen in undesirable ways (see #16). return quote @@ -123,4 +129,4 @@ macro setup_workload(ex::Expr) end end end -end \ No newline at end of file +end diff --git a/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl index dfa0b7da..ae901a16 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/PrecompileTools.jl @@ -4,7 +4,9 @@ export @setup_workload, @compile_workload, @recompile_invalidations const verbose = Ref(false) # if true, prints all the precompiles const have_inference_tracking = isdefined(Core.Compiler, :__set_measure_typeinf) -const have_force_compile = isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("#@force_compile")) +const have_force_compile = + isdefined(Base, :Experimental) && + isdefined(Base.Experimental, Symbol("#@force_compile")) function precompile_mi(mi) precompile(mi.specTypes) # TODO: Julia should allow one to pass `mi` directly (would handle `invoke` properly) @@ -14,4 +16,4 @@ end include("workloads.jl") -end \ No newline at end of file +end diff --git a/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl index 3d5a29e7..19b304fa 100644 --- a/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl +++ b/src/QuartoNotebookWorker/src/PrecompileTools-pre1.12/workloads.jl @@ -70,7 +70,10 @@ macro compile_workload(ex::Expr) local iscompiling = if Base.VERSION < v"1.6" :(ccall(:jl_generating_output, Cint, ()) == 1) else - :((ccall(:jl_generating_output, Cint, ()) == 1 && $PrecompileTools.workload_enabled(@__MODULE__))) + :(( + ccall(:jl_generating_output, Cint, ()) == 1 && + $PrecompileTools.workload_enabled(@__MODULE__) + )) end if have_force_compile ex = quote @@ -82,7 +85,8 @@ macro compile_workload(ex::Expr) else # Use the hack on earlier Julia versions that blocks the interpreter ex = quote - while false end + while false + end $(esc(ex)) end end @@ -132,7 +136,10 @@ macro setup_workload(ex::Expr) local iscompiling = if Base.VERSION < v"1.6" :(ccall(:jl_generating_output, Cint, ()) == 1) else - :((ccall(:jl_generating_output, Cint, ()) == 1 && $PrecompileTools.workload_enabled(@__MODULE__))) + :(( + ccall(:jl_generating_output, Cint, ()) == 1 && + $PrecompileTools.workload_enabled(@__MODULE__) + )) end # Ideally we'd like a `let` around this to prevent namespace pollution, but that seem to # trigger inference & codegen in undesirable ways (see #16). @@ -141,4 +148,4 @@ macro setup_workload(ex::Expr) $(esc(ex)) end end -end \ No newline at end of file +end diff --git a/src/QuartoNotebookWorker/src/precompile.jl b/src/QuartoNotebookWorker/src/precompile.jl index e17e844c..d2f50847 100644 --- a/src/QuartoNotebookWorker/src/precompile.jl +++ b/src/QuartoNotebookWorker/src/precompile.jl @@ -1,24 +1,46 @@ precompile(Tuple{typeof(QuartoNotebookWorker.Malt.main)}) -precompile(Tuple{typeof(QuartoNotebookWorker.Malt._bson_deserialize), QuartoNotebookWorker.Malt.Sockets.TCPSocket}) +precompile( + Tuple{ + typeof(QuartoNotebookWorker.Malt._bson_deserialize), + QuartoNotebookWorker.Malt.Sockets.TCPSocket, + }, +) if VERSION >= v"1.9" - for type in [Int,Float64,String,Nothing,Missing] + for type in [Int, Float64, String, Nothing, Missing] precompile( Tuple{ typeof(Core.kwcall), - NamedTuple{(:inline,), Tuple{Bool}}, + NamedTuple{(:inline,),Tuple{Bool}}, typeof(QuartoNotebookWorker.render_mimetypes), type, - Base.Dict{String, Any} - } + Base.Dict{String,Any}, + }, ) end - precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:file, :line, :cell_options), Tuple{String, Int64, Base.Dict{String, Any}}}, typeof(QuartoNotebookWorker.include_str), Module, String}) + precompile( + Tuple{ + typeof(Core.kwcall), + NamedTuple{ + (:file, :line, :cell_options), + Tuple{String,Int64,Base.Dict{String,Any}}, + }, + typeof(QuartoNotebookWorker.include_str), + Module, + String, + }, + ) end -precompile(Tuple{typeof(Base.Filesystem.mkpath), String}) -precompile(Tuple{typeof(QuartoNotebookWorker.refresh!), Base.Dict{String, Any}}) -precompile(Tuple{typeof(QuartoNotebookWorker.refresh!), Base.Dict{String, Any}, Base.Dict{String, Any}}) +precompile(Tuple{typeof(Base.Filesystem.mkpath),String}) +precompile(Tuple{typeof(QuartoNotebookWorker.refresh!),Base.Dict{String,Any}}) +precompile( + Tuple{ + typeof(QuartoNotebookWorker.refresh!), + Base.Dict{String,Any}, + Base.Dict{String,Any}, + }, +) module __PrecompilationModule end @@ -34,10 +56,11 @@ PrecompileTools.@compile_workload begin inline = false, ) io = IOBuffer() - bson = QuartoNotebookWorker.Packages.BSON.bson(io, Dict{Symbol,Any}(:data => result)) + bson = + QuartoNotebookWorker.Packages.BSON.bson(io, Dict{Symbol,Any}(:data => result)) seekstart(io) QuartoNotebookWorker.Packages.BSON.load(io)[:data] end end -QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = nothing \ No newline at end of file +QuartoNotebookWorker.NotebookState.NotebookModuleForPrecompile[] = nothing diff --git a/src/server.jl b/src/server.jl index 62abdc98..b7e86060 100644 --- a/src/server.jl +++ b/src/server.jl @@ -69,7 +69,10 @@ end function startup_env_variables() qnw_env_dir = Scratch.@get_scratch!("qnw-env-$(hash(Malt.worker_package))") - return ["QUARTONOTEBOOKWORKER_ENV_DIR=$qnw_env_dir", "QUARTONOTEBOOKWORKER_PACKAGE_DIR=$(Malt.worker_package)"] + return [ + "QUARTONOTEBOOKWORKER_ENV_DIR=$qnw_env_dir", + "QUARTONOTEBOOKWORKER_PACKAGE_DIR=$(Malt.worker_package)", + ] end struct SourceRange @@ -243,7 +246,11 @@ function refresh!(file::File, options::Dict) Malt.stop(file.worker) exe, _exeflags = _julia_exe(exeflags) file.worker = cd( - () -> Malt.Worker(; exe, exeflags = _exeflags, env = vcat(env, quarto_env, startup_env_variables())), + () -> Malt.Worker(; + exe, + exeflags = _exeflags, + env = vcat(env, quarto_env, startup_env_variables()), + ), dirname(file.path), ) file.exe = exe From 1046cf9352989a9b808d4df1848e320d2c58b7d4 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 1 Oct 2025 10:56:38 +0200 Subject: [PATCH 21/21] add changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a44e6f..f25fa1e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Improved precompilation of QuartoNotebookWorker and avoided Pkg startup costs by using a scratch space to cache QNW project files [#341]. + ## [v0.17.4] - 2025-09-26 ### Added @@ -473,3 +475,4 @@ caching is enabled. Delete this folder to clear the cache. [#259] [#306]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/306 [#335]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/335 [#339]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/339 +[#341]: https://github.com/PumasAI/QuartoNotebookRunner.jl/issues/341