Add test that optimization_clean doesn't trim, marked broken

Author: RomeoV

test/trim/main.jl renamed to test/trim/main_clean.jl

@@ -2,7 +2,7 @@ using TrimTest
 
 function (@main)(argv::Vector{String})::Cint
     λ = parse(Float64, argv[2])
-    sol = TrimTest.minimize(λ)
+    sol = TrimTest.TestModuleClean.minimize(λ)
     println(Core.stdout, sum(sol.u))
     return 0
 end

test/trim/main_trimmable.jl

@@ -0,0 +1,8 @@
+using TrimTest
+
+function (@main)(argv::Vector{String})::Cint
+    λ = parse(Float64, argv[2])
+    sol = TrimTest.TestModuleTrimmable.minimize(λ)
+    println(Core.stdout, sum(sol.u))
+    return 0
+end

test/trim/clean_optimization.jl renamed to test/trim/optimization_clean.jl

@@ -1,3 +1,4 @@
+module TestModuleClean
 using NonlinearSolveFirstOrder
 using ADTypes: AutoForwardDiff
 using ForwardDiff
@@ -23,10 +24,11 @@ struct MyParams{T, M}
 end
 
 function minimize(x)
-    autodiff = AutoForwardDiff(; chunksize=1)
-    alg = TrustRegion(; autodiff, linsolve=LS.CholeskyFactorization())
-    ps = MyParams(rand(), hermitianpart(rand(2,2)+2I))
+    autodiff = AutoForwardDiff(; chunksize = 1)
+    alg = TrustRegion(; autodiff, linsolve = LS.CholeskyFactorization())
+    ps = MyParams(rand(), hermitianpart(rand(2, 2) + 2I))
     prob = NonlinearLeastSquaresProblem{false}(f, rand(2), ps)
     sol = solve(prob, alg)
     return sol
 end
+end

test/trim/trimmable_optimization.jl renamed to test/trim/optimization_trimmable.jl

@@ -1,3 +1,4 @@
+module TestModuleTrimmable
 using NonlinearSolveFirstOrder
 using DiffEqBase
 using ADTypes: AutoForwardDiff
@@ -35,3 +36,4 @@ function minimize(x)
     solve!(cache)
     return cache
 end
+end

test/trim/runtests.jl

@@ -3,21 +3,21 @@ using SafeTestsets
 @safetestset "Clean implementation (non-trimmable)" begin
     using JET
     using SciMLBase: successful_retcode
-    include("clean_optimization.jl")
-    @test successful_retcode(minimize(1.0).retcode)
-    # can't use `@test_opt` macro here because it would eval before `using JET`
-    # is processed
-    test_opt(minimize, (typeof(1.0),))
+    include("optimization_clean.jl")
+    @test successful_retcode(TestModuleClean.minimize(1.0).retcode)
+    # can't use `@test_opt` macro here because it would try to eval before
+    # `using JET` is processed
+    test_opt(TestModuleClean.minimize, (typeof(1.0),))
 end
 
 @safetestset "Trimmable implementation" begin
     using JET
     using SciMLBase: successful_retcode
-    include("trimmable_optimization.jl")
-    @test successful_retcode(minimize(1.0).retcode)
-    # can't use `@test_opt` macro here because it would eval before `using JET`
-    # is processed
-    test_opt(minimize, (typeof(1.0),))
+    include("optimization_trimmable.jl")
+    @test successful_retcode(TestModuleClean.minimize(1.0).retcode)
+    # can't use `@test_opt` macro here because it would try to eval before
+    # `using JET` is processed
+    test_opt(TestModuleTrimmable.minimize, (typeof(1.0),))
 end
 
 @safetestset "Run trim" begin
@@ -43,23 +43,27 @@ end
         )
     )
     @test isfile(JULIAC)
-    binpath = tempname()
-    cmd = `$(Base.julia_cmd()) --project=. --depwarn=error $(JULIAC) --experimental --trim=unsafe-warn --output-exe $(binpath) main.jl`
 
-    # since we are calling Julia from Julia, we first need to clean some
-    # environment variables
-    clean_env = copy(ENV)
-    delete!(clean_env, "JULIA_PROJECT")
-    delete!(clean_env, "JULIA_LOAD_PATH")
-    # We could just check for success, but then failures are hard to debug.
-    # Instead we use `_execute` to also capture `stdout` and `stderr`.
-    # @test success(setenv(cmd, clean_env))
-    trimcall = _execute(setenv(cmd, clean_env; dir = @__DIR__))
-    if trimcall.exitcode != 0
-        @show trimcall.stdout
-        @show trimcall.stderr
+    for (mainfile, shouldpass) in [("main_trimmable.jl", true),
+                                   ("main_clean.jl", false)]
+        binpath = tempname()
+        cmd = `$(Base.julia_cmd()) --project=. --depwarn=error $(JULIAC) --experimental --trim=unsafe-warn --output-exe $(binpath) $(mainfile)`
+
+        # since we are calling Julia from Julia, we first need to clean some
+        # environment variables
+        clean_env = copy(ENV)
+        delete!(clean_env, "JULIA_PROJECT")
+        delete!(clean_env, "JULIA_LOAD_PATH")
+        # We could just check for success, but then failures are hard to debug.
+        # Instead we use `_execute` to also capture `stdout` and `stderr`.
+        # @test success(setenv(cmd, clean_env))
+        trimcall = _execute(setenv(cmd, clean_env; dir = @__DIR__))
+        if trimcall.exitcode != 0 && shouldpass
+            @show trimcall.stdout
+            @show trimcall.stderr
+        end
+        @test trimcall.exitcode == 0 broken=!shouldpass
+        @test isfile(binpath) broken=!shouldpass
+        @test success(`$(binpath) 1.0`) broken=!shouldpass
     end
-    @test trimcall.exitcode == 0
-    @test isfile(binpath)
-    @test success(`$(binpath) 1.0`)
 end

test/trim/src/TrimTest.jl

@@ -2,7 +2,7 @@ module TrimTest
 #=
 Currently, trimming only works if the target code is in a package. I.e., trying to trim
 ```julia
-include("trimmable_optimization.jl")
+include("optimization_trimmable.jl")
 function (@main)(argv::Vector{String})::Cint
     minimize(1.0)
     return 0
@@ -11,7 +11,7 @@ end
 or even
 ```julia
 mod MyMod
-    include("trimmable_optimization.jl")
+    include("optimization_trimmable.jl")
 end
 function (@main)(argv::Vector{String})::Cint
     MyMod.minimize(1.0)
@@ -22,5 +22,6 @@ segfaults `juliac`. Looking at the segfault stacktrace it seems the culprit is
 `const cache = init(...)`. Either way, we circumvent the segfault by putting
 this below code into a package definition.
 =#
-include("../trimmable_optimization.jl")
+include("../optimization_trimmable.jl")
+include("../optimization_clean.jl")
 end