From 7d6c264f66b1851102bfe4d529024d86bf849ece Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 5 Jun 2025 01:15:39 +0200 Subject: [PATCH 01/35] wip: upgrade to JuliaSyntax v1 --- Project.toml | 4 +- src/get_names_used.jl | 34 +- src/parse_utilities.jl | 12 +- test/runtests.jl | 1912 ++++++++++++++++++++-------------------- 4 files changed, 1007 insertions(+), 955 deletions(-) diff --git a/Project.toml b/Project.toml index 23ac81bd..9c440d04 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ExplicitImports" uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7" -version = "1.11.2" +version = "1.11.3" authors = ["Eric P. Hanson"] [deps] @@ -17,7 +17,7 @@ AbstractTrees = "0.4.5" Aqua = "0.8.4" Compat = "4.15" DataFrames = "1.6" -JuliaSyntax = "0.4.8" +JuliaSyntax = "1" LinearAlgebra = "<0.0.1, 1" Logging = "<0.0.1, 1" Markdown = "<0.0.1, 1" diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 3049a89a..2373a848 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -270,9 +270,7 @@ function in_for_argument_position(node) # We must be on the LHS of a `for` `equal`. if !has_parent(node, 2) return false - elseif parents_match(node, (K"=", K"for")) - return child_index(node) == 1 - elseif parents_match(node, (K"=", K"cartesian_iterator", K"for")) + elseif parents_match(node, (K"iteration", K"for")) return child_index(node) == 1 elseif kind(parent(node)) in (K"tuple", K"parameters") return in_for_argument_position(get_parent(node)) @@ -293,13 +291,11 @@ end function in_generator_arg_position(node) # We must be on the LHS of a `=` inside a generator - # (possibly inside a filter, possibly inside a `cartesian_iterator`) + # (possibly inside a filter, possibly inside a `iteration`) if !has_parent(node, 2) return false - elseif parents_match(node, (K"=", K"generator")) || - parents_match(node, (K"=", K"cartesian_iterator", K"generator")) || - parents_match(node, (K"=", K"filter")) || - parents_match(node, (K"=", K"cartesian_iterator", K"filter")) + elseif parents_match(node, (K"iteration", K"generator")) || + parents_match(node, (K"iteration", K"filter")) return child_index(node) == 1 elseif kind(parent(node)) in (K"tuple", K"parameters") return in_generator_arg_position(get_parent(node)) @@ -356,7 +352,7 @@ function analyze_name(leaf; debug=false) # update our state val = get_val(node) k = kind(node) - args = nodevalue(node).node.raw.args + args = nodevalue(node).node.raw.children debug && println(val, ": ", k) # Constructs that start a new local scope. Note `let` & `macro` *arguments* are not explicitly supported/tested yet, @@ -494,6 +490,11 @@ function analyze_all_names(file) return analyze_per_usage_info(per_usage_info), untainted_modules end +# this would ideally be identity, but hashing SyntaxNode's is slow on v1.0.2 +# https://github.com/JuliaLang/JuliaSyntax.jl/issues/558 +# so we will settle for some unlikely chance at collisions and just check the string rep of the values +_simplify_hashing(scope_path) = map(string ∘ get_val, scope_path) + function is_name_internal_in_higher_local_scope(name, scope_path, seen) # We will recurse up the `scope_path`. Note the order is "reversed", # so the first entry of `scope_path` is deepest. @@ -506,7 +507,7 @@ function is_name_internal_in_higher_local_scope(name, scope_path, seen) end # Ok, now pop off the first scope and check. scope_path = scope_path[2:end] - ret = get(seen, (; name, scope_path), nothing) + ret = get(seen, (; name, scope_path=_simplify_hashing(scope_path)), nothing) if ret === nothing # Not introduced here yet, trying recursing further continue @@ -527,9 +528,9 @@ function analyze_per_usage_info(per_usage_info) # Otherwise, we are in local scope: # 1. Next, if the name is a function arg, then this is not a global name (essentially first usage is assignment) # 2. Otherwise, if first usage is assignment, then it is local, otherwise it is global - seen = Dict{@NamedTuple{name::Symbol,scope_path::Vector{JuliaSyntax.SyntaxNode}},Bool}() + seen = Dict{@NamedTuple{name::Symbol,scope_path::Vector{String}},Bool}() return map(per_usage_info) do nt - @compat if (; nt.name, nt.scope_path) in keys(seen) + @compat if (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) in keys(seen) return PerUsageInfo(; nt..., first_usage_in_scope=false, external_global_name=missing, analysis_code=IgnoredNonFirst) @@ -561,7 +562,8 @@ function analyze_per_usage_info(per_usage_info) (nt.is_assignment, InternalAssignment)) if is_local external_global_name = false - push!(seen, (; nt.name, nt.scope_path) => external_global_name) + push!(seen, + (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=reason) @@ -572,13 +574,15 @@ function analyze_per_usage_info(per_usage_info) nt.scope_path, seen) external_global_name = false - push!(seen, (; nt.name, nt.scope_path) => external_global_name) + push!(seen, + (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=InternalHigherScope) end external_global_name = true - push!(seen, (; nt.name, nt.scope_path) => external_global_name) + push!(seen, + (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=External) end diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index fbfdecef..cfab0377 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -51,7 +51,7 @@ AbstractTrees.children(::SkippedFile) = () function AbstractTrees.children(wrapper::SyntaxNodeWrapper) node = wrapper.node if JuliaSyntax.kind(node) == K"call" - children = JuliaSyntax.children(node) + children = js_children(node) if length(children) == 2 f, arg = children::Vector{JuliaSyntax.SyntaxNode} # make JET happy if f.val === :include @@ -60,7 +60,7 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) return [SkippedFile(location)] end if JuliaSyntax.kind(arg) == K"string" - children = JuliaSyntax.children(arg) + children = js_children(arg) # if we have interpolation, there may be >1 child length(children) == 1 || @goto dynamic c = only(children) @@ -92,10 +92,14 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) end end return map(n -> SyntaxNodeWrapper(n, wrapper.file, wrapper.bad_locations), - JuliaSyntax.children(node)) + js_children(node)) end -js_children(n::Union{TreeCursor,SyntaxNodeWrapper}) = JuliaSyntax.children(js_node(n)) +js_children(n::Union{TreeCursor,SyntaxNodeWrapper}) = js_children(js_node(n)) + +# https://github.com/JuliaLang/JuliaSyntax.jl/issues/557 +js_children(n::Union{JuliaSyntax.SyntaxNode}) = something(JuliaSyntax.children(n), ()) + js_node(n::SyntaxNodeWrapper) = n.node js_node(n::TreeCursor) = js_node(nodevalue(n)) diff --git a/test/runtests.jl b/test/runtests.jl index 5ef97315..45707fab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -74,1026 +74,1070 @@ include("test_explicit_imports.jl") include("main.jl") include("Test_Mod_Underscores.jl") -# For deprecations, we are using `maxlog`, which -# the TestLogger only respects in Julia 1.8+. -# (https://github.com/JuliaLang/julia/commit/02f7332027bd542b0701956a0f838bc75fa2eebd) -if VERSION >= v"1.8-" - @testset "deprecations" begin - include("deprecated.jl") +@testset "ExplicitImports" begin + # For deprecations, we are using `maxlog`, which + # the TestLogger only respects in Julia 1.8+. + # (https://github.com/JuliaLang/julia/commit/02f7332027bd542b0701956a0f838bc75fa2eebd) + if VERSION >= v"1.8-" + @testset "deprecations" begin + include("deprecated.jl") + end end -end -# package extension support needs Julia 1.9+ -if VERSION > v"1.9-" - @testset "Extensions" begin - submods = ExplicitImports.find_submodules(TestPkg) - @test length(submods) == 2 - DataFramesExt = Base.get_extension(TestPkg, :DataFramesExt) - @test haskey(Dict(submods), DataFramesExt) - - ext_imports = Dict(only_name_source(explicit_imports(TestPkg)))[DataFramesExt] - @test ext_imports == [(; name=:DataFrames, source=DataFrames), - (; name=:DataFrame, source=DataFrames), - (; name=:groupby, source=DataFrames)] || - ext_imports == [(; name=:DataFrames, source=DataFrames), - (; name=:DataFrame, source=DataFrames), - (; name=:groupby, source=DataFrames.DataAPI)] + # package extension support needs Julia 1.9+ + if VERSION > v"1.9-" + @testset "Extensions" begin + submods = ExplicitImports.find_submodules(TestPkg) + @test length(submods) == 2 + DataFramesExt = Base.get_extension(TestPkg, :DataFramesExt) + @test haskey(Dict(submods), DataFramesExt) + + ext_imports = Dict(only_name_source(explicit_imports(TestPkg)))[DataFramesExt] + @test ext_imports == [(; name=:DataFrames, source=DataFrames), + (; name=:DataFrame, source=DataFrames), + (; name=:groupby, source=DataFrames)] || + ext_imports == [(; name=:DataFrames, source=DataFrames), + (; name=:DataFrame, source=DataFrames), + (; name=:groupby, source=DataFrames.DataAPI)] + end end -end -@testset "function arg bug" begin - # https://github.com/ericphanson/ExplicitImports.jl/issues/62 - df = DataFrame(get_names_used("test_mods.jl").per_usage_info) - subset!(df, :name => ByRow(==(:norm)), :module_path => ByRow(==([:TestMod13]))) + @testset "function arg bug" begin + # https://github.com/ericphanson/ExplicitImports.jl/issues/62 + df = DataFrame(get_names_used("test_mods.jl").per_usage_info) + subset!(df, :name => ByRow(==(:norm)), :module_path => ByRow(==([:TestMod13]))) - @test_broken check_no_stale_explicit_imports(TestMod13, "test_mods.jl") === nothing -end + @test_broken check_no_stale_explicit_imports(TestMod13, "test_mods.jl") === nothing + end -@testset "owner_mod_for_printing" begin - @test owner_mod_for_printing(Core, :throw, Core.throw) == Base - @test owner_mod_for_printing(Core, :println, Core.println) == Core -end + @testset "owner_mod_for_printing" begin + @test owner_mod_for_printing(Core, :throw, Core.throw) == Base + @test owner_mod_for_printing(Core, :println, Core.println) == Core + end -# https://github.com/ericphanson/ExplicitImports.jl/issues/69 -@testset "Reexport support" begin - @test check_no_stale_explicit_imports(TestMod15, "test_mods.jl") === nothing - @test isempty(improper_explicit_imports_nonrecursive(TestMod15, "test_mods.jl")) - @test isempty(improper_explicit_imports(TestMod15, "test_mods.jl")[1][2]) -end + # https://github.com/ericphanson/ExplicitImports.jl/issues/69 + @testset "Reexport support" begin + @test check_no_stale_explicit_imports(TestMod15, "test_mods.jl") === nothing + @test isempty(improper_explicit_imports_nonrecursive(TestMod15, "test_mods.jl")) + @test isempty(improper_explicit_imports(TestMod15, "test_mods.jl")[1][2]) + end -if VERSION >= v"1.7-" - # https://github.com/ericphanson/ExplicitImports.jl/issues/70 - @testset "Compat skipping" begin - @test check_all_explicit_imports_via_owners(TestMod14, "test_mods.jl") === nothing - @test check_all_qualified_accesses_via_owners(TestMod14, "test_mods.jl") === nothing + if VERSION >= v"1.7-" + # https://github.com/ericphanson/ExplicitImports.jl/issues/70 + @testset "Compat skipping" begin + @test check_all_explicit_imports_via_owners(TestMod14, "test_mods.jl") === + nothing + @test check_all_qualified_accesses_via_owners(TestMod14, "test_mods.jl") === + nothing - @test isempty(improper_explicit_imports_nonrecursive(TestMod14, "test_mods.jl")) - @test isempty(improper_explicit_imports(TestMod14, "test_mods.jl")[1][2]) + @test isempty(improper_explicit_imports_nonrecursive(TestMod14, "test_mods.jl")) + @test isempty(improper_explicit_imports(TestMod14, "test_mods.jl")[1][2]) - @test isempty(improper_qualified_accesses_nonrecursive(TestMod14, "test_mods.jl")) + @test isempty(improper_qualified_accesses_nonrecursive(TestMod14, + "test_mods.jl")) - @test isempty(improper_qualified_accesses(TestMod14, "test_mods.jl")[1][2]) + @test isempty(improper_qualified_accesses(TestMod14, "test_mods.jl")[1][2]) + end end -end -@testset "imports" begin - cursor = TreeCursor(SyntaxNodeWrapper("imports.jl")) - leaves = collect(Leaves(cursor)) - import_type_pairs = get_val.(leaves) .=> analyze_import_type.(leaves) - filter!(import_type_pairs) do (k, v) - return v !== :not_import + @testset "imports" begin + cursor = TreeCursor(SyntaxNodeWrapper("imports.jl")) + leaves = collect(Leaves(cursor)) + import_type_pairs = get_val.(leaves) .=> analyze_import_type.(leaves) + filter!(import_type_pairs) do (k, v) + return v !== :not_import + end + @test import_type_pairs == + [:Exporter => :import_LHS, + :exported_a => :import_RHS, + :exported_c => :import_RHS, + :Exporter => :import_LHS, + :exported_c => :import_RHS, + :TestModA => :blanket_using_member, + :SubModB => :blanket_using, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :h2 => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :h3 => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :h => :import_RHS, + :Exporter => :blanket_using, + :Exporter => :plain_import, + :LinearAlgebra => :import_LHS, + :map => :import_RHS, + :_svd! => :import_RHS, + :LinearAlgebra => :import_LHS, + :svd => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :exported_b => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :f => :import_RHS] + + inds = findall(==(:import_RHS), analyze_import_type.(leaves)) + lhs_rhs_pairs = get_import_lhs.(leaves[inds]) .=> get_val.(leaves[inds]) + @test lhs_rhs_pairs == [[:., :., :Exporter] => :exported_a, + [:., :., :Exporter] => :exported_c, + [:., :., :Exporter] => :exported_c, + [:., :., :TestModA, :SubModB] => :h2, + [:., :., :TestModA, :SubModB] => :h3, + [:., :., :TestModA, :SubModB] => :h, + [:LinearAlgebra] => :map, + [:LinearAlgebra] => :_svd!, + [:LinearAlgebra] => :svd, + [:., :., :TestModA, :SubModB] => :exported_b, + [:., :., :TestModA, :SubModB] => :f] + + imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; + allow_internal_imports=false)) + h_row = only(subset(imps, :name => ByRow(==(:h)))) + @test !h_row.public_import + # Note: if this fails locally, try `include("imports.jl")` to rebuild the module + @test h_row.whichmodule == TestModA.SubModB + @test h_row.importing_from == TestModA.SubModB + + h2_row = only(subset(imps, :name => ByRow(==(:h2)))) + @test h2_row.public_import + @test h2_row.whichmodule === TestModA.SubModB + @test h2_row.importing_from == TestModA.SubModB + _svd!_row = only(subset(imps, :name => ByRow(==(:_svd!)))) + @test !_svd!_row.public_import + + f_row = only(subset(imps, :name => ByRow(==(:f)))) + @test !f_row.public_import # not public in `TestModA.SubModB` + @test f_row.whichmodule == TestModA + @test f_row.importing_from == TestModA.SubModB + + imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; + allow_internal_imports=true)) + # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: + @test all(==(LinearAlgebra), imps.importing_from) end - @test import_type_pairs == - [:Exporter => :import_LHS, - :exported_a => :import_RHS, - :exported_c => :import_RHS, - :Exporter => :import_LHS, - :exported_c => :import_RHS, - :TestModA => :blanket_using_member, - :SubModB => :blanket_using, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :h2 => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :h3 => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :h => :import_RHS, - :Exporter => :blanket_using, - :Exporter => :plain_import, - :LinearAlgebra => :import_LHS, - :map => :import_RHS, - :_svd! => :import_RHS, - :LinearAlgebra => :import_LHS, - :svd => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :exported_b => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :f => :import_RHS] - - inds = findall(==(:import_RHS), analyze_import_type.(leaves)) - lhs_rhs_pairs = get_import_lhs.(leaves[inds]) .=> get_val.(leaves[inds]) - @test lhs_rhs_pairs == [[:., :., :Exporter] => :exported_a, - [:., :., :Exporter] => :exported_c, - [:., :., :Exporter] => :exported_c, - [:., :., :TestModA, :SubModB] => :h2, - [:., :., :TestModA, :SubModB] => :h3, - [:., :., :TestModA, :SubModB] => :h, - [:LinearAlgebra] => :map, - [:LinearAlgebra] => :_svd!, - [:LinearAlgebra] => :svd, - [:., :., :TestModA, :SubModB] => :exported_b, - [:., :., :TestModA, :SubModB] => :f] - - imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; - allow_internal_imports=false)) - h_row = only(subset(imps, :name => ByRow(==(:h)))) - @test !h_row.public_import - # Note: if this fails locally, try `include("imports.jl")` to rebuild the module - @test h_row.whichmodule == TestModA.SubModB - @test h_row.importing_from == TestModA.SubModB - - h2_row = only(subset(imps, :name => ByRow(==(:h2)))) - @test h2_row.public_import - @test h2_row.whichmodule === TestModA.SubModB - @test h2_row.importing_from == TestModA.SubModB - _svd!_row = only(subset(imps, :name => ByRow(==(:_svd!)))) - @test !_svd!_row.public_import - - f_row = only(subset(imps, :name => ByRow(==(:f)))) - @test !f_row.public_import # not public in `TestModA.SubModB` - @test f_row.whichmodule == TestModA - @test f_row.importing_from == TestModA.SubModB - - imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; - allow_internal_imports=true)) - # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: - @test all(==(LinearAlgebra), imps.importing_from) -end -##### -##### To analyze a test case -##### -# using ExplicitImports: js_node, get_parent, kind, parents_match -# using JuliaSyntax: @K_str - -# cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")); -# leaves = collect(Leaves(cursor)) -# leaf = leaves[end - 2] # select a leaf -# js_node(leaf) # inspect it -# p = js_node(get_parent(leaf, 3)) # see the tree, etc -# kind(p) - -@testset "qualified access" begin - # analyze_qualified_names - qualified = analyze_qualified_names(TestQualifiedAccess, "test_qualified_access.jl") - @test length(qualified) == 6 - ABC, DEF, HIJ, X, map, x = qualified - @test ABC.name == :ABC - @test DEF.public_access - @test HIJ.public_access - @test DEF.name == :DEF - @test HIJ.name == :HIJ - @test X.name == :X - @test map.name == :map - @test x.name == :x - @test x.self_qualified - - # improper_qualified_accesses - ret = Dict(improper_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false)) - @test isempty(ret[TestQualifiedAccess.Bar]) - @test isempty(ret[TestQualifiedAccess.FooModule]) - @test !isempty(ret[TestQualifiedAccess]) - - @test length(ret[TestQualifiedAccess]) == 4 - ABC, X, map, x = ret[TestQualifiedAccess] - # Can add keys, but removing them is breaking - @test keys(ABC) ⊇ - [:name, :location, :value, :accessing_from, :whichmodule, :public_access, - :accessing_from_owns_name, :accessing_from_submodule_owns_name, :internal_access] - @test ABC.name == :ABC - @test ABC.location isa AbstractString - @test ABC.whichmodule == TestQualifiedAccess.Bar - @test ABC.accessing_from == TestQualifiedAccess.FooModule - @test ABC.public_access == false - @test ABC.accessing_from_submodule_owns_name == false - - @test X.name == :X - @test X.whichmodule == TestQualifiedAccess.FooModule.FooSub - @test X.accessing_from == TestQualifiedAccess.FooModule - @test X.public_access == false - @test X.accessing_from_submodule_owns_name == true - - @test map.name == :map - - @test x.name == :x - @test x.self_qualified - - imps = DataFrame(improper_qualified_accesses_nonrecursive(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=true)) - subset!(imps, :self_qualified => ByRow(!)) # drop self-qualified - # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: - @test all(==(LinearAlgebra), imps.accessing_from) - - # check_no_self_qualified_accesses - ex = SelfQualifiedAccessException - @test_throws ex check_no_self_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl") - - str = exception_string() do - return check_no_self_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl") - end - @test contains(str, "has self-qualified accesses:\n- `x` was accessed as") - - @test check_no_self_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl"; ignore=(:x,)) === - nothing - - str = sprint(print_explicit_imports, TestQualifiedAccess, - "test_qualified_access.jl") - @test contains(str, "has 1 self-qualified access:\n\n • x was accessed as ") - - # check_all_qualified_accesses_via_owners - ex = QualifiedAccessesFromNonOwnerException - @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - - # Test the printing is hitting our formatted errors - str = exception_string() do - return check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - end - @test contains(str, - "has qualified accesses to names via modules other than their owner as determined") + ##### + ##### To analyze a test case + ##### + # using ExplicitImports: js_node, get_parent, kind, parents_match + # using JuliaSyntax: @K_str + + # cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")); + # leaves = collect(Leaves(cursor)) + # leaf = leaves[end - 2] # select a leaf + # js_node(leaf) # inspect it + # p = js_node(get_parent(leaf, 3)) # see the tree, etc + # kind(p) + + @testset "qualified access" begin + # analyze_qualified_names + qualified = analyze_qualified_names(TestQualifiedAccess, "test_qualified_access.jl") + @test length(qualified) == 6 + ABC, DEF, HIJ, X, map, x = qualified + @test ABC.name == :ABC + @test DEF.public_access + @test HIJ.public_access + @test DEF.name == :DEF + @test HIJ.name == :HIJ + @test X.name == :X + @test map.name == :map + @test x.name == :x + @test x.self_qualified + + # improper_qualified_accesses + ret = Dict(improper_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false)) + @test isempty(ret[TestQualifiedAccess.Bar]) + @test isempty(ret[TestQualifiedAccess.FooModule]) + @test !isempty(ret[TestQualifiedAccess]) + + @test length(ret[TestQualifiedAccess]) == 4 + ABC, X, map, x = ret[TestQualifiedAccess] + # Can add keys, but removing them is breaking + @test keys(ABC) ⊇ + [:name, :location, :value, :accessing_from, :whichmodule, :public_access, + :accessing_from_owns_name, :accessing_from_submodule_owns_name, + :internal_access] + @test ABC.name == :ABC + @test ABC.location isa AbstractString + @test ABC.whichmodule == TestQualifiedAccess.Bar + @test ABC.accessing_from == TestQualifiedAccess.FooModule + @test ABC.public_access == false + @test ABC.accessing_from_submodule_owns_name == false + + @test X.name == :X + @test X.whichmodule == TestQualifiedAccess.FooModule.FooSub + @test X.accessing_from == TestQualifiedAccess.FooModule + @test X.public_access == false + @test X.accessing_from_submodule_owns_name == true + + @test map.name == :map + + @test x.name == :x + @test x.self_qualified + + imps = DataFrame(improper_qualified_accesses_nonrecursive(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=true)) + subset!(imps, :self_qualified => ByRow(!)) # drop self-qualified + # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: + @test all(==(LinearAlgebra), imps.accessing_from) + + # check_no_self_qualified_accesses + ex = SelfQualifiedAccessException + @test_throws ex check_no_self_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl") - skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, ignore=(:map,), - allow_internal_accesses=false) === - nothing + str = exception_string() do + return check_no_self_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl") + end + @test contains(str, "has self-qualified accesses:\n- `x` was accessed as") - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - ignore=(:ABC, :map), - allow_internal_accesses=false) === nothing + @test check_no_self_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl"; ignore=(:x,)) === + nothing - # allow_internal_accesses=true - @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl", - ignore=(:ABC,)) + str = sprint(print_explicit_imports, TestQualifiedAccess, + "test_qualified_access.jl") + @test contains(str, "has 1 self-qualified access:\n\n • x was accessed as ") - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - ignore=(:ABC, :map)) === nothing - - @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, - require_submodule_access=true, - allow_internal_accesses=false) - - skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar, - TestQualifiedAccess.FooModule => TestQualifiedAccess.FooModule.FooSub, - LinearAlgebra => Base) - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, - require_submodule_access=true, - allow_internal_accesses=false) === nothing - - # Printing via `print_explicit_imports` - str = sprint(io -> print_explicit_imports(io, TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "accesses 2 names from non-owner modules") - @test contains(str, "ABC has owner") - - ex = NonPublicQualifiedAccessException - @test_throws ex check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - str = exception_string() do - return check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - end - @test contains(str, "- `ABC` is not public in") + # check_all_qualified_accesses_via_owners + ex = QualifiedAccessesFromNonOwnerException + @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) - @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + # Test the printing is hitting our formatted errors + str = exception_string() do + return check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) + end + @test contains(str, + "has qualified accesses to names via modules other than their owner as determined") + + skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, ignore=(:map,), + allow_internal_accesses=false) === + nothing + + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:ABC, :map), + allow_internal_accesses=false) === + nothing + + # allow_internal_accesses=true + @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl", + ignore=(:ABC,)) + + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:ABC, :map)) === nothing + + @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, + require_submodule_access=true, + allow_internal_accesses=false) + + skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar, + TestQualifiedAccess.FooModule => TestQualifiedAccess.FooModule.FooSub, + LinearAlgebra => Base) + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, + require_submodule_access=true, + allow_internal_accesses=false) === + nothing + + # Printing via `print_explicit_imports` + str = sprint(io -> print_explicit_imports(io, TestQualifiedAccess, "test_qualified_access.jl"; - ignore=(:X, :ABC, :map), - allow_internal_accesses=false) === nothing - - skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) + allow_internal_accesses=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "accesses 2 names from non-owner modules") + @test contains(str, "ABC has owner") - @test check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, ignore=(:X, :map), - allow_internal_accesses=false) === nothing + ex = NonPublicQualifiedAccessException + @test_throws ex check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) + str = exception_string() do + return check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) + end + @test contains(str, "- `ABC` is not public in") + + @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:X, :ABC, :map), + allow_internal_accesses=false) === + nothing + + skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) + + @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, ignore=(:X, :map), + allow_internal_accesses=false) === + nothing + + # allow_internal_accesses=true + @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:map,)) === nothing + end - # allow_internal_accesses=true - @test check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - ignore=(:map,)) === nothing -end + @testset "improper explicit imports" begin + imps = Dict(improper_explicit_imports(TestModA, "TestModA.jl"; + allow_internal_imports=false)) + row = only(imps[TestModA]) + @test row.name == :un_exported + @test row.whichmodule == Exporter + + row1, row2 = imps[TestModA.SubModB.TestModA.TestModC] + # Can add keys, but removing them is breaking + @test keys(row1) ⊇ + [:name, :location, :value, :importing_from, :whichmodule, :public_import, + :importing_from_owns_name, :importing_from_submodule_owns_name, :stale, + :internal_import] + @test row1.name == :exported_c + @test row1.stale == true + @test row2.name == :exported_d + @test row2.stale == true + + @test check_all_explicit_imports_via_owners(TestModA, "TestModA.jl"; + allow_internal_imports=false) === + nothing + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, + "imports.jl"; + allow_internal_imports=false) + + # allow_internal_imports=true + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, + "imports.jl";) + @test check_all_explicit_imports_via_owners(ModImports, + "imports.jl"; ignore=(:map,)) === + nothing + # Test the printing is hitting our formatted errors + str = exception_string() do + return check_all_explicit_imports_via_owners(ModImports, + "imports.jl"; + allow_internal_imports=false) + end -@testset "improper explicit imports" begin - imps = Dict(improper_explicit_imports(TestModA, "TestModA.jl"; - allow_internal_imports=false)) - row = only(imps[TestModA]) - @test row.name == :un_exported - @test row.whichmodule == Exporter - - row1, row2 = imps[TestModA.SubModB.TestModA.TestModC] - # Can add keys, but removing them is breaking - @test keys(row1) ⊇ - [:name, :location, :value, :importing_from, :whichmodule, :public_import, - :importing_from_owns_name, :importing_from_submodule_owns_name, :stale, - :internal_import] - @test row1.name == :exported_c - @test row1.stale == true - @test row2.name == :exported_d - @test row2.stale == true - - @test check_all_explicit_imports_via_owners(TestModA, "TestModA.jl"; - allow_internal_imports=false) === nothing - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, - "imports.jl"; - allow_internal_imports=false) - - # allow_internal_imports=true - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, - "imports.jl";) - @test check_all_explicit_imports_via_owners(ModImports, - "imports.jl"; ignore=(:map,)) === nothing - # Test the printing is hitting our formatted errors - str = exception_string() do - return check_all_explicit_imports_via_owners(ModImports, - "imports.jl"; - allow_internal_imports=false) + @test contains(str, + "explicit imports of names from modules other than their owner as determined ") + + @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; + ignore=(:exported_b, :f, :map), + allow_internal_imports=false) === + nothing + + # We can pass `skip` to ignore non-owning explicit imports from LinearAlgebra that are owned by Base + @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; + skip=(LinearAlgebra => Base,), + ignore=(:exported_b, :f), + allow_internal_imports=false) === + nothing + + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + allow_internal_imports=false) + + # test ignore + @test check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + ignore=(:ABC,), + allow_internal_imports=false) === + nothing + + # test skip + @test check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + skip=(TestExplicitImports.FooModule => TestExplicitImports.Bar,), + allow_internal_imports=false) === + nothing + + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + ignore=(:ABC,), + require_submodule_import=true, + allow_internal_imports=false) + + @test check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + ignore=(:ABC, :X), + require_submodule_import=true, + allow_internal_imports=false) === + nothing + + # allow_internal_imports = true + @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, + "imports.jl";) + @test check_all_explicit_imports_are_public(ModImports, + "imports.jl"; ignore=(:map, :_svd!)) === + nothing end - @test contains(str, - "explicit imports of names from modules other than their owner as determined ") - - @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; - ignore=(:exported_b, :f, :map), - allow_internal_imports=false) === nothing - - # We can pass `skip` to ignore non-owning explicit imports from LinearAlgebra that are owned by Base - @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; - skip=(LinearAlgebra => Base,), - ignore=(:exported_b, :f), - allow_internal_imports=false) === nothing - - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - allow_internal_imports=false) - - # test ignore - @test check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - ignore=(:ABC,), - allow_internal_imports=false) === nothing - - # test skip - @test check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - skip=(TestExplicitImports.FooModule => TestExplicitImports.Bar,), - allow_internal_imports=false) === - nothing - - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - ignore=(:ABC,), - require_submodule_import=true, - allow_internal_imports=false) - - @test check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - ignore=(:ABC, :X), - require_submodule_import=true, - allow_internal_imports=false) === nothing - - # allow_internal_imports = true - @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, - "imports.jl";) - @test check_all_explicit_imports_are_public(ModImports, - "imports.jl"; ignore=(:map, :_svd!)) === - nothing -end + @testset "structs" begin + cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) + leaves = collect(Leaves(cursor)) + @test map(get_val, filter(is_struct_type_param, leaves)) == [:X, :Y, :QR] -@testset "structs" begin - cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) - leaves = collect(Leaves(cursor)) - @test map(get_val, filter(is_struct_type_param, leaves)) == [:X, :Y, :QR] + @test map(get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] - @test map(get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] + # Tests #34 and #36 + @test using_statement.(explicit_imports_nonrecursive(TestMod5, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra"] + end - # Tests #34 and #36 - @test using_statement.(explicit_imports_nonrecursive(TestMod5, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra"] -end + if VERSION >= v"1.7-" + @testset "loops" begin + cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) + leaves = collect(Leaves(cursor)) + @test map(get_val, filter(is_for_arg, leaves)) == + [:i, :I, :j, :k, :k, :j, :xi, :yi] + + # Tests #35 + @test using_statement.(explicit_imports_nonrecursive(TestMod6, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra"] + end + end -if VERSION >= v"1.7-" - @testset "loops" begin + @testset "nested local scope" begin cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) leaves = collect(Leaves(cursor)) - @test map(get_val, filter(is_for_arg, leaves)) == [:i, :I, :j, :k, :k, :j, :xi, :yi] - - # Tests #35 - @test using_statement.(explicit_imports_nonrecursive(TestMod6, "test_mods.jl")) == + # Test nested local scope + @test using_statement.(explicit_imports_nonrecursive(TestMod7, "test_mods.jl")) == ["using LinearAlgebra: LinearAlgebra"] end -end -@testset "nested local scope" begin - cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) - leaves = collect(Leaves(cursor)) - # Test nested local scope - @test using_statement.(explicit_imports_nonrecursive(TestMod7, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra"] -end + @testset "types without values in function signatures" begin + # https://github.com/ericphanson/ExplicitImports.jl/issues/33 + @test using_statement.(explicit_imports_nonrecursive(TestMod8, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: QR"] + end -@testset "types without values in function signatures" begin - # https://github.com/ericphanson/ExplicitImports.jl/issues/33 - @test using_statement.(explicit_imports_nonrecursive(TestMod8, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: QR"] -end + @testset "generators" begin + cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) + leaves = collect(Leaves(cursor)) -@testset "generators" begin - cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) - leaves = collect(Leaves(cursor)) + v = [:i1, :I, :i2, :I, :i3, :I, :i4, :I] + w = [:i1, :I] + @test map(get_val, filter(is_generator_arg, leaves)) == + [v; v; w; w; w; w; w] - v = [:i1, :I, :i2, :I, :i3, :I, :i4, :I] - w = [:i1, :I] - @test map(get_val, filter(is_generator_arg, leaves)) == - [v; v; w; w; w; w; w] + if VERSION >= v"1.7-" + @test using_statement.(explicit_imports_nonrecursive(TestMod9, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra"] - if VERSION >= v"1.7-" - @test using_statement.(explicit_imports_nonrecursive(TestMod9, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra"] + per_usage_info, _ = analyze_all_names("test_mods.jl") + df = DataFrame(per_usage_info) + subset!(df, :module_path => ByRow(==([:TestMod9])), :name => ByRow(==(:i1))) + @test all(==(ExplicitImports.InternalGenerator), df.analysis_code) + end + end + + @testset "while loops" begin + @test using_statement.(explicit_imports_nonrecursive(TestMod10, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: I"] per_usage_info, _ = analyze_all_names("test_mods.jl") df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod9])), :name => ByRow(==(:i1))) - @test all(==(ExplicitImports.InternalGenerator), df.analysis_code) + subset!(df, :module_path => ByRow(==([:TestMod10])), :name => ByRow(==(:I))) + # First one is internal, second one external + @test df.analysis_code == + [ExplicitImports.InternalAssignment, ExplicitImports.External] end -end -@testset "while loops" begin - @test using_statement.(explicit_imports_nonrecursive(TestMod10, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: I"] - - per_usage_info, _ = analyze_all_names("test_mods.jl") - df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod10])), :name => ByRow(==(:I))) - # First one is internal, second one external - @test df.analysis_code == [ExplicitImports.InternalAssignment, ExplicitImports.External] -end + if VERSION >= v"1.7-" + @testset "do- syntax" begin + @test using_statement.(explicit_imports_nonrecursive(TestMod11, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra", + "using LinearAlgebra: Hermitian", + "using LinearAlgebra: svd"] + + per_usage_info, _ = analyze_all_names("test_mods.jl") + df = DataFrame(per_usage_info) + subset!(df, :module_path => ByRow(==([:TestMod11]))) + + I_codes = subset(df, :name => ByRow(==(:I))).analysis_code + @test I_codes == + [ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, + ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, + ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, + ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst] + svd_codes = subset(df, :name => ByRow(==(:svd))).analysis_code + @test svd_codes == + [ExplicitImports.InternalFunctionArg, ExplicitImports.External] + Hermitian_codes = subset(df, :name => ByRow(==(:Hermitian))).analysis_code + @test Hermitian_codes == + [ExplicitImports.External, ExplicitImports.IgnoredNonFirst] + end + end -if VERSION >= v"1.7-" - @testset "do- syntax" begin - @test using_statement.(explicit_imports_nonrecursive(TestMod11, "test_mods.jl")) == + @testset "try-catch" begin + @test using_statement.(explicit_imports_nonrecursive(TestMod12, "test_mods.jl")) == ["using LinearAlgebra: LinearAlgebra", - "using LinearAlgebra: Hermitian", + "using LinearAlgebra: I", "using LinearAlgebra: svd"] per_usage_info, _ = analyze_all_names("test_mods.jl") df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod11]))) + subset!(df, :module_path => ByRow(==([:TestMod12]))) I_codes = subset(df, :name => ByRow(==(:I))).analysis_code - @test I_codes == - [ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, - ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, - ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, - ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst] + @test I_codes == [ExplicitImports.InternalAssignment, + ExplicitImports.External, + ExplicitImports.External, + ExplicitImports.InternalAssignment, + ExplicitImports.InternalCatchArgument, + ExplicitImports.IgnoredNonFirst, + ExplicitImports.External] svd_codes = subset(df, :name => ByRow(==(:svd))).analysis_code - @test svd_codes == [ExplicitImports.InternalFunctionArg, ExplicitImports.External] - Hermitian_codes = subset(df, :name => ByRow(==(:Hermitian))).analysis_code - @test Hermitian_codes == [ExplicitImports.External, ExplicitImports.IgnoredNonFirst] + @test svd_codes == [ExplicitImports.InternalAssignment, + ExplicitImports.External, + ExplicitImports.InternalAssignment, + ExplicitImports.External] end -end - -@testset "try-catch" begin - @test using_statement.(explicit_imports_nonrecursive(TestMod12, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra", - "using LinearAlgebra: I", - "using LinearAlgebra: svd"] - - per_usage_info, _ = analyze_all_names("test_mods.jl") - df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod12]))) - - I_codes = subset(df, :name => ByRow(==(:I))).analysis_code - @test I_codes == [ExplicitImports.InternalAssignment, - ExplicitImports.External, - ExplicitImports.External, - ExplicitImports.InternalAssignment, - ExplicitImports.InternalCatchArgument, - ExplicitImports.IgnoredNonFirst, - ExplicitImports.External] - svd_codes = subset(df, :name => ByRow(==(:svd))).analysis_code - @test svd_codes == [ExplicitImports.InternalAssignment, - ExplicitImports.External, - ExplicitImports.InternalAssignment, - ExplicitImports.External] -end - -@testset "scripts" begin - str = sprint(print_explicit_imports_script, "script.jl") - str = replace(str, r"\s+" => " ") - @test contains(str, "Script script.jl") - @test contains(str, "relying on implicit imports for 1 name") - @test contains(str, "using LinearAlgebra: norm") - @test contains(str, "stale explicit imports for this 1 unused name") - @test contains(str, "• qr") -end - -@testset "Handle public symbols with same name as exported Base symbols (#88)" begin - statements = using_statement.(explicit_imports_nonrecursive(Mod88, "examples.jl")) - @test statements == ["using .ModWithTryparse: ModWithTryparse"] - -end -@testset "Don't skip source modules (#29)" begin - # In this case `UUID` is defined in Base but exported in UUIDs - ret = ExplicitImports.find_implicit_imports(Mod29)[:UUID] - @test ret.source == Base - @test ret.exporters == [UUIDs] - # We should NOT skip it, even though `skip` includes `Base`, since the exporters - # are not skipped. - statements = using_statement.(explicit_imports_nonrecursive(Mod29, "examples.jl")) - @test statements == ["using UUIDs: UUIDs", "using UUIDs: UUID"] -end - -@testset "Exported module (#24)" begin - statements = using_statement.(explicit_imports_nonrecursive(Mod24, "examples.jl")) - # The key thing here is we do not have `using .Exporter: exported_a`, - # since we haven't done `using .Exporter` in `Mod24`, only `using .Exporter2` - @test statements == ["using .Exporter2: Exporter2", "using .Exporter2: exported_a"] -end -@testset "string macros (#20)" begin - foo = only_name_source(explicit_imports_nonrecursive(Foo20, "examples.jl")) - @test foo == [(; name=:Markdown, source=Markdown), - (; name=Symbol("@doc_str"), source=Markdown)] - bar = explicit_imports_nonrecursive(Bar20, "examples.jl") - @test isempty(bar) -end - -@testset "TestModArgs" begin - # don't detect `a`! - statements = using_statement.(explicit_imports_nonrecursive(TestModArgs, - "TestModArgs.jl")) - @test statements == - ["using .Exporter4: Exporter4", "using .Exporter4: A", "using .Exporter4: Z"] + @testset "scripts" begin + str = sprint(print_explicit_imports_script, "script.jl") + str = replace(str, r"\s+" => " ") + @test contains(str, "Script script.jl") + @test contains(str, "relying on implicit imports for 1 name") + @test contains(str, "using LinearAlgebra: norm") + @test contains(str, "stale explicit imports for this 1 unused name") + @test contains(str, "• qr") + end - statements = using_statement.(explicit_imports_nonrecursive(ThreadPinning, - "examples.jl")) + @testset "Handle public symbols with same name as exported Base symbols (#88)" begin + statements = using_statement.(explicit_imports_nonrecursive(Mod88, "examples.jl")) + @test statements == ["using .ModWithTryparse: ModWithTryparse"] + end + @testset "Don't skip source modules (#29)" begin + # In this case `UUID` is defined in Base but exported in UUIDs + ret = ExplicitImports.find_implicit_imports(Mod29)[:UUID] + @test ret.source == Base + @test ret.exporters == [UUIDs] + # We should NOT skip it, even though `skip` includes `Base`, since the exporters + # are not skipped. + statements = using_statement.(explicit_imports_nonrecursive(Mod29, "examples.jl")) + @test statements == ["using UUIDs: UUIDs", "using UUIDs: UUID"] + end - @test statements == ["using LinearAlgebra: LinearAlgebra"] -end + @testset "Exported module (#24)" begin + statements = using_statement.(explicit_imports_nonrecursive(Mod24, "examples.jl")) + # The key thing here is we do not have `using .Exporter: exported_a`, + # since we haven't done `using .Exporter` in `Mod24`, only `using .Exporter2` + @test statements == ["using .Exporter2: Exporter2", "using .Exporter2: exported_a"] + end -@testset "is_function_definition_arg" begin - cursor = TreeCursor(SyntaxNodeWrapper("TestModArgs.jl")) - leaves = collect(Leaves(cursor)) - purported_function_args = filter(is_function_definition_arg, leaves) + @testset "string macros (#20)" begin + foo = only_name_source(explicit_imports_nonrecursive(Foo20, "examples.jl")) + @test foo == [(; name=:Markdown, source=Markdown), + (; name=Symbol("@doc_str"), source=Markdown)] + bar = explicit_imports_nonrecursive(Bar20, "examples.jl") + @test isempty(bar) + end - # written this way to get clearer test failure messages - vals = unique(get_val.(purported_function_args)) - @test vals == [:a] + @testset "TestModArgs" begin + # don't detect `a`! + statements = using_statement.(explicit_imports_nonrecursive(TestModArgs, + "TestModArgs.jl")) + @test statements == + ["using .Exporter4: Exporter4", "using .Exporter4: A", "using .Exporter4: Z"] - # we have 9*4 functions with one argument `a`, plus 2 macros - @test length(purported_function_args) == 9 * 4 + 2 - non_function_args = filter(!is_function_definition_arg, leaves) - missed = filter(x -> get_val(x) === :a, non_function_args) - @test isempty(missed) -end + statements = using_statement.(explicit_imports_nonrecursive(ThreadPinning, + "examples.jl")) -@testset "has_ancestor" begin - @test has_ancestor(TestModA.SubModB, TestModA) - @test !has_ancestor(TestModA, TestModA.SubModB) + @test statements == ["using LinearAlgebra: LinearAlgebra"] + end - @test should_skip(Base.Iterators; skip=(Base, Core)) -end + @testset "is_function_definition_arg" begin + cursor = TreeCursor(SyntaxNodeWrapper("TestModArgs.jl")) + leaves = collect(Leaves(cursor)) + purported_function_args = filter(is_function_definition_arg, leaves) -function get_per_scope(per_usage_info) - per_usage_df = DataFrame(per_usage_info) - dropmissing!(per_usage_df, :external_global_name) - return per_usage_df -end + # written this way to get clearer test failure messages + vals = unique(get_val.(purported_function_args)) + @test vals == [:a] -@testset "file not found" begin - for f in (check_no_implicit_imports, check_no_stale_explicit_imports, - check_all_explicit_imports_via_owners, check_all_qualified_accesses_via_owners, - explicit_imports, - explicit_imports_nonrecursive, print_explicit_imports, - improper_explicit_imports, improper_explicit_imports_nonrecursive, - improper_qualified_accesses, improper_qualified_accesses_nonrecursive) - @test_throws FileNotFoundException f(TestModA) + # we have 9*4 functions with one argument `a`, plus 2 macros + @test length(purported_function_args) == 9 * 4 + 2 + non_function_args = filter(!is_function_definition_arg, leaves) + missed = filter(x -> get_val(x) === :a, non_function_args) + @test isempty(missed) end - str = sprint(Base.showerror, FileNotFoundException()) - @test contains(str, "module which is not top-level in a package") -end -@testset "ExplicitImports.jl" begin - @test using_statement.(explicit_imports_nonrecursive(TestModA, "TestModA.jl")) == - ["using .Exporter: Exporter", "using .Exporter: @mac", - "using .Exporter2: Exporter2", - "using .Exporter2: exported_a", "using .Exporter3: Exporter3"] - - per_usage_info, _ = analyze_all_names("TestModA.jl") - df = get_per_scope(per_usage_info) - locals = contains.(string.(df.name), Ref("local")) - @test all(!, df.external_global_name[locals]) - - # we use `x` in two scopes - xs = subset(df, :name => ByRow(==(:x))) - @test !xs[1, :external_global_name] - @test !xs[2, :external_global_name] - @test xs[2, :analysis_code] == ExplicitImports.InternalAssignment - - # we use `exported_a` in two scopes; both times refer to the global name - exported_as = subset(df, :name => ByRow(==(:exported_a))) - @test exported_as[1, :external_global_name] - @test exported_as[2, :external_global_name] - @test !exported_as[2, :is_assignment] - - # Test submodules - @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB, "TestModA.jl")) == - ["using .Exporter3: Exporter3", "using .Exporter3: exported_b", - "using .TestModA: f"] - - mod_path = module_path(TestModA.SubModB) - @test mod_path == [:SubModB, :TestModA, :Main] - sub_df = restrict_to_module(df, TestModA.SubModB) - - h = only(subset(sub_df, :name => ByRow(==(:h)))) - @test h.external_global_name - @test !h.is_assignment - - # Nested submodule with same name as outer module... - @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA, - "TestModA.jl")) == - ["using .Exporter3: Exporter3", "using .Exporter3: exported_b"] - - # Check we are getting innermost names and not outer ones - subsub_df = restrict_to_module(df, TestModA.SubModB.TestModA) - @test :inner_h in subsub_df.name - @test :h ∉ subsub_df.name - # ...we do currently get the outer ones when the module path prefixes collide - @test_broken :f ∉ subsub_df.name - @test_broken :func ∉ subsub_df.name - - # starts from innermost - @test module_path(TestModA.SubModB.TestModA.TestModC) == - [:TestModC, :TestModA, :SubModB, :TestModA, :Main] - - from_outer_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, - "TestModA.jl")) - from_inner_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl")) - @test from_inner_file == from_outer_file - @test "using .TestModA: f" in from_inner_file - # This one isn't needed bc all usages are fully qualified - @test "using .Exporter: exported_a" ∉ from_inner_file - - # This one isn't needed; it is already explicitly imported - @test "using .Exporter: exported_b" ∉ from_inner_file - - # This one shouldn't be there; we never use it, only explicitly import it. - # So actually it should be on a list of unnecessary imports. BUT it can show up - # because by importing it, we have the name in the file, so we used to detect it. - @test "using .Exporter: exported_c" ∉ from_inner_file - - @test from_inner_file == ["using .TestModA: TestModA", "using .TestModA: f"] - - ret = improper_explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl"; - allow_internal_imports=false) - - @test [(; row.name) for row in ret if row.stale] == - [(; name=:exported_c), (; name=:exported_d)] - - # Recursive version - lookup = Dict(improper_explicit_imports(TestModA, - "TestModA.jl"; allow_internal_imports=false)) - ret = lookup[TestModA.SubModB.TestModA.TestModC] - - @test [(; row.name) for row in ret if row.stale] == - [(; name=:exported_c), (; name=:exported_d)] - @test isempty((row for row in lookup[TestModA] if row.stale)) - - per_usage_info, _ = analyze_all_names("TestModC.jl") - testmodc = DataFrame(per_usage_info) - qualified_row = only(subset(testmodc, :name => ByRow(==(:exported_a)))) - @test qualified_row.analysis_code == ExplicitImports.IgnoredQualified - @test qualified_row.qualified_by == [:Exporter] - - qualified_row2 = only(subset(testmodc, :name => ByRow(==(:h)))) - @test qualified_row2.qualified_by == [:TestModA, :SubModB] - - @test using_statement.(explicit_imports_nonrecursive(TestMod1, - "test_mods.jl")) == - ["using ExplicitImports: print_explicit_imports"] - - # Recursion - nested = explicit_imports(TestModA, "TestModA.jl") - @test nested isa Vector{Pair{Module, - Vector{@NamedTuple{name::Symbol,source::Module, - exporters::Vector{Module},location::String}}}} - @test TestModA in first.(nested) - @test TestModA.SubModB in first.(nested) - @test TestModA.SubModB.TestModA in first.(nested) - @test TestModA.SubModB.TestModA.TestModC in first.(nested) - - # Printing - # should be no logs - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - allow_internal_imports=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "Module Main.TestModA is relying on implicit imports") - @test contains(str, "using .Exporter2: Exporter2, exported_a") - @test contains(str, - "However, module Main.TestModA.SubModB.TestModA.TestModC has stale explicit imports for these 2 unused names") - - # should be no logs - # try with linewidth tiny - should put one name per line - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - linewidth=0)) - @test contains(str, "using .Exporter2: Exporter2,\n exported_a") - - # test `show_locations=true` - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - show_locations=true, - allow_internal_imports=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "using .Exporter3: Exporter3 # used at TestModA.jl:") - @test contains(str, "is unused but it was imported from Main.Exporter at TestModC.jl") - - # test `separate_lines=true`` - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - separate_lines=true, - allow_internal_imports=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "using .Exporter3: Exporter3 using .Exporter3: exported_b") - - # `warn_improper_explicit_imports=false` does something (also still no logs) - str_no_warn = @test_logs sprint(io -> print_explicit_imports(io, TestModA, - "TestModA.jl"; - warn_improper_explicit_imports=false)) - str = replace(str, r"\s+" => " ") - @test length(str_no_warn) <= length(str) - - # in particular, this ensures we add `using Foo: Foo` as the first line - @test using_statement.(explicit_imports_nonrecursive(TestMod4, "test_mods.jl")) == - ["using .Exporter4: Exporter4" - "using .Exporter4: A" - "using .Exporter4: Z" - "using .Exporter4: a" - "using .Exporter4: z"] -end + @testset "has_ancestor" begin + @test has_ancestor(TestModA.SubModB, TestModA) + @test !has_ancestor(TestModA, TestModA.SubModB) -@testset "checks" begin - @test check_no_implicit_imports(TestModEmpty, "test_mods.jl") === nothing - @test check_no_stale_explicit_imports(TestModEmpty, "test_mods.jl") === nothing - @test check_no_stale_explicit_imports(TestMod1, "test_mods.jl") === nothing - - @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, - "test_mods.jl") - - # test name ignores - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - ignore=(:print_explicit_imports,)) === nothing - - # test name mod pair ignores - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - ignore=(:print_explicit_imports => ExplicitImports,)) === - nothing - - # if you pass the module in the pair, you must get the right one - @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, - "test_mods.jl"; - ignore=(:print_explicit_imports => TestModA,)) === - nothing - - # non-existent names are OK - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - ignore=(:print_explicit_imports => ExplicitImports, - :does_not_exist)) === nothing - - # you can use skip to skip whole modules - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - skip=(Base, Core, ExplicitImports)) === nothing - - @test_throws ImplicitImportsException check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl") - - # test submodule ignores - @test check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, "TestModC.jl"; - ignore=(TestModA.SubModB.TestModA.TestModC,)) === - nothing - - @test_throws StaleImportsException check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl") - - # make sure ignored names don't show up in error - e = try - check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl"; - ignore=(:exported_d,)) - @test false # should error before this - catch e - e - end - str = sprint(Base.showerror, e) - @test contains(str, "exported_c") - @test !contains(str, "exported_d") - - # ignore works: - @test check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl"; - ignore=(:exported_c, :exported_d)) === - nothing - - # Test the printing is hitting our formatted errors - str = exception_string() do - return check_no_implicit_imports(TestMod1, "test_mods.jl") + @test should_skip(Base.Iterators; skip=(Base, Core)) end - @test contains(str, "is relying on the following implicit imports") - str = exception_string() do - return check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl") + function get_per_scope(per_usage_info) + per_usage_df = DataFrame(per_usage_info) + dropmissing!(per_usage_df, :external_global_name) + return per_usage_df end - @test contains(str, "has stale (unused) explicit imports for:") - @test check_all_explicit_imports_are_public(TestMod1, "test_mods.jl") === nothing - @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, - "imports.jl") - str = exception_string() do - return check_all_explicit_imports_are_public(ModImports, "imports.jl") - end - @test contains(str, "`_svd!` is not public in `LinearAlgebra` but it was imported") - @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; - ignore=(:_svd!, :exported_b, :f, :h, :map)) === - nothing - - @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; - ignore=(:_svd!, :exported_b, :f, :h), - skip=(LinearAlgebra => Base,)) === - nothing - - @testset "Tainted modules" begin - # 3 dynamic include statements - l = (:warn, r"Dynamic") - log = (l, l, l) - - @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl")) == - [DynMod => nothing, DynMod.Hidden => nothing] - @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl"; - strict=false)) == - [DynMod => [(; name=:print_explicit_imports, - source=ExplicitImports)], - # Wrong! Missing explicit export - DynMod.Hidden => []] - - @test_logs log... @test explicit_imports_nonrecursive(DynMod, "DynMod.jl") === - nothing - - @test_logs log... @test only_name_source(explicit_imports_nonrecursive(DynMod, - "DynMod.jl"; - strict=false)) == - [(; name=:print_explicit_imports, source=ExplicitImports)] - - @test @test_logs log... improper_explicit_imports(DynMod, - "DynMod.jl") == - [DynMod => nothing, - DynMod.Hidden => nothing] - - @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, - "DynMod.jl") === - nothing - - @test_logs log... @test improper_explicit_imports(DynMod, - "DynMod.jl"; - strict=false) == - [DynMod => [], - # Wrong! Missing stale explicit export - DynMod.Hidden => []] - - @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, - "DynMod.jl"; - strict=false) == - [] - - str = @test_logs log... sprint(print_explicit_imports, DynMod, "DynMod.jl") - @test contains(str, "DynMod could not be accurately analyzed") - - @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod, - DynMod.Hidden)) === - nothing - - # Ignore also works - @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod,), - ignore=(DynMod.Hidden,)) === - nothing - - e = UnanalyzableModuleException - @test_logs log... @test_throws e check_no_implicit_imports(DynMod, - "DynMod.jl") - - # Missed `Hidden` - @test_logs log... @test_throws e check_no_implicit_imports(DynMod, - "DynMod.jl"; - allow_unanalyzable=(DynMod,),) - - @test_logs log... @test check_no_stale_explicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod, - DynMod.Hidden)) === - nothing - - @test_logs log... @test_throws e check_no_stale_explicit_imports(DynMod, - "DynMod.jl") - - str = sprint(Base.showerror, UnanalyzableModuleException(DynMod)) - @test contains(str, "was found to be unanalyzable") - - @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod, - DynMod.Hidden)) === - nothing - - @test_logs log... @test_throws e check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod,)) + @testset "file not found" begin + for f in (check_no_implicit_imports, check_no_stale_explicit_imports, + check_all_explicit_imports_via_owners, check_all_qualified_accesses_via_owners, + explicit_imports, + explicit_imports_nonrecursive, print_explicit_imports, + improper_explicit_imports, improper_explicit_imports_nonrecursive, + improper_qualified_accesses, improper_qualified_accesses_nonrecursive) + @test_throws FileNotFoundException f(TestModA) + end + str = sprint(Base.showerror, FileNotFoundException()) + @test contains(str, "module which is not top-level in a package") end -end -@testset "Aqua" begin - Aqua.test_all(ExplicitImports; ambiguities=false) -end + @testset "ExplicitImports.jl" begin + @test using_statement.(explicit_imports_nonrecursive(TestModA, "TestModA.jl")) == + ["using .Exporter: Exporter", "using .Exporter: @mac", + "using .Exporter2: Exporter2", + "using .Exporter2: exported_a", "using .Exporter3: Exporter3"] + + per_usage_info, _ = analyze_all_names("TestModA.jl") + df = get_per_scope(per_usage_info) + locals = contains.(string.(df.name), Ref("local")) + @test all(!, df.external_global_name[locals]) + + # we use `x` in two scopes + xs = subset(df, :name => ByRow(==(:x))) + @test !xs[1, :external_global_name] + @test !xs[2, :external_global_name] + @test xs[2, :analysis_code] == ExplicitImports.InternalAssignment + + # we use `exported_a` in two scopes; both times refer to the global name + exported_as = subset(df, :name => ByRow(==(:exported_a))) + @test exported_as[1, :external_global_name] + @test exported_as[2, :external_global_name] + @test !exported_as[2, :is_assignment] + + # Test submodules + @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB, + "TestModA.jl")) == + ["using .Exporter3: Exporter3", "using .Exporter3: exported_b", + "using .TestModA: f"] + + mod_path = module_path(TestModA.SubModB) + @test mod_path == [:SubModB, :TestModA, :Main] + sub_df = restrict_to_module(df, TestModA.SubModB) + + h = only(subset(sub_df, :name => ByRow(==(:h)))) + @test h.external_global_name + @test !h.is_assignment + + # Nested submodule with same name as outer module... + @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA, + "TestModA.jl")) == + ["using .Exporter3: Exporter3", "using .Exporter3: exported_b"] + + # Check we are getting innermost names and not outer ones + subsub_df = restrict_to_module(df, TestModA.SubModB.TestModA) + @test :inner_h in subsub_df.name + @test :h ∉ subsub_df.name + # ...we do currently get the outer ones when the module path prefixes collide + @test_broken :f ∉ subsub_df.name + @test_broken :func ∉ subsub_df.name + + # starts from innermost + @test module_path(TestModA.SubModB.TestModA.TestModC) == + [:TestModC, :TestModA, :SubModB, :TestModA, :Main] + + from_outer_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, + "TestModA.jl")) + from_inner_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl")) + @test from_inner_file == from_outer_file + @test "using .TestModA: f" in from_inner_file + # This one isn't needed bc all usages are fully qualified + @test "using .Exporter: exported_a" ∉ from_inner_file + + # This one isn't needed; it is already explicitly imported + @test "using .Exporter: exported_b" ∉ from_inner_file + + # This one shouldn't be there; we never use it, only explicitly import it. + # So actually it should be on a list of unnecessary imports. BUT it can show up + # because by importing it, we have the name in the file, so we used to detect it. + @test "using .Exporter: exported_c" ∉ from_inner_file + + @test from_inner_file == ["using .TestModA: TestModA", "using .TestModA: f"] + + ret = improper_explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl"; + allow_internal_imports=false) -@testset "`inspect_session`" begin - # We just want to make sure we are robust enough that this doesn't error - big_str = with_logger(Logging.NullLogger()) do - return sprint(inspect_session) - end -end + @test [(; row.name) for row in ret if row.stale] == + [(; name=:exported_c), (; name=:exported_d)] + + # Recursive version + lookup = Dict(improper_explicit_imports(TestModA, + "TestModA.jl"; + allow_internal_imports=false)) + ret = lookup[TestModA.SubModB.TestModA.TestModC] + + @test [(; row.name) for row in ret if row.stale] == + [(; name=:exported_c), (; name=:exported_d)] + @test isempty((row for row in lookup[TestModA] if row.stale)) + + per_usage_info, _ = analyze_all_names("TestModC.jl") + testmodc = DataFrame(per_usage_info) + qualified_row = only(subset(testmodc, :name => ByRow(==(:exported_a)))) + @test qualified_row.analysis_code == ExplicitImports.IgnoredQualified + @test qualified_row.qualified_by == [:Exporter] + + qualified_row2 = only(subset(testmodc, :name => ByRow(==(:h)))) + @test qualified_row2.qualified_by == [:TestModA, :SubModB] + + @test using_statement.(explicit_imports_nonrecursive(TestMod1, + "test_mods.jl")) == + ["using ExplicitImports: print_explicit_imports"] + + # Recursion + nested = explicit_imports(TestModA, "TestModA.jl") + @test nested isa Vector{Pair{Module, + Vector{@NamedTuple{name::Symbol,source::Module, + exporters::Vector{Module},location::String}}}} + @test TestModA in first.(nested) + @test TestModA.SubModB in first.(nested) + @test TestModA.SubModB.TestModA in first.(nested) + @test TestModA.SubModB.TestModA.TestModC in first.(nested) + + # Printing + # should be no logs + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + allow_internal_imports=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "Module Main.TestModA is relying on implicit imports") + @test contains(str, "using .Exporter2: Exporter2, exported_a") + @test contains(str, + "However, module Main.TestModA.SubModB.TestModA.TestModC has stale explicit imports for these 2 unused names") + + # should be no logs + # try with linewidth tiny - should put one name per line + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + linewidth=0)) + @test contains(str, "using .Exporter2: Exporter2,\n exported_a") + + # test `show_locations=true` + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + show_locations=true, + allow_internal_imports=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "using .Exporter3: Exporter3 # used at TestModA.jl:") + @test contains(str, + "is unused but it was imported from Main.Exporter at TestModC.jl") + + # test `separate_lines=true`` + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + separate_lines=true, + allow_internal_imports=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "using .Exporter3: Exporter3 using .Exporter3: exported_b") -@testset "backtick modules and locations" begin - @testset "print_explicit_imports" begin - # Test that module names and file:line locations are surrounded by backticks - # and that underscores in module and file names are printed and do not cause italics. - str = sprint() do io - print_explicit_imports(io, Test_Mod_Underscores, "Test_Mod_Underscores.jl"; report_non_public=true) - end + # `warn_improper_explicit_imports=false` does something (also still no logs) + str_no_warn = @test_logs sprint(io -> print_explicit_imports(io, TestModA, + "TestModA.jl"; + warn_improper_explicit_imports=false)) str = replace(str, r"\s+" => " ") - # stale import - @test contains(str, "Test_Mod_Underscores has stale explicit imports") - @test contains(str, "svd is unused but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") - # non-owner module - @test contains(str, "Test_Mod_Underscores explicitly imports 1 name from non-owner module") - @test contains(str, "map has owner Base but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") - # non-public name - @test contains(str, "Test_Mod_Underscores explicitly imports 1 non-public name") - @test contains(str, "_svd! is not public in LinearAlgebra but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") - # self-qualified access - @test contains(str, "Test_Mod_Underscores has 1 self-qualified access") - @test contains(str, "foo was accessed as Main.Test_Mod_Underscores.foo inside Main.TestModUnderscores at Test_Mod_Underscores.jl") - # access non-owner module - @test contains(str, "Test_Mod_Underscores accesses 1 name from non-owner modules") - @test contains(str, "Number has owner Base but it was accessed from Base.Sys at Test_Mod_Underscores.jl") - # access non-public name - @test contains(str, "Test_Mod_Underscores accesses 1 non-public name") - @test contains(str, "__unsafe_string! is not public in Base but it was accessed via Base at Test_Mod_Underscores.jl") + @test length(str_no_warn) <= length(str) + + # in particular, this ensures we add `using Foo: Foo` as the first line + @test using_statement.(explicit_imports_nonrecursive(TestMod4, "test_mods.jl")) == + ["using .Exporter4: Exporter4" + "using .Exporter4: A" + "using .Exporter4: Z" + "using .Exporter4: a" + "using .Exporter4: z"] end - @testset "check_*" begin - str = exception_string() do - check_no_implicit_imports(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + + @testset "checks" begin + @test check_no_implicit_imports(TestModEmpty, "test_mods.jl") === nothing + @test check_no_stale_explicit_imports(TestModEmpty, "test_mods.jl") === nothing + @test check_no_stale_explicit_imports(TestMod1, "test_mods.jl") === nothing + + @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, + "test_mods.jl") + + # test name ignores + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + ignore=(:print_explicit_imports,)) === nothing + + # test name mod pair ignores + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + ignore=(:print_explicit_imports => ExplicitImports,)) === + nothing + + # if you pass the module in the pair, you must get the right one + @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, + "test_mods.jl"; + ignore=(:print_explicit_imports => TestModA,)) === + nothing + + # non-existent names are OK + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + ignore=(:print_explicit_imports => ExplicitImports, + :does_not_exist)) === nothing + + # you can use skip to skip whole modules + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + skip=(Base, Core, ExplicitImports)) === nothing + + @test_throws ImplicitImportsException check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl") + + # test submodule ignores + @test check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, "TestModC.jl"; + ignore=(TestModA.SubModB.TestModA.TestModC,)) === + nothing + + @test_throws StaleImportsException check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl") + + # make sure ignored names don't show up in error + e = try + check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl"; + ignore=(:exported_d,)) + @test false # should error before this + catch e + e end - @test contains(str, "`Main.Test_Mod_Underscores` is relying on") + str = sprint(Base.showerror, e) + @test contains(str, "exported_c") + @test !contains(str, "exported_d") + + # ignore works: + @test check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl"; + ignore=(:exported_c, :exported_d)) === + nothing + + # Test the printing is hitting our formatted errors str = exception_string() do - check_all_explicit_imports_via_owners(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + return check_no_implicit_imports(TestMod1, "test_mods.jl") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has explicit imports") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + @test contains(str, "is relying on the following implicit imports") + str = exception_string() do - check_all_explicit_imports_are_public(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + return check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + @test contains(str, "has stale (unused) explicit imports for:") + + @test check_all_explicit_imports_are_public(TestMod1, "test_mods.jl") === nothing + @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, + "imports.jl") str = exception_string() do - check_no_stale_explicit_imports(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + return check_all_explicit_imports_are_public(ModImports, "imports.jl") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has stale") - str = exception_string() do - check_all_qualified_accesses_via_owners(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + @test contains(str, "`_svd!` is not public in `LinearAlgebra` but it was imported") + @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; + ignore=(:_svd!, :exported_b, :f, :h, + :map)) === + nothing + + @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; + ignore=(:_svd!, :exported_b, :f, :h), + skip=(LinearAlgebra => Base,)) === + nothing + + @testset "Tainted modules" begin + # 3 dynamic include statements + l = (:warn, r"Dynamic") + log = (l, l, l) + + @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl")) == + [DynMod => nothing, DynMod.Hidden => nothing] + @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl"; + strict=false)) == + [DynMod => [(; name=:print_explicit_imports, + source=ExplicitImports)], + # Wrong! Missing explicit export + DynMod.Hidden => []] + + @test_logs log... @test explicit_imports_nonrecursive(DynMod, "DynMod.jl") === + nothing + + @test_logs log... @test only_name_source(explicit_imports_nonrecursive(DynMod, + "DynMod.jl"; + strict=false)) == + [(; name=:print_explicit_imports, + source=ExplicitImports)] + + @test @test_logs log... improper_explicit_imports(DynMod, + "DynMod.jl") == + [DynMod => nothing, + DynMod.Hidden => nothing] + + @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, + "DynMod.jl") === + nothing + + @test_logs log... @test improper_explicit_imports(DynMod, + "DynMod.jl"; + strict=false) == + [DynMod => [], + # Wrong! Missing stale explicit export + DynMod.Hidden => []] + + @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, + "DynMod.jl"; + strict=false) == + [] + + str = @test_logs log... sprint(print_explicit_imports, DynMod, "DynMod.jl") + @test contains(str, "DynMod could not be accurately analyzed") + + @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod, + DynMod.Hidden)) === + nothing + + # Ignore also works + @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod,), + ignore=(DynMod.Hidden,)) === + nothing + + e = UnanalyzableModuleException + @test_logs log... @test_throws e check_no_implicit_imports(DynMod, + "DynMod.jl") + + # Missed `Hidden` + @test_logs log... @test_throws e check_no_implicit_imports(DynMod, + "DynMod.jl"; + allow_unanalyzable=(DynMod,),) + + @test_logs log... @test check_no_stale_explicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod, + DynMod.Hidden)) === + nothing + + @test_logs log... @test_throws e check_no_stale_explicit_imports(DynMod, + "DynMod.jl") + + str = sprint(Base.showerror, UnanalyzableModuleException(DynMod)) + @test contains(str, "was found to be unanalyzable") + + @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod, + DynMod.Hidden)) === + nothing + + @test_logs log... @test_throws e check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod,)) end - @test contains(str, "Module `Main.Test_Mod_Underscores` has qualified accesses") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") - str = exception_string() do - check_all_qualified_accesses_are_public(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + end + + @testset "Aqua" begin + Aqua.test_all(ExplicitImports; ambiguities=false) + end + + @testset "`inspect_session`" begin + # We just want to make sure we are robust enough that this doesn't error + big_str = with_logger(Logging.NullLogger()) do + return sprint(inspect_session) end - @test contains(str, "Module `Main.Test_Mod_Underscores` has explicit imports of") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") - str = exception_string() do - check_no_self_qualified_accesses(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + end + + @testset "backtick modules and locations" begin + @testset "print_explicit_imports" begin + # Test that module names and file:line locations are surrounded by backticks + # and that underscores in module and file names are printed and do not cause italics. + str = sprint() do io + return print_explicit_imports(io, Test_Mod_Underscores, + "Test_Mod_Underscores.jl"; + report_non_public=true) + end + str = replace(str, r"\s+" => " ") + # stale import + @test contains(str, "Test_Mod_Underscores has stale explicit imports") + @test contains(str, + "svd is unused but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") + # non-owner module + @test contains(str, + "Test_Mod_Underscores explicitly imports 1 name from non-owner module") + @test contains(str, + "map has owner Base but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") + # non-public name + @test contains(str, "Test_Mod_Underscores explicitly imports 1 non-public name") + @test contains(str, + "_svd! is not public in LinearAlgebra but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") + # self-qualified access + @test contains(str, "Test_Mod_Underscores has 1 self-qualified access") + @test contains(str, + "foo was accessed as Main.Test_Mod_Underscores.foo inside Main.TestModUnderscores at Test_Mod_Underscores.jl") + # access non-owner module + @test contains(str, + "Test_Mod_Underscores accesses 1 name from non-owner modules") + @test contains(str, + "Number has owner Base but it was accessed from Base.Sys at Test_Mod_Underscores.jl") + # access non-public name + @test contains(str, "Test_Mod_Underscores accesses 1 non-public name") + @test contains(str, + "__unsafe_string! is not public in Base but it was accessed via Base at Test_Mod_Underscores.jl") + end + @testset "check_*" begin + str = exception_string() do + return check_no_implicit_imports(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "`Main.Test_Mod_Underscores` is relying on") + str = exception_string() do + return check_all_explicit_imports_via_owners(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has explicit imports") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_all_explicit_imports_are_public(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_no_stale_explicit_imports(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has stale") + str = exception_string() do + return check_all_qualified_accesses_via_owners(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has qualified accesses") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_all_qualified_accesses_are_public(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, + "Module `Main.Test_Mod_Underscores` has explicit imports of") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_no_self_qualified_accesses(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, + "Module `Main.Test_Mod_Underscores` has self-qualified accesses") + @test contains(str, + r"accessed as `Main.Test_Mod_Underscores.foo` inside `Main.Test_Mod_Underscores` at `Test_Mod_Underscores.jl:10:40`") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has self-qualified accesses") - @test contains(str, r"accessed as `Main.Test_Mod_Underscores.foo` inside `Main.Test_Mod_Underscores` at `Test_Mod_Underscores.jl:10:40`") end end From c14d8b44018e6c9dcc86dae40ca69a63edf378b8 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 12 Jun 2025 22:39:15 +0200 Subject: [PATCH 02/35] gitignore for ]dev --local --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1c02e5e1..603f9ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.jl.mem Manifest.toml /docs/build/ +dev From 3ec279df467b286238a4d305759eef2407b66900 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 12 Jun 2025 22:43:02 +0200 Subject: [PATCH 03/35] undo simplify hashing hack --- src/get_names_used.jl | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 2373a848..9748580d 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -490,11 +490,6 @@ function analyze_all_names(file) return analyze_per_usage_info(per_usage_info), untainted_modules end -# this would ideally be identity, but hashing SyntaxNode's is slow on v1.0.2 -# https://github.com/JuliaLang/JuliaSyntax.jl/issues/558 -# so we will settle for some unlikely chance at collisions and just check the string rep of the values -_simplify_hashing(scope_path) = map(string ∘ get_val, scope_path) - function is_name_internal_in_higher_local_scope(name, scope_path, seen) # We will recurse up the `scope_path`. Note the order is "reversed", # so the first entry of `scope_path` is deepest. @@ -507,7 +502,7 @@ function is_name_internal_in_higher_local_scope(name, scope_path, seen) end # Ok, now pop off the first scope and check. scope_path = scope_path[2:end] - ret = get(seen, (; name, scope_path=_simplify_hashing(scope_path)), nothing) + ret = get(seen, (; name, scope_path), nothing) if ret === nothing # Not introduced here yet, trying recursing further continue @@ -528,9 +523,9 @@ function analyze_per_usage_info(per_usage_info) # Otherwise, we are in local scope: # 1. Next, if the name is a function arg, then this is not a global name (essentially first usage is assignment) # 2. Otherwise, if first usage is assignment, then it is local, otherwise it is global - seen = Dict{@NamedTuple{name::Symbol,scope_path::Vector{String}},Bool}() + seen = IdDict{@NamedTuple{name::Symbol,scope_path::Vector{JuliaSyntax.SyntaxNode}},Bool}() return map(per_usage_info) do nt - @compat if (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) in keys(seen) + @compat if (; nt.name, nt.scope_path) in keys(seen) return PerUsageInfo(; nt..., first_usage_in_scope=false, external_global_name=missing, analysis_code=IgnoredNonFirst) @@ -563,7 +558,7 @@ function analyze_per_usage_info(per_usage_info) if is_local external_global_name = false push!(seen, - (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) => external_global_name) + (; nt.name, nt.scope_path) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=reason) @@ -575,14 +570,14 @@ function analyze_per_usage_info(per_usage_info) seen) external_global_name = false push!(seen, - (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) => external_global_name) + (; nt.name, nt.scope_path) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=InternalHigherScope) end external_global_name = true push!(seen, - (; nt.name, scope_path=_simplify_hashing(nt.scope_path)) => external_global_name) + (; nt.name, nt.scope_path) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=External) end From e0c4f8df175c5e61258370265f84aab55c071154 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 12 Jun 2025 22:43:54 +0200 Subject: [PATCH 04/35] pkg roundtrip --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9c440d04..a2bca170 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ExplicitImports" uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7" -version = "1.11.3" authors = ["Eric P. Hanson"] +version = "1.11.3" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" From 27fcf8d15b9c0502bb54714b42e364f414e73674 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:27:52 +0200 Subject: [PATCH 05/35] improve debugging --- src/improper_qualified_accesses.jl | 5 ++++- src/parse_utilities.jl | 11 +++++++++++ test/runtests.jl | 11 +++++++---- test/start_tests.jl | 8 ++++++++ 4 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 test/start_tests.jl diff --git a/src/improper_qualified_accesses.jl b/src/improper_qualified_accesses.jl index b9754dd9..8589eada 100644 --- a/src/improper_qualified_accesses.jl +++ b/src/improper_qualified_accesses.jl @@ -11,9 +11,12 @@ function analyze_qualified_names(mod::Module, file=pathof(mod); # something there would invalidate the qualified names with issues we did find. # For now let's ignore it. + @debug "[analyze_qualified_names] per_usage_info has $(length(per_usage_info)) rows" # Filter to qualified names qualified = [row for row in per_usage_info if row.qualified_by !== nothing] + @debug "[analyze_qualified_names] qualified has $(length(qualified)) rows" + # which are in our module mod_path = module_path(mod) match = module_path -> all(Base.splat(isequal), zip(module_path, mod_path)) @@ -54,7 +57,7 @@ end function process_qualified_row(row, mod) # for JET @assert !isnothing(row.qualified_by) - + isempty(row.qualified_by) && return nothing current_mod = mod for submod in row.qualified_by diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index cfab0377..2c4e019f 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -139,6 +139,17 @@ function parents_match(n::TreeCursor, kinds::Tuple) return parents_match(p, Base.tail(kinds)) end + +function parent_kinds(n::TreeCursor) + kinds = [] + while true + n = parent(n) + n === nothing && return kinds + push!(kinds, kind(n)) + end + return kinds +end + function get_parent(n, i=1) for _ in i:-1:1 n = parent(n) diff --git a/test/runtests.jl b/test/runtests.jl index 45707fab..5f902b4d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,7 +25,9 @@ function exception_string(f) catch e sprint(showerror, e) end - @test str isa String + if str === false + error("Expected `f` to throw an exception, but it did not") + end return str end @@ -1046,9 +1048,10 @@ include("Test_Mod_Underscores.jl") end end - @testset "Aqua" begin - Aqua.test_all(ExplicitImports; ambiguities=false) - end + # TODO reenable this + # @testset "Aqua" begin + # Aqua.test_all(ExplicitImports; ambiguities=false) + # end @testset "`inspect_session`" begin # We just want to make sure we are robust enough that this doesn't error diff --git a/test/start_tests.jl b/test/start_tests.jl new file mode 100644 index 00000000..5eef0135 --- /dev/null +++ b/test/start_tests.jl @@ -0,0 +1,8 @@ +using TestEnv # assumed globally installed +using Pkg +Pkg.activate(joinpath(@__DIR__, "..")) +TestEnv.activate() +using ExplicitImports + +cd(joinpath(pkgdir(ExplicitImports), "test")) +include("runtests.jl") From b6a5e4e033a64a867e3d00fd6233c139e5a8ed6e Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:28:09 +0200 Subject: [PATCH 06/35] change for quoting in qualified names (JS#324) --- src/get_names_used.jl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 9748580d..e203f1f3 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -23,6 +23,8 @@ Base.@kwdef struct PerUsageInfo analysis_code::AnalysisCode end +Base.show(io::IO, r::PerUsageInfo) = print(io, "PerUsageInfo (`$(r.name)` @ $(r.location), `qualified_by`=$(r.qualified_by))") + function Base.NamedTuple(r::PerUsageInfo) names = fieldnames(typeof(r)) return NamedTuple{names}(map(x -> getfield(r, x), names)) @@ -53,12 +55,18 @@ end # returns `nothing` for no qualifying module, otherwise a symbol function qualifying_module(leaf) + @debug "[qualifying_module] leaf: $(js_node(leaf)) start" + # introspect leaf and its tree of parents + @debug "[qualifying_module] leaf: $(js_node(leaf)) parents: $(parent_kinds(leaf))" + # is this name being used in a qualified context, like `X.y`? - parents_match(leaf, (K"quote", K".")) || return nothing + parents_match(leaf, (K".",)) || return nothing + @debug "[qualifying_module] leaf: $(js_node(leaf)) passed dot" # Are we on the right-hand side? - child_index(parent(leaf)) == 2 || return nothing + child_index(leaf) == 2 || return nothing + @debug "[qualifying_module] leaf: $(js_node(leaf)) passed right-hand side" # Ok, now try to retrieve the child on the left-side - node = first(AbstractTrees.children(get_parent(leaf, 2))) + node = first(AbstractTrees.children(parent(leaf))) path = Symbol[] retrieve_module_path!(path, node) return path From b3871aa7b102893bf594a32e2ce86ad7a6580034 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 12 Jun 2025 23:56:19 +0200 Subject: [PATCH 07/35] wip: hashing fix --- src/get_names_used.jl | 44 +++++++++++++++++++++++++++++++++++++------ test/runtests.jl | 6 ++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index e203f1f3..17481240 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -499,6 +499,10 @@ function analyze_all_names(file) end function is_name_internal_in_higher_local_scope(name, scope_path, seen) + if name == :QR + println("--------------------------------") + println("is_name_internal_in_higher_local_scope: $name, length(scope_path)=$(length(scope_path)), scope_path= $(map(nodevalue,scope_path)))") + end # We will recurse up the `scope_path`. Note the order is "reversed", # so the first entry of `scope_path` is deepest. @@ -510,7 +514,10 @@ function is_name_internal_in_higher_local_scope(name, scope_path, seen) end # Ok, now pop off the first scope and check. scope_path = scope_path[2:end] - ret = get(seen, (; name, scope_path), nothing) + ret = get(seen, (; name, scope_path=SyntaxNodeList(scope_path)), nothing) + if name == :QR + println("[is_name_internal_in_higher_local_scope] ret: $ret, length(scope_path): $(length(scope_path))") + end if ret === nothing # Not introduced here yet, trying recursing further continue @@ -523,6 +530,31 @@ function is_name_internal_in_higher_local_scope(name, scope_path, seen) return false end +struct SyntaxNodeList + nodes::Vector{JuliaSyntax.SyntaxNode} +end + +Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) = a.nodes == b.nodes + +function syntax_node_weak_hash(node::JuliaSyntax.SyntaxNode, h::UInt) + # We want to hash the node cheaply instead of fully recursively + h = hash(kind(node), h) + for child in js_children(node) + h = hash(kind(child), h) + h = hash(length(js_children(child)), h) + end + return h +end +function Base.hash(a::SyntaxNodeList, h::UInt) + # We implement a workaround for https://github.com/JuliaLang/JuliaSyntax.jl/issues/558 + # There, hashing is extremely slow on SyntaxNode since it was changed to be fully recursive + # We can tolerate some collisions, so we want to implement our own hashing that is not fully recursive. + for node in a.nodes + h = syntax_node_weak_hash(node, h) + end + return h +end + function analyze_per_usage_info(per_usage_info) # For each scope, we want to understand if there are any global usages of the name in that scope # First, throw away all qualified usages, they are irrelevant @@ -531,9 +563,9 @@ function analyze_per_usage_info(per_usage_info) # Otherwise, we are in local scope: # 1. Next, if the name is a function arg, then this is not a global name (essentially first usage is assignment) # 2. Otherwise, if first usage is assignment, then it is local, otherwise it is global - seen = IdDict{@NamedTuple{name::Symbol,scope_path::Vector{JuliaSyntax.SyntaxNode}},Bool}() + seen = Dict{@NamedTuple{name::Symbol,scope_path::SyntaxNodeList},Bool}() return map(per_usage_info) do nt - @compat if (; nt.name, nt.scope_path) in keys(seen) + @compat if (; nt.name, scope_path=SyntaxNodeList(nt.scope_path)) in keys(seen) return PerUsageInfo(; nt..., first_usage_in_scope=false, external_global_name=missing, analysis_code=IgnoredNonFirst) @@ -566,7 +598,7 @@ function analyze_per_usage_info(per_usage_info) if is_local external_global_name = false push!(seen, - (; nt.name, nt.scope_path) => external_global_name) + (; nt.name, scope_path=SyntaxNodeList(nt.scope_path)) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=reason) @@ -578,14 +610,14 @@ function analyze_per_usage_info(per_usage_info) seen) external_global_name = false push!(seen, - (; nt.name, nt.scope_path) => external_global_name) + (; nt.name, scope_path=SyntaxNodeList(nt.scope_path)) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=InternalHigherScope) end external_global_name = true push!(seen, - (; nt.name, nt.scope_path) => external_global_name) + (; nt.name, scope_path=SyntaxNodeList(nt.scope_path)) => external_global_name) return PerUsageInfo(; nt..., first_usage_in_scope=true, external_global_name, analysis_code=External) end diff --git a/test/runtests.jl b/test/runtests.jl index 5f902b4d..bf0fde25 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -498,6 +498,12 @@ include("Test_Mod_Underscores.jl") @test map(get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] + df = DataFrame(get_names_used("test_mods.jl").per_usage_info) + subset!(df, :name => ByRow(==(:QR)), :module_path => ByRow(==([:TestMod5]))) + # two uses of QR: one is as a type param, the other a usage of that type param in the same struct definition + @test df.analysis_code == [ExplicitImports.InternalStruct, ExplicitImports.IgnoredNonFirst] + @test df.struct_field_or_type_param == [true, false] + # Tests #34 and #36 @test using_statement.(explicit_imports_nonrecursive(TestMod5, "test_mods.jl")) == ["using LinearAlgebra: LinearAlgebra"] From 3303b8c22d25d6ea6540aa8862b983468a385dbc Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:07:35 +0200 Subject: [PATCH 08/35] switch to object identity --- src/get_names_used.jl | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 17481240..6e606060 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -499,10 +499,6 @@ function analyze_all_names(file) end function is_name_internal_in_higher_local_scope(name, scope_path, seen) - if name == :QR - println("--------------------------------") - println("is_name_internal_in_higher_local_scope: $name, length(scope_path)=$(length(scope_path)), scope_path= $(map(nodevalue,scope_path)))") - end # We will recurse up the `scope_path`. Note the order is "reversed", # so the first entry of `scope_path` is deepest. @@ -515,9 +511,6 @@ function is_name_internal_in_higher_local_scope(name, scope_path, seen) # Ok, now pop off the first scope and check. scope_path = scope_path[2:end] ret = get(seen, (; name, scope_path=SyntaxNodeList(scope_path)), nothing) - if name == :QR - println("[is_name_internal_in_higher_local_scope] ret: $ret, length(scope_path): $(length(scope_path))") - end if ret === nothing # Not introduced here yet, trying recursing further continue @@ -530,29 +523,19 @@ function is_name_internal_in_higher_local_scope(name, scope_path, seen) return false end +# We implement a workaround for https://github.com/JuliaLang/JuliaSyntax.jl/issues/558 +# Hashing and equality for SyntaxNodes were changed from object identity to a recursive comparison +# in JuliaSyntax 1.0. This is very slow and also not quite the semantics we want anyway. +# Here, we wrap our nodes in a custom type that only compares object identity. struct SyntaxNodeList nodes::Vector{JuliaSyntax.SyntaxNode} end -Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) = a.nodes == b.nodes +Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) = map(objectid, a.nodes) == map(objectid, b.nodes) +Base.isequal(a::SyntaxNodeList, b::SyntaxNodeList) = isequal(map(objectid, a.nodes), map(objectid, b.nodes)) -function syntax_node_weak_hash(node::JuliaSyntax.SyntaxNode, h::UInt) - # We want to hash the node cheaply instead of fully recursively - h = hash(kind(node), h) - for child in js_children(node) - h = hash(kind(child), h) - h = hash(length(js_children(child)), h) - end - return h -end function Base.hash(a::SyntaxNodeList, h::UInt) - # We implement a workaround for https://github.com/JuliaLang/JuliaSyntax.jl/issues/558 - # There, hashing is extremely slow on SyntaxNode since it was changed to be fully recursive - # We can tolerate some collisions, so we want to implement our own hashing that is not fully recursive. - for node in a.nodes - h = syntax_node_weak_hash(node, h) - end - return h + return hash(map(objectid, a.nodes), h) end function analyze_per_usage_info(per_usage_info) From 010701974a41f50d99777452906a7f6c15b76ac3 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:17:00 +0200 Subject: [PATCH 09/35] fix for and generator iteration --- src/get_names_used.jl | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 6e606060..5b03c2c6 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -23,7 +23,10 @@ Base.@kwdef struct PerUsageInfo analysis_code::AnalysisCode end -Base.show(io::IO, r::PerUsageInfo) = print(io, "PerUsageInfo (`$(r.name)` @ $(r.location), `qualified_by`=$(r.qualified_by))") +function Base.show(io::IO, r::PerUsageInfo) + return print(io, + "PerUsageInfo (`$(r.name)` @ $(r.location), `qualified_by`=$(r.qualified_by))") +end function Base.NamedTuple(r::PerUsageInfo) names = fieldnames(typeof(r)) @@ -278,7 +281,16 @@ function in_for_argument_position(node) # We must be on the LHS of a `for` `equal`. if !has_parent(node, 2) return false - elseif parents_match(node, (K"iteration", K"for")) + elseif parents_match(node, (K"in", K"iteration", K"for")) + @debug """ + [in_for_argument_position] node: $(js_node(node)) + parents: $(parent_kinds(node)) + child_index=$(child_index(node)) + parent_child_index=$(child_index(get_parent(node, 1))) + parent_child_index2=$(child_index(get_parent(node, 2))) + """ + + # child_index(node) == 1 means we are the first argument of the `in`, like `yi in y` return child_index(node) == 1 elseif kind(parent(node)) in (K"tuple", K"parameters") return in_for_argument_position(get_parent(node)) @@ -302,8 +314,8 @@ function in_generator_arg_position(node) # (possibly inside a filter, possibly inside a `iteration`) if !has_parent(node, 2) return false - elseif parents_match(node, (K"iteration", K"generator")) || - parents_match(node, (K"iteration", K"filter")) + elseif parents_match(node, (K"in", K"iteration", K"generator")) || + parents_match(node, (K"in", K"iteration", K"filter")) return child_index(node) == 1 elseif kind(parent(node)) in (K"tuple", K"parameters") return in_generator_arg_position(get_parent(node)) @@ -531,8 +543,12 @@ struct SyntaxNodeList nodes::Vector{JuliaSyntax.SyntaxNode} end -Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) = map(objectid, a.nodes) == map(objectid, b.nodes) -Base.isequal(a::SyntaxNodeList, b::SyntaxNodeList) = isequal(map(objectid, a.nodes), map(objectid, b.nodes)) +function Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) + return map(objectid, a.nodes) == map(objectid, b.nodes) +end +function Base.isequal(a::SyntaxNodeList, b::SyntaxNodeList) + return isequal(map(objectid, a.nodes), map(objectid, b.nodes)) +end function Base.hash(a::SyntaxNodeList, h::UInt) return hash(map(objectid, a.nodes), h) From a3afdec13db1b38b06dd30376072b82736ded65a Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:17:56 +0200 Subject: [PATCH 10/35] stricter --- src/get_names_used.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 5b03c2c6..ed9947ed 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -279,7 +279,7 @@ end # https://github.com/JuliaLang/JuliaSyntax.jl/issues/432 function in_for_argument_position(node) # We must be on the LHS of a `for` `equal`. - if !has_parent(node, 2) + if !has_parent(node, 3) return false elseif parents_match(node, (K"in", K"iteration", K"for")) @debug """ @@ -312,7 +312,7 @@ end function in_generator_arg_position(node) # We must be on the LHS of a `=` inside a generator # (possibly inside a filter, possibly inside a `iteration`) - if !has_parent(node, 2) + if !has_parent(node, 3) return false elseif parents_match(node, (K"in", K"iteration", K"generator")) || parents_match(node, (K"in", K"iteration", K"filter")) From 73c8b4b6c85dc25b59f3b4c4052e387ea060af18 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:21:14 +0200 Subject: [PATCH 11/35] inline function defs do not have K"=" anymore --- src/get_names_used.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index ed9947ed..940ebe76 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -242,10 +242,6 @@ function call_is_func_def(node) # note: macros only support full-form function definitions # (not inline) kind(p) in (K"function", K"macro") && return true - if kind(p) == K"=" - # call should be the first arg in an inline function def - return child_index(node) == 1 - end return false end From 43f6b77e455f3529dcb24e2875bcc39f4267dfcd Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:28:05 +0200 Subject: [PATCH 12/35] fix do-blocks --- src/get_names_used.jl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 940ebe76..1219364f 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -150,8 +150,8 @@ function is_anonymous_do_function_definition_arg(leaf) if !has_parent(leaf, 2) return false elseif parents_match(leaf, (K"tuple", K"do")) - # second argument of `do`-block - return child_index(parent(leaf)) == 2 + # first argument of `do`-block (args then function body since JuliaSyntax 1.0) + return child_index(parent(leaf)) == 1 elseif kind(parent(leaf)) in (K"tuple", K"parameters") # Ok, let's just step up one level and see again return is_anonymous_do_function_definition_arg(parent(leaf)) @@ -374,10 +374,7 @@ function analyze_name(leaf; debug=false) # Constructs that start a new local scope. Note `let` & `macro` *arguments* are not explicitly supported/tested yet, # but we can at least keep track of scope properly. if k in - (K"let", K"for", K"function", K"struct", K"generator", K"while", K"macro") || - # Or do-block when we are considering a path that did not go through the first-arg - # (which is the function name, and NOT part of the local scope) - (k == K"do" && child_index(prev_node) > 1) || + (K"let", K"for", K"function", K"struct", K"generator", K"while", K"macro", K"do") || # any child of `try` gets it's own individual scope (I think) (parents_match(node, (K"try",))) push!(scope_path, nodevalue(node).node) From 1099a73a89594b5d2b7506d1ace910ac33d95408 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:31:26 +0200 Subject: [PATCH 13/35] bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a2bca170..d1db8fe0 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ExplicitImports" uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7" authors = ["Eric P. Hanson"] -version = "1.11.3" +version = "1.12.0" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" From 9bb164ad0cf0a26e557944d803e716daea730487 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 00:41:42 +0200 Subject: [PATCH 14/35] support new error on 1.12 --- src/find_implicit_imports.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/find_implicit_imports.jl b/src/find_implicit_imports.jl index 490f78e7..97696697 100644 --- a/src/find_implicit_imports.jl +++ b/src/find_implicit_imports.jl @@ -45,6 +45,7 @@ function find_implicit_imports(mod::Module; skip=(mod, Base, Core)) # `WARNING: both Exporter3 and Exporter2 export "exported_a"; uses of it in module TestModA must be qualified` # and there is an ambiguity, and the name is in fact not resolved in `mod` clash = (err == ErrorException("\"$name\" is not defined in module $mod"))::Bool + clash |= (err == ErrorException("Constant binding was imported from multiple modules"))::Bool # if it is something else, rethrow clash || rethrow() missing From 8cecc34470d11bc6cc20fba4396d6ab8e22b3837 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 19:51:09 +0200 Subject: [PATCH 15/35] wip --- Project.toml | 7 +++- src/improper_explicit_imports.jl | 18 +++++----- test/lowering.jl | 58 ++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 test/lowering.jl diff --git a/Project.toml b/Project.toml index d1db8fe0..e66ec5c3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,17 +1,22 @@ name = "ExplicitImports" uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7" -authors = ["Eric P. Hanson"] version = "1.12.0" +authors = ["Eric P. Hanson"] [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" +JuliaLowering = "f3c80556-a63f-4383-b822-37d64f81a311" JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +[sources] +JuliaLowering = {path = "dev/JuliaLowering"} +JuliaSyntax = {path = "dev/JuliaSyntax"} + [compat] AbstractTrees = "0.4.5" Aqua = "0.8.4" diff --git a/src/improper_explicit_imports.jl b/src/improper_explicit_imports.jl index 901509ae..b7fd3802 100644 --- a/src/improper_explicit_imports.jl +++ b/src/improper_explicit_imports.jl @@ -125,17 +125,17 @@ function process_explicitly_imported_row(row, mod) isempty(explicitly_imported_by) && return nothing - if explicitly_imported_by[end] !== nameof(current_mod) - error(""" - Encountered implementation bug in `process_explicitly_imported_row`. - Please file an issue on ExplicitImports.jl (https://github.com/ericphanson/ExplicitImports.jl/issues/new). + # if explicitly_imported_by[end] !== nameof(current_mod) + # error(""" + # Encountered implementation bug in `process_explicitly_imported_row`. + # Please file an issue on ExplicitImports.jl (https://github.com/ericphanson/ExplicitImports.jl/issues/new). - Info: + # Info: - `explicitly_imported_by`=$(explicitly_imported_by) - `nameof(current_mod)`=$(nameof(current_mod)) - """) - end + # `explicitly_imported_by`=$(explicitly_imported_by) + # `nameof(current_mod)`=$(nameof(current_mod)) + # """) + # end # Ok, now `current_mod` should contain the actual module we imported the name from # This lets us query if the name is public in *that* module, get the value, etc diff --git a/test/lowering.jl b/test/lowering.jl new file mode 100644 index 00000000..3fb8d520 --- /dev/null +++ b/test/lowering.jl @@ -0,0 +1,58 @@ +using JuliaLowering, JuliaSyntax +using JuliaLowering: SyntaxTree, ensure_attributes, showprov +using AbstractTrees +# piracy +AbstractTrees.children(t::SyntaxTree) = something(JuliaSyntax.children(t), ()) + +include("test_mods.jl") + +src = read("test_mods.jl", String) +tree = parseall(JuliaLowering.SyntaxTree, src; filename="tests_mods.jl") + +testmod1_code = JuliaSyntax.children(JuliaSyntax.children(tree)[2])[2] +func = JuliaSyntax.children(testmod1_code)[end - 1] + +leaf = JuliaSyntax.children(func)[2] + +ex = testmod1_code +ex = ensure_attributes(ex; var_id=Int) + +in_mod = TestMod1 +# in_mod=Main +ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex) +ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) +ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) + +leaf = collect(Leaves(ex_scoped))[end - 3] +showprov(leaf) + +binding_info = ctx3.bindings.info[leaf.var_id] +binding_info.kind == :global + +global_bindings = filter(ctx3.bindings.info) do binding + # want globals + keep = binding_info.kind == :global + + # internal ones seem non-interesting (`#self#` etc) + keep &= !binding.is_internal + + # I think we want ones that aren't assigned to? otherwise we are _defining_ the global here, not using it + keep &= binding.n_assigned == 0 + return keep +end + + +# notes +# global names seem "easy": they show up as BindingID in the source tree and have an info populated in `ctx.binding.info` +# qualified names seem a bit harder, they show up like this: +# +# [call] │ +# top.getproperty :: top │ +# #₈/ExplicitImports :: BindingId │ +# :check_no_implicit_imports :: Symbol │ scope_layer=1 +# +# so here `check_no_implicit_imports` is a qualified name, we can see it as a child of call, +# where we are calling getproperty on ExplicitImports and `check_no_implicit_imports`. +# so if we want to check you are calling it from the "right" module, we need to follow the tree, +# find this pattern, then check the module against the symbol. +# That's what we already do, but now we should have more precision in knowing the module I think From 7f21afd53bf142245539ddc4fba886cd1cee0239 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 22:35:13 +0200 Subject: [PATCH 16/35] support module aliases --- .gitignore | 2 ++ Project.toml | 2 +- src/improper_explicit_imports.jl | 12 ------------ test/module_alias.jl | 12 ++++++++++++ test/runtests.jl | 8 ++++++++ 5 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 test/module_alias.jl diff --git a/.gitignore b/.gitignore index 1c02e5e1..ad7d911e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ *.jl.cov *.jl.mem Manifest.toml +Manifest-v*.toml /docs/build/ +dev diff --git a/Project.toml b/Project.toml index 23ac81bd..319c010f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ExplicitImports" uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7" -version = "1.11.2" +version = "1.11.3" authors = ["Eric P. Hanson"] [deps] diff --git a/src/improper_explicit_imports.jl b/src/improper_explicit_imports.jl index 901509ae..69acfb90 100644 --- a/src/improper_explicit_imports.jl +++ b/src/improper_explicit_imports.jl @@ -125,18 +125,6 @@ function process_explicitly_imported_row(row, mod) isempty(explicitly_imported_by) && return nothing - if explicitly_imported_by[end] !== nameof(current_mod) - error(""" - Encountered implementation bug in `process_explicitly_imported_row`. - Please file an issue on ExplicitImports.jl (https://github.com/ericphanson/ExplicitImports.jl/issues/new). - - Info: - - `explicitly_imported_by`=$(explicitly_imported_by) - `nameof(current_mod)`=$(nameof(current_mod)) - """) - end - # Ok, now `current_mod` should contain the actual module we imported the name from # This lets us query if the name is public in *that* module, get the value, etc value = trygetproperty(current_mod, row.name) diff --git a/test/module_alias.jl b/test/module_alias.jl new file mode 100644 index 00000000..07096bf6 --- /dev/null +++ b/test/module_alias.jl @@ -0,0 +1,12 @@ +# https://github.com/ericphanson/ExplicitImports.jl/issues/106 +module ModAlias + +module M1 + const g = 9.8 +end + +const M1′ = M1 + +using .M1′: g + +end # module ModAlias diff --git a/test/runtests.jl b/test/runtests.jl index 5ef97315..d753167a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -73,6 +73,7 @@ include("test_qualified_access.jl") include("test_explicit_imports.jl") include("main.jl") include("Test_Mod_Underscores.jl") +include("module_alias.jl") # For deprecations, we are using `maxlog`, which # the TestLogger only respects in Julia 1.8+. @@ -101,6 +102,13 @@ if VERSION > v"1.9-" end end +@testset "module aliases (#106)" begin + # https://github.com/ericphanson/ExplicitImports.jl/issues/106 + ret = Dict(improper_explicit_imports(ModAlias, "module_alias.jl")) + @test isempty(ret[ModAlias]) + @test isempty(ret[ModAlias.M1]) +end + @testset "function arg bug" begin # https://github.com/ericphanson/ExplicitImports.jl/issues/62 df = DataFrame(get_names_used("test_mods.jl").per_usage_info) From ed96729820c2e27a05f0bdda270433a19d0afd17 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 22:41:20 +0200 Subject: [PATCH 17/35] format tests & wrap in testset --- test/runtests.jl | 1936 ++++++++++++++++++++++++---------------------- 1 file changed, 995 insertions(+), 941 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index d753167a..1b6315fe 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,7 +25,9 @@ function exception_string(f) catch e sprint(showerror, e) end - @test str isa String + if str === false + error("Expected `f` to throw an exception, but it did not") + end return str end @@ -75,1033 +77,1085 @@ include("main.jl") include("Test_Mod_Underscores.jl") include("module_alias.jl") -# For deprecations, we are using `maxlog`, which -# the TestLogger only respects in Julia 1.8+. -# (https://github.com/JuliaLang/julia/commit/02f7332027bd542b0701956a0f838bc75fa2eebd) -if VERSION >= v"1.8-" - @testset "deprecations" begin - include("deprecated.jl") +@testset "ExplicitImports" begin + # For deprecations, we are using `maxlog`, which + # the TestLogger only respects in Julia 1.8+. + # (https://github.com/JuliaLang/julia/commit/02f7332027bd542b0701956a0f838bc75fa2eebd) + if VERSION >= v"1.8-" + @testset "deprecations" begin + include("deprecated.jl") + end end -end -# package extension support needs Julia 1.9+ -if VERSION > v"1.9-" - @testset "Extensions" begin - submods = ExplicitImports.find_submodules(TestPkg) - @test length(submods) == 2 - DataFramesExt = Base.get_extension(TestPkg, :DataFramesExt) - @test haskey(Dict(submods), DataFramesExt) - - ext_imports = Dict(only_name_source(explicit_imports(TestPkg)))[DataFramesExt] - @test ext_imports == [(; name=:DataFrames, source=DataFrames), - (; name=:DataFrame, source=DataFrames), - (; name=:groupby, source=DataFrames)] || - ext_imports == [(; name=:DataFrames, source=DataFrames), - (; name=:DataFrame, source=DataFrames), - (; name=:groupby, source=DataFrames.DataAPI)] + # package extension support needs Julia 1.9+ + if VERSION > v"1.9-" + @testset "Extensions" begin + submods = ExplicitImports.find_submodules(TestPkg) + @test length(submods) == 2 + DataFramesExt = Base.get_extension(TestPkg, :DataFramesExt) + @test haskey(Dict(submods), DataFramesExt) + + ext_imports = Dict(only_name_source(explicit_imports(TestPkg)))[DataFramesExt] + @test ext_imports == [(; name=:DataFrames, source=DataFrames), + (; name=:DataFrame, source=DataFrames), + (; name=:groupby, source=DataFrames)] || + ext_imports == [(; name=:DataFrames, source=DataFrames), + (; name=:DataFrame, source=DataFrames), + (; name=:groupby, source=DataFrames.DataAPI)] + end end -end -@testset "module aliases (#106)" begin - # https://github.com/ericphanson/ExplicitImports.jl/issues/106 - ret = Dict(improper_explicit_imports(ModAlias, "module_alias.jl")) - @test isempty(ret[ModAlias]) - @test isempty(ret[ModAlias.M1]) -end + @testset "module aliases (#106)" begin + # https://github.com/ericphanson/ExplicitImports.jl/issues/106 + ret = Dict(improper_explicit_imports(ModAlias, "module_alias.jl")) + @test isempty(ret[ModAlias]) + @test isempty(ret[ModAlias.M1]) + end -@testset "function arg bug" begin - # https://github.com/ericphanson/ExplicitImports.jl/issues/62 - df = DataFrame(get_names_used("test_mods.jl").per_usage_info) - subset!(df, :name => ByRow(==(:norm)), :module_path => ByRow(==([:TestMod13]))) + @testset "function arg bug" begin + # https://github.com/ericphanson/ExplicitImports.jl/issues/62 + df = DataFrame(get_names_used("test_mods.jl").per_usage_info) + subset!(df, :name => ByRow(==(:norm)), :module_path => ByRow(==([:TestMod13]))) - @test_broken check_no_stale_explicit_imports(TestMod13, "test_mods.jl") === nothing -end + @test_broken check_no_stale_explicit_imports(TestMod13, "test_mods.jl") === nothing + end -@testset "owner_mod_for_printing" begin - @test owner_mod_for_printing(Core, :throw, Core.throw) == Base - @test owner_mod_for_printing(Core, :println, Core.println) == Core -end + @testset "owner_mod_for_printing" begin + @test owner_mod_for_printing(Core, :throw, Core.throw) == Base + @test owner_mod_for_printing(Core, :println, Core.println) == Core + end -# https://github.com/ericphanson/ExplicitImports.jl/issues/69 -@testset "Reexport support" begin - @test check_no_stale_explicit_imports(TestMod15, "test_mods.jl") === nothing - @test isempty(improper_explicit_imports_nonrecursive(TestMod15, "test_mods.jl")) - @test isempty(improper_explicit_imports(TestMod15, "test_mods.jl")[1][2]) -end + # https://github.com/ericphanson/ExplicitImports.jl/issues/69 + @testset "Reexport support" begin + @test check_no_stale_explicit_imports(TestMod15, "test_mods.jl") === nothing + @test isempty(improper_explicit_imports_nonrecursive(TestMod15, "test_mods.jl")) + @test isempty(improper_explicit_imports(TestMod15, "test_mods.jl")[1][2]) + end -if VERSION >= v"1.7-" - # https://github.com/ericphanson/ExplicitImports.jl/issues/70 - @testset "Compat skipping" begin - @test check_all_explicit_imports_via_owners(TestMod14, "test_mods.jl") === nothing - @test check_all_qualified_accesses_via_owners(TestMod14, "test_mods.jl") === nothing + if VERSION >= v"1.7-" + # https://github.com/ericphanson/ExplicitImports.jl/issues/70 + @testset "Compat skipping" begin + @test check_all_explicit_imports_via_owners(TestMod14, "test_mods.jl") === + nothing + @test check_all_qualified_accesses_via_owners(TestMod14, "test_mods.jl") === + nothing - @test isempty(improper_explicit_imports_nonrecursive(TestMod14, "test_mods.jl")) - @test isempty(improper_explicit_imports(TestMod14, "test_mods.jl")[1][2]) + @test isempty(improper_explicit_imports_nonrecursive(TestMod14, "test_mods.jl")) + @test isempty(improper_explicit_imports(TestMod14, "test_mods.jl")[1][2]) - @test isempty(improper_qualified_accesses_nonrecursive(TestMod14, "test_mods.jl")) + @test isempty(improper_qualified_accesses_nonrecursive(TestMod14, + "test_mods.jl")) - @test isempty(improper_qualified_accesses(TestMod14, "test_mods.jl")[1][2]) + @test isempty(improper_qualified_accesses(TestMod14, "test_mods.jl")[1][2]) + end end -end -@testset "imports" begin - cursor = TreeCursor(SyntaxNodeWrapper("imports.jl")) - leaves = collect(Leaves(cursor)) - import_type_pairs = get_val.(leaves) .=> analyze_import_type.(leaves) - filter!(import_type_pairs) do (k, v) - return v !== :not_import + @testset "imports" begin + cursor = TreeCursor(SyntaxNodeWrapper("imports.jl")) + leaves = collect(Leaves(cursor)) + import_type_pairs = get_val.(leaves) .=> analyze_import_type.(leaves) + filter!(import_type_pairs) do (k, v) + return v !== :not_import + end + @test import_type_pairs == + [:Exporter => :import_LHS, + :exported_a => :import_RHS, + :exported_c => :import_RHS, + :Exporter => :import_LHS, + :exported_c => :import_RHS, + :TestModA => :blanket_using_member, + :SubModB => :blanket_using, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :h2 => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :h3 => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :h => :import_RHS, + :Exporter => :blanket_using, + :Exporter => :plain_import, + :LinearAlgebra => :import_LHS, + :map => :import_RHS, + :_svd! => :import_RHS, + :LinearAlgebra => :import_LHS, + :svd => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :exported_b => :import_RHS, + :TestModA => :import_LHS, + :SubModB => :import_LHS, + :f => :import_RHS] + + inds = findall(==(:import_RHS), analyze_import_type.(leaves)) + lhs_rhs_pairs = get_import_lhs.(leaves[inds]) .=> get_val.(leaves[inds]) + @test lhs_rhs_pairs == [[:., :., :Exporter] => :exported_a, + [:., :., :Exporter] => :exported_c, + [:., :., :Exporter] => :exported_c, + [:., :., :TestModA, :SubModB] => :h2, + [:., :., :TestModA, :SubModB] => :h3, + [:., :., :TestModA, :SubModB] => :h, + [:LinearAlgebra] => :map, + [:LinearAlgebra] => :_svd!, + [:LinearAlgebra] => :svd, + [:., :., :TestModA, :SubModB] => :exported_b, + [:., :., :TestModA, :SubModB] => :f] + + imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; + allow_internal_imports=false)) + h_row = only(subset(imps, :name => ByRow(==(:h)))) + @test !h_row.public_import + # Note: if this fails locally, try `include("imports.jl")` to rebuild the module + @test h_row.whichmodule == TestModA.SubModB + @test h_row.importing_from == TestModA.SubModB + + h2_row = only(subset(imps, :name => ByRow(==(:h2)))) + @test h2_row.public_import + @test h2_row.whichmodule === TestModA.SubModB + @test h2_row.importing_from == TestModA.SubModB + _svd!_row = only(subset(imps, :name => ByRow(==(:_svd!)))) + @test !_svd!_row.public_import + + f_row = only(subset(imps, :name => ByRow(==(:f)))) + @test !f_row.public_import # not public in `TestModA.SubModB` + @test f_row.whichmodule == TestModA + @test f_row.importing_from == TestModA.SubModB + + imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; + allow_internal_imports=true)) + # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: + @test all(==(LinearAlgebra), imps.importing_from) end - @test import_type_pairs == - [:Exporter => :import_LHS, - :exported_a => :import_RHS, - :exported_c => :import_RHS, - :Exporter => :import_LHS, - :exported_c => :import_RHS, - :TestModA => :blanket_using_member, - :SubModB => :blanket_using, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :h2 => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :h3 => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :h => :import_RHS, - :Exporter => :blanket_using, - :Exporter => :plain_import, - :LinearAlgebra => :import_LHS, - :map => :import_RHS, - :_svd! => :import_RHS, - :LinearAlgebra => :import_LHS, - :svd => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :exported_b => :import_RHS, - :TestModA => :import_LHS, - :SubModB => :import_LHS, - :f => :import_RHS] - - inds = findall(==(:import_RHS), analyze_import_type.(leaves)) - lhs_rhs_pairs = get_import_lhs.(leaves[inds]) .=> get_val.(leaves[inds]) - @test lhs_rhs_pairs == [[:., :., :Exporter] => :exported_a, - [:., :., :Exporter] => :exported_c, - [:., :., :Exporter] => :exported_c, - [:., :., :TestModA, :SubModB] => :h2, - [:., :., :TestModA, :SubModB] => :h3, - [:., :., :TestModA, :SubModB] => :h, - [:LinearAlgebra] => :map, - [:LinearAlgebra] => :_svd!, - [:LinearAlgebra] => :svd, - [:., :., :TestModA, :SubModB] => :exported_b, - [:., :., :TestModA, :SubModB] => :f] - - imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; - allow_internal_imports=false)) - h_row = only(subset(imps, :name => ByRow(==(:h)))) - @test !h_row.public_import - # Note: if this fails locally, try `include("imports.jl")` to rebuild the module - @test h_row.whichmodule == TestModA.SubModB - @test h_row.importing_from == TestModA.SubModB - - h2_row = only(subset(imps, :name => ByRow(==(:h2)))) - @test h2_row.public_import - @test h2_row.whichmodule === TestModA.SubModB - @test h2_row.importing_from == TestModA.SubModB - _svd!_row = only(subset(imps, :name => ByRow(==(:_svd!)))) - @test !_svd!_row.public_import - - f_row = only(subset(imps, :name => ByRow(==(:f)))) - @test !f_row.public_import # not public in `TestModA.SubModB` - @test f_row.whichmodule == TestModA - @test f_row.importing_from == TestModA.SubModB - - imps = DataFrame(improper_explicit_imports_nonrecursive(ModImports, "imports.jl"; - allow_internal_imports=true)) - # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: - @test all(==(LinearAlgebra), imps.importing_from) -end -##### -##### To analyze a test case -##### -# using ExplicitImports: js_node, get_parent, kind, parents_match -# using JuliaSyntax: @K_str - -# cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")); -# leaves = collect(Leaves(cursor)) -# leaf = leaves[end - 2] # select a leaf -# js_node(leaf) # inspect it -# p = js_node(get_parent(leaf, 3)) # see the tree, etc -# kind(p) - -@testset "qualified access" begin - # analyze_qualified_names - qualified = analyze_qualified_names(TestQualifiedAccess, "test_qualified_access.jl") - @test length(qualified) == 6 - ABC, DEF, HIJ, X, map, x = qualified - @test ABC.name == :ABC - @test DEF.public_access - @test HIJ.public_access - @test DEF.name == :DEF - @test HIJ.name == :HIJ - @test X.name == :X - @test map.name == :map - @test x.name == :x - @test x.self_qualified - - # improper_qualified_accesses - ret = Dict(improper_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false)) - @test isempty(ret[TestQualifiedAccess.Bar]) - @test isempty(ret[TestQualifiedAccess.FooModule]) - @test !isempty(ret[TestQualifiedAccess]) - - @test length(ret[TestQualifiedAccess]) == 4 - ABC, X, map, x = ret[TestQualifiedAccess] - # Can add keys, but removing them is breaking - @test keys(ABC) ⊇ - [:name, :location, :value, :accessing_from, :whichmodule, :public_access, - :accessing_from_owns_name, :accessing_from_submodule_owns_name, :internal_access] - @test ABC.name == :ABC - @test ABC.location isa AbstractString - @test ABC.whichmodule == TestQualifiedAccess.Bar - @test ABC.accessing_from == TestQualifiedAccess.FooModule - @test ABC.public_access == false - @test ABC.accessing_from_submodule_owns_name == false - - @test X.name == :X - @test X.whichmodule == TestQualifiedAccess.FooModule.FooSub - @test X.accessing_from == TestQualifiedAccess.FooModule - @test X.public_access == false - @test X.accessing_from_submodule_owns_name == true - - @test map.name == :map - - @test x.name == :x - @test x.self_qualified - - imps = DataFrame(improper_qualified_accesses_nonrecursive(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=true)) - subset!(imps, :self_qualified => ByRow(!)) # drop self-qualified - # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: - @test all(==(LinearAlgebra), imps.accessing_from) - - # check_no_self_qualified_accesses - ex = SelfQualifiedAccessException - @test_throws ex check_no_self_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl") - - str = exception_string() do - return check_no_self_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl") - end - @test contains(str, "has self-qualified accesses:\n- `x` was accessed as") - - @test check_no_self_qualified_accesses(TestQualifiedAccess, - "test_qualified_access.jl"; ignore=(:x,)) === - nothing - - str = sprint(print_explicit_imports, TestQualifiedAccess, - "test_qualified_access.jl") - @test contains(str, "has 1 self-qualified access:\n\n • x was accessed as ") - - # check_all_qualified_accesses_via_owners - ex = QualifiedAccessesFromNonOwnerException - @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - - # Test the printing is hitting our formatted errors - str = exception_string() do - return check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - end - @test contains(str, - "has qualified accesses to names via modules other than their owner as determined") + ##### + ##### To analyze a test case + ##### + # using ExplicitImports: js_node, get_parent, kind, parents_match + # using JuliaSyntax: @K_str + + # cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")); + # leaves = collect(Leaves(cursor)) + # leaf = leaves[end - 2] # select a leaf + # js_node(leaf) # inspect it + # p = js_node(get_parent(leaf, 3)) # see the tree, etc + # kind(p) + + @testset "qualified access" begin + # analyze_qualified_names + qualified = analyze_qualified_names(TestQualifiedAccess, "test_qualified_access.jl") + @test length(qualified) == 6 + ABC, DEF, HIJ, X, map, x = qualified + @test ABC.name == :ABC + @test DEF.public_access + @test HIJ.public_access + @test DEF.name == :DEF + @test HIJ.name == :HIJ + @test X.name == :X + @test map.name == :map + @test x.name == :x + @test x.self_qualified + + # improper_qualified_accesses + ret = Dict(improper_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false)) + @test isempty(ret[TestQualifiedAccess.Bar]) + @test isempty(ret[TestQualifiedAccess.FooModule]) + @test !isempty(ret[TestQualifiedAccess]) + + @test length(ret[TestQualifiedAccess]) == 4 + ABC, X, map, x = ret[TestQualifiedAccess] + # Can add keys, but removing them is breaking + @test keys(ABC) ⊇ + [:name, :location, :value, :accessing_from, :whichmodule, :public_access, + :accessing_from_owns_name, :accessing_from_submodule_owns_name, + :internal_access] + @test ABC.name == :ABC + @test ABC.location isa AbstractString + @test ABC.whichmodule == TestQualifiedAccess.Bar + @test ABC.accessing_from == TestQualifiedAccess.FooModule + @test ABC.public_access == false + @test ABC.accessing_from_submodule_owns_name == false + + @test X.name == :X + @test X.whichmodule == TestQualifiedAccess.FooModule.FooSub + @test X.accessing_from == TestQualifiedAccess.FooModule + @test X.public_access == false + @test X.accessing_from_submodule_owns_name == true + + @test map.name == :map + + @test x.name == :x + @test x.self_qualified + + imps = DataFrame(improper_qualified_accesses_nonrecursive(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=true)) + subset!(imps, :self_qualified => ByRow(!)) # drop self-qualified + # in this case we rule out all the `Main` ones, so only LinearAlgebra is left: + @test all(==(LinearAlgebra), imps.accessing_from) + + # check_no_self_qualified_accesses + ex = SelfQualifiedAccessException + @test_throws ex check_no_self_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl") - skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, ignore=(:map,), - allow_internal_accesses=false) === - nothing + str = exception_string() do + return check_no_self_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl") + end + @test contains(str, "has self-qualified accesses:\n- `x` was accessed as") - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - ignore=(:ABC, :map), - allow_internal_accesses=false) === nothing + @test check_no_self_qualified_accesses(TestQualifiedAccess, + "test_qualified_access.jl"; ignore=(:x,)) === + nothing - # allow_internal_accesses=true - @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl", - ignore=(:ABC,)) + str = sprint(print_explicit_imports, TestQualifiedAccess, + "test_qualified_access.jl") + @test contains(str, "has 1 self-qualified access:\n\n • x was accessed as ") - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - ignore=(:ABC, :map)) === nothing - - @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, - require_submodule_access=true, - allow_internal_accesses=false) - - skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar, - TestQualifiedAccess.FooModule => TestQualifiedAccess.FooModule.FooSub, - LinearAlgebra => Base) - @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, - require_submodule_access=true, - allow_internal_accesses=false) === nothing - - # Printing via `print_explicit_imports` - str = sprint(io -> print_explicit_imports(io, TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "accesses 2 names from non-owner modules") - @test contains(str, "ABC has owner") - - ex = NonPublicQualifiedAccessException - @test_throws ex check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - str = exception_string() do - return check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - allow_internal_accesses=false) - end - @test contains(str, "- `ABC` is not public in") + # check_all_qualified_accesses_via_owners + ex = QualifiedAccessesFromNonOwnerException + @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) - @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + # Test the printing is hitting our formatted errors + str = exception_string() do + return check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) + end + @test contains(str, + "has qualified accesses to names via modules other than their owner as determined") + + skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, ignore=(:map,), + allow_internal_accesses=false) === + nothing + + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:ABC, :map), + allow_internal_accesses=false) === + nothing + + # allow_internal_accesses=true + @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl", + ignore=(:ABC,)) + + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:ABC, :map)) === nothing + + @test_throws ex check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, + require_submodule_access=true, + allow_internal_accesses=false) + + skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar, + TestQualifiedAccess.FooModule => TestQualifiedAccess.FooModule.FooSub, + LinearAlgebra => Base) + @test check_all_qualified_accesses_via_owners(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, + require_submodule_access=true, + allow_internal_accesses=false) === + nothing + + # Printing via `print_explicit_imports` + str = sprint(io -> print_explicit_imports(io, TestQualifiedAccess, "test_qualified_access.jl"; - ignore=(:X, :ABC, :map), - allow_internal_accesses=false) === nothing - - skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) + allow_internal_accesses=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "accesses 2 names from non-owner modules") + @test contains(str, "ABC has owner") - @test check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - skip, ignore=(:X, :map), - allow_internal_accesses=false) === nothing + ex = NonPublicQualifiedAccessException + @test_throws ex check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) + str = exception_string() do + return check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + allow_internal_accesses=false) + end + @test contains(str, "- `ABC` is not public in") + + @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:X, :ABC, :map), + allow_internal_accesses=false) === + nothing + + skip = (TestQualifiedAccess.FooModule => TestQualifiedAccess.Bar,) + + @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + skip, ignore=(:X, :map), + allow_internal_accesses=false) === + nothing + + # allow_internal_accesses=true + @test check_all_qualified_accesses_are_public(TestQualifiedAccess, + "test_qualified_access.jl"; + ignore=(:map,)) === nothing + end - # allow_internal_accesses=true - @test check_all_qualified_accesses_are_public(TestQualifiedAccess, - "test_qualified_access.jl"; - ignore=(:map,)) === nothing -end + @testset "improper explicit imports" begin + imps = Dict(improper_explicit_imports(TestModA, "TestModA.jl"; + allow_internal_imports=false)) + row = only(imps[TestModA]) + @test row.name == :un_exported + @test row.whichmodule == Exporter + + row1, row2 = imps[TestModA.SubModB.TestModA.TestModC] + # Can add keys, but removing them is breaking + @test keys(row1) ⊇ + [:name, :location, :value, :importing_from, :whichmodule, :public_import, + :importing_from_owns_name, :importing_from_submodule_owns_name, :stale, + :internal_import] + @test row1.name == :exported_c + @test row1.stale == true + @test row2.name == :exported_d + @test row2.stale == true + + @test check_all_explicit_imports_via_owners(TestModA, "TestModA.jl"; + allow_internal_imports=false) === + nothing + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, + "imports.jl"; + allow_internal_imports=false) + + # allow_internal_imports=true + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, + "imports.jl";) + @test check_all_explicit_imports_via_owners(ModImports, + "imports.jl"; ignore=(:map,)) === + nothing + # Test the printing is hitting our formatted errors + str = exception_string() do + return check_all_explicit_imports_via_owners(ModImports, + "imports.jl"; + allow_internal_imports=false) + end -@testset "improper explicit imports" begin - imps = Dict(improper_explicit_imports(TestModA, "TestModA.jl"; - allow_internal_imports=false)) - row = only(imps[TestModA]) - @test row.name == :un_exported - @test row.whichmodule == Exporter - - row1, row2 = imps[TestModA.SubModB.TestModA.TestModC] - # Can add keys, but removing them is breaking - @test keys(row1) ⊇ - [:name, :location, :value, :importing_from, :whichmodule, :public_import, - :importing_from_owns_name, :importing_from_submodule_owns_name, :stale, - :internal_import] - @test row1.name == :exported_c - @test row1.stale == true - @test row2.name == :exported_d - @test row2.stale == true - - @test check_all_explicit_imports_via_owners(TestModA, "TestModA.jl"; - allow_internal_imports=false) === nothing - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, - "imports.jl"; - allow_internal_imports=false) - - # allow_internal_imports=true - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(ModImports, - "imports.jl";) - @test check_all_explicit_imports_via_owners(ModImports, - "imports.jl"; ignore=(:map,)) === nothing - # Test the printing is hitting our formatted errors - str = exception_string() do - return check_all_explicit_imports_via_owners(ModImports, - "imports.jl"; - allow_internal_imports=false) + @test contains(str, + "explicit imports of names from modules other than their owner as determined ") + + @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; + ignore=(:exported_b, :f, :map), + allow_internal_imports=false) === + nothing + + # We can pass `skip` to ignore non-owning explicit imports from LinearAlgebra that are owned by Base + @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; + skip=(LinearAlgebra => Base,), + ignore=(:exported_b, :f), + allow_internal_imports=false) === + nothing + + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + allow_internal_imports=false) + + # test ignore + @test check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + ignore=(:ABC,), + allow_internal_imports=false) === + nothing + + # test skip + @test check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + skip=(TestExplicitImports.FooModule => TestExplicitImports.Bar,), + allow_internal_imports=false) === + nothing + + @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + ignore=(:ABC,), + require_submodule_import=true, + allow_internal_imports=false) + + @test check_all_explicit_imports_via_owners(TestExplicitImports, + "test_explicit_imports.jl"; + ignore=(:ABC, :X), + require_submodule_import=true, + allow_internal_imports=false) === + nothing + + # allow_internal_imports = true + @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, + "imports.jl";) + @test check_all_explicit_imports_are_public(ModImports, + "imports.jl"; ignore=(:map, :_svd!)) === + nothing end - @test contains(str, - "explicit imports of names from modules other than their owner as determined ") - - @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; - ignore=(:exported_b, :f, :map), - allow_internal_imports=false) === nothing - - # We can pass `skip` to ignore non-owning explicit imports from LinearAlgebra that are owned by Base - @test check_all_explicit_imports_via_owners(ModImports, "imports.jl"; - skip=(LinearAlgebra => Base,), - ignore=(:exported_b, :f), - allow_internal_imports=false) === nothing - - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - allow_internal_imports=false) - - # test ignore - @test check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - ignore=(:ABC,), - allow_internal_imports=false) === nothing - - # test skip - @test check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - skip=(TestExplicitImports.FooModule => TestExplicitImports.Bar,), - allow_internal_imports=false) === - nothing - - @test_throws ExplicitImportsFromNonOwnerException check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - ignore=(:ABC,), - require_submodule_import=true, - allow_internal_imports=false) - - @test check_all_explicit_imports_via_owners(TestExplicitImports, - "test_explicit_imports.jl"; - ignore=(:ABC, :X), - require_submodule_import=true, - allow_internal_imports=false) === nothing - - # allow_internal_imports = true - @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, - "imports.jl";) - @test check_all_explicit_imports_are_public(ModImports, - "imports.jl"; ignore=(:map, :_svd!)) === - nothing -end + @testset "structs" begin + cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) + leaves = collect(Leaves(cursor)) + @test map(get_val, filter(is_struct_type_param, leaves)) == [:X, :Y, :QR] -@testset "structs" begin - cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) - leaves = collect(Leaves(cursor)) - @test map(get_val, filter(is_struct_type_param, leaves)) == [:X, :Y, :QR] + @test map(get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] - @test map(get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] + df = DataFrame(get_names_used("test_mods.jl").per_usage_info) + subset!(df, :name => ByRow(==(:QR)), :module_path => ByRow(==([:TestMod5]))) + # two uses of QR: one is as a type param, the other a usage of that type param in the same struct definition + @test df.analysis_code == + [ExplicitImports.InternalStruct, ExplicitImports.IgnoredNonFirst] + @test df.struct_field_or_type_param == [true, false] - # Tests #34 and #36 - @test using_statement.(explicit_imports_nonrecursive(TestMod5, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra"] -end + # Tests #34 and #36 + @test using_statement.(explicit_imports_nonrecursive(TestMod5, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra"] + end -if VERSION >= v"1.7-" - @testset "loops" begin + if VERSION >= v"1.7-" + @testset "loops" begin + cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) + leaves = collect(Leaves(cursor)) + @test map(get_val, filter(is_for_arg, leaves)) == + [:i, :I, :j, :k, :k, :j, :xi, :yi] + + # Tests #35 + @test using_statement.(explicit_imports_nonrecursive(TestMod6, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra"] + end + end + + @testset "nested local scope" begin cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) leaves = collect(Leaves(cursor)) - @test map(get_val, filter(is_for_arg, leaves)) == [:i, :I, :j, :k, :k, :j, :xi, :yi] - - # Tests #35 - @test using_statement.(explicit_imports_nonrecursive(TestMod6, "test_mods.jl")) == + # Test nested local scope + @test using_statement.(explicit_imports_nonrecursive(TestMod7, "test_mods.jl")) == ["using LinearAlgebra: LinearAlgebra"] end -end -@testset "nested local scope" begin - cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) - leaves = collect(Leaves(cursor)) - # Test nested local scope - @test using_statement.(explicit_imports_nonrecursive(TestMod7, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra"] -end + @testset "types without values in function signatures" begin + # https://github.com/ericphanson/ExplicitImports.jl/issues/33 + @test using_statement.(explicit_imports_nonrecursive(TestMod8, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: QR"] + end -@testset "types without values in function signatures" begin - # https://github.com/ericphanson/ExplicitImports.jl/issues/33 - @test using_statement.(explicit_imports_nonrecursive(TestMod8, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: QR"] -end + @testset "generators" begin + cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) + leaves = collect(Leaves(cursor)) -@testset "generators" begin - cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) - leaves = collect(Leaves(cursor)) + v = [:i1, :I, :i2, :I, :i3, :I, :i4, :I] + w = [:i1, :I] + @test map(get_val, filter(is_generator_arg, leaves)) == + [v; v; w; w; w; w; w] - v = [:i1, :I, :i2, :I, :i3, :I, :i4, :I] - w = [:i1, :I] - @test map(get_val, filter(is_generator_arg, leaves)) == - [v; v; w; w; w; w; w] + if VERSION >= v"1.7-" + @test using_statement.(explicit_imports_nonrecursive(TestMod9, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra"] - if VERSION >= v"1.7-" - @test using_statement.(explicit_imports_nonrecursive(TestMod9, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra"] + per_usage_info, _ = analyze_all_names("test_mods.jl") + df = DataFrame(per_usage_info) + subset!(df, :module_path => ByRow(==([:TestMod9])), :name => ByRow(==(:i1))) + @test all(==(ExplicitImports.InternalGenerator), df.analysis_code) + end + end + + @testset "while loops" begin + @test using_statement.(explicit_imports_nonrecursive(TestMod10, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: I"] per_usage_info, _ = analyze_all_names("test_mods.jl") df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod9])), :name => ByRow(==(:i1))) - @test all(==(ExplicitImports.InternalGenerator), df.analysis_code) + subset!(df, :module_path => ByRow(==([:TestMod10])), :name => ByRow(==(:I))) + # First one is internal, second one external + @test df.analysis_code == + [ExplicitImports.InternalAssignment, ExplicitImports.External] end -end -@testset "while loops" begin - @test using_statement.(explicit_imports_nonrecursive(TestMod10, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra", "using LinearAlgebra: I"] - - per_usage_info, _ = analyze_all_names("test_mods.jl") - df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod10])), :name => ByRow(==(:I))) - # First one is internal, second one external - @test df.analysis_code == [ExplicitImports.InternalAssignment, ExplicitImports.External] -end + if VERSION >= v"1.7-" + @testset "do- syntax" begin + @test using_statement.(explicit_imports_nonrecursive(TestMod11, "test_mods.jl")) == + ["using LinearAlgebra: LinearAlgebra", + "using LinearAlgebra: Hermitian", + "using LinearAlgebra: svd"] + + per_usage_info, _ = analyze_all_names("test_mods.jl") + df = DataFrame(per_usage_info) + subset!(df, :module_path => ByRow(==([:TestMod11]))) + + I_codes = subset(df, :name => ByRow(==(:I))).analysis_code + @test I_codes == + [ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, + ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, + ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, + ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst] + svd_codes = subset(df, :name => ByRow(==(:svd))).analysis_code + @test svd_codes == + [ExplicitImports.InternalFunctionArg, ExplicitImports.External] + Hermitian_codes = subset(df, :name => ByRow(==(:Hermitian))).analysis_code + @test Hermitian_codes == + [ExplicitImports.External, ExplicitImports.IgnoredNonFirst] + end + end -if VERSION >= v"1.7-" - @testset "do- syntax" begin - @test using_statement.(explicit_imports_nonrecursive(TestMod11, "test_mods.jl")) == + @testset "try-catch" begin + @test using_statement.(explicit_imports_nonrecursive(TestMod12, "test_mods.jl")) == ["using LinearAlgebra: LinearAlgebra", - "using LinearAlgebra: Hermitian", + "using LinearAlgebra: I", "using LinearAlgebra: svd"] per_usage_info, _ = analyze_all_names("test_mods.jl") df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod11]))) + subset!(df, :module_path => ByRow(==([:TestMod12]))) I_codes = subset(df, :name => ByRow(==(:I))).analysis_code - @test I_codes == - [ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, - ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, - ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst, - ExplicitImports.InternalFunctionArg, ExplicitImports.IgnoredNonFirst] + @test I_codes == [ExplicitImports.InternalAssignment, + ExplicitImports.External, + ExplicitImports.External, + ExplicitImports.InternalAssignment, + ExplicitImports.InternalCatchArgument, + ExplicitImports.IgnoredNonFirst, + ExplicitImports.External] svd_codes = subset(df, :name => ByRow(==(:svd))).analysis_code - @test svd_codes == [ExplicitImports.InternalFunctionArg, ExplicitImports.External] - Hermitian_codes = subset(df, :name => ByRow(==(:Hermitian))).analysis_code - @test Hermitian_codes == [ExplicitImports.External, ExplicitImports.IgnoredNonFirst] + @test svd_codes == [ExplicitImports.InternalAssignment, + ExplicitImports.External, + ExplicitImports.InternalAssignment, + ExplicitImports.External] end -end - -@testset "try-catch" begin - @test using_statement.(explicit_imports_nonrecursive(TestMod12, "test_mods.jl")) == - ["using LinearAlgebra: LinearAlgebra", - "using LinearAlgebra: I", - "using LinearAlgebra: svd"] - - per_usage_info, _ = analyze_all_names("test_mods.jl") - df = DataFrame(per_usage_info) - subset!(df, :module_path => ByRow(==([:TestMod12]))) - - I_codes = subset(df, :name => ByRow(==(:I))).analysis_code - @test I_codes == [ExplicitImports.InternalAssignment, - ExplicitImports.External, - ExplicitImports.External, - ExplicitImports.InternalAssignment, - ExplicitImports.InternalCatchArgument, - ExplicitImports.IgnoredNonFirst, - ExplicitImports.External] - svd_codes = subset(df, :name => ByRow(==(:svd))).analysis_code - @test svd_codes == [ExplicitImports.InternalAssignment, - ExplicitImports.External, - ExplicitImports.InternalAssignment, - ExplicitImports.External] -end - -@testset "scripts" begin - str = sprint(print_explicit_imports_script, "script.jl") - str = replace(str, r"\s+" => " ") - @test contains(str, "Script script.jl") - @test contains(str, "relying on implicit imports for 1 name") - @test contains(str, "using LinearAlgebra: norm") - @test contains(str, "stale explicit imports for this 1 unused name") - @test contains(str, "• qr") -end - -@testset "Handle public symbols with same name as exported Base symbols (#88)" begin - statements = using_statement.(explicit_imports_nonrecursive(Mod88, "examples.jl")) - @test statements == ["using .ModWithTryparse: ModWithTryparse"] -end -@testset "Don't skip source modules (#29)" begin - # In this case `UUID` is defined in Base but exported in UUIDs - ret = ExplicitImports.find_implicit_imports(Mod29)[:UUID] - @test ret.source == Base - @test ret.exporters == [UUIDs] - # We should NOT skip it, even though `skip` includes `Base`, since the exporters - # are not skipped. - statements = using_statement.(explicit_imports_nonrecursive(Mod29, "examples.jl")) - @test statements == ["using UUIDs: UUIDs", "using UUIDs: UUID"] -end - -@testset "Exported module (#24)" begin - statements = using_statement.(explicit_imports_nonrecursive(Mod24, "examples.jl")) - # The key thing here is we do not have `using .Exporter: exported_a`, - # since we haven't done `using .Exporter` in `Mod24`, only `using .Exporter2` - @test statements == ["using .Exporter2: Exporter2", "using .Exporter2: exported_a"] -end - -@testset "string macros (#20)" begin - foo = only_name_source(explicit_imports_nonrecursive(Foo20, "examples.jl")) - @test foo == [(; name=:Markdown, source=Markdown), - (; name=Symbol("@doc_str"), source=Markdown)] - bar = explicit_imports_nonrecursive(Bar20, "examples.jl") - @test isempty(bar) -end - -@testset "TestModArgs" begin - # don't detect `a`! - statements = using_statement.(explicit_imports_nonrecursive(TestModArgs, - "TestModArgs.jl")) - @test statements == - ["using .Exporter4: Exporter4", "using .Exporter4: A", "using .Exporter4: Z"] + @testset "scripts" begin + str = sprint(print_explicit_imports_script, "script.jl") + str = replace(str, r"\s+" => " ") + @test contains(str, "Script script.jl") + @test contains(str, "relying on implicit imports for 1 name") + @test contains(str, "using LinearAlgebra: norm") + @test contains(str, "stale explicit imports for this 1 unused name") + @test contains(str, "• qr") + end - statements = using_statement.(explicit_imports_nonrecursive(ThreadPinning, - "examples.jl")) + @testset "Handle public symbols with same name as exported Base symbols (#88)" begin + statements = using_statement.(explicit_imports_nonrecursive(Mod88, "examples.jl")) + @test statements == ["using .ModWithTryparse: ModWithTryparse"] + end + @testset "Don't skip source modules (#29)" begin + # In this case `UUID` is defined in Base but exported in UUIDs + ret = ExplicitImports.find_implicit_imports(Mod29)[:UUID] + @test ret.source == Base + @test ret.exporters == [UUIDs] + # We should NOT skip it, even though `skip` includes `Base`, since the exporters + # are not skipped. + statements = using_statement.(explicit_imports_nonrecursive(Mod29, "examples.jl")) + @test statements == ["using UUIDs: UUIDs", "using UUIDs: UUID"] + end - @test statements == ["using LinearAlgebra: LinearAlgebra"] -end + @testset "Exported module (#24)" begin + statements = using_statement.(explicit_imports_nonrecursive(Mod24, "examples.jl")) + # The key thing here is we do not have `using .Exporter: exported_a`, + # since we haven't done `using .Exporter` in `Mod24`, only `using .Exporter2` + @test statements == ["using .Exporter2: Exporter2", "using .Exporter2: exported_a"] + end -@testset "is_function_definition_arg" begin - cursor = TreeCursor(SyntaxNodeWrapper("TestModArgs.jl")) - leaves = collect(Leaves(cursor)) - purported_function_args = filter(is_function_definition_arg, leaves) + @testset "string macros (#20)" begin + foo = only_name_source(explicit_imports_nonrecursive(Foo20, "examples.jl")) + @test foo == [(; name=:Markdown, source=Markdown), + (; name=Symbol("@doc_str"), source=Markdown)] + bar = explicit_imports_nonrecursive(Bar20, "examples.jl") + @test isempty(bar) + end - # written this way to get clearer test failure messages - vals = unique(get_val.(purported_function_args)) - @test vals == [:a] + @testset "TestModArgs" begin + # don't detect `a`! + statements = using_statement.(explicit_imports_nonrecursive(TestModArgs, + "TestModArgs.jl")) + @test statements == + ["using .Exporter4: Exporter4", "using .Exporter4: A", "using .Exporter4: Z"] - # we have 9*4 functions with one argument `a`, plus 2 macros - @test length(purported_function_args) == 9 * 4 + 2 - non_function_args = filter(!is_function_definition_arg, leaves) - missed = filter(x -> get_val(x) === :a, non_function_args) - @test isempty(missed) -end + statements = using_statement.(explicit_imports_nonrecursive(ThreadPinning, + "examples.jl")) -@testset "has_ancestor" begin - @test has_ancestor(TestModA.SubModB, TestModA) - @test !has_ancestor(TestModA, TestModA.SubModB) + @test statements == ["using LinearAlgebra: LinearAlgebra"] + end - @test should_skip(Base.Iterators; skip=(Base, Core)) -end + @testset "is_function_definition_arg" begin + cursor = TreeCursor(SyntaxNodeWrapper("TestModArgs.jl")) + leaves = collect(Leaves(cursor)) + purported_function_args = filter(is_function_definition_arg, leaves) -function get_per_scope(per_usage_info) - per_usage_df = DataFrame(per_usage_info) - dropmissing!(per_usage_df, :external_global_name) - return per_usage_df -end + # written this way to get clearer test failure messages + vals = unique(get_val.(purported_function_args)) + @test vals == [:a] -@testset "file not found" begin - for f in (check_no_implicit_imports, check_no_stale_explicit_imports, - check_all_explicit_imports_via_owners, check_all_qualified_accesses_via_owners, - explicit_imports, - explicit_imports_nonrecursive, print_explicit_imports, - improper_explicit_imports, improper_explicit_imports_nonrecursive, - improper_qualified_accesses, improper_qualified_accesses_nonrecursive) - @test_throws FileNotFoundException f(TestModA) + # we have 9*4 functions with one argument `a`, plus 2 macros + @test length(purported_function_args) == 9 * 4 + 2 + non_function_args = filter(!is_function_definition_arg, leaves) + missed = filter(x -> get_val(x) === :a, non_function_args) + @test isempty(missed) end - str = sprint(Base.showerror, FileNotFoundException()) - @test contains(str, "module which is not top-level in a package") -end -@testset "ExplicitImports.jl" begin - @test using_statement.(explicit_imports_nonrecursive(TestModA, "TestModA.jl")) == - ["using .Exporter: Exporter", "using .Exporter: @mac", - "using .Exporter2: Exporter2", - "using .Exporter2: exported_a", "using .Exporter3: Exporter3"] - - per_usage_info, _ = analyze_all_names("TestModA.jl") - df = get_per_scope(per_usage_info) - locals = contains.(string.(df.name), Ref("local")) - @test all(!, df.external_global_name[locals]) - - # we use `x` in two scopes - xs = subset(df, :name => ByRow(==(:x))) - @test !xs[1, :external_global_name] - @test !xs[2, :external_global_name] - @test xs[2, :analysis_code] == ExplicitImports.InternalAssignment - - # we use `exported_a` in two scopes; both times refer to the global name - exported_as = subset(df, :name => ByRow(==(:exported_a))) - @test exported_as[1, :external_global_name] - @test exported_as[2, :external_global_name] - @test !exported_as[2, :is_assignment] - - # Test submodules - @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB, "TestModA.jl")) == - ["using .Exporter3: Exporter3", "using .Exporter3: exported_b", - "using .TestModA: f"] - - mod_path = module_path(TestModA.SubModB) - @test mod_path == [:SubModB, :TestModA, :Main] - sub_df = restrict_to_module(df, TestModA.SubModB) - - h = only(subset(sub_df, :name => ByRow(==(:h)))) - @test h.external_global_name - @test !h.is_assignment - - # Nested submodule with same name as outer module... - @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA, - "TestModA.jl")) == - ["using .Exporter3: Exporter3", "using .Exporter3: exported_b"] - - # Check we are getting innermost names and not outer ones - subsub_df = restrict_to_module(df, TestModA.SubModB.TestModA) - @test :inner_h in subsub_df.name - @test :h ∉ subsub_df.name - # ...we do currently get the outer ones when the module path prefixes collide - @test_broken :f ∉ subsub_df.name - @test_broken :func ∉ subsub_df.name - - # starts from innermost - @test module_path(TestModA.SubModB.TestModA.TestModC) == - [:TestModC, :TestModA, :SubModB, :TestModA, :Main] - - from_outer_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, - "TestModA.jl")) - from_inner_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl")) - @test from_inner_file == from_outer_file - @test "using .TestModA: f" in from_inner_file - # This one isn't needed bc all usages are fully qualified - @test "using .Exporter: exported_a" ∉ from_inner_file - - # This one isn't needed; it is already explicitly imported - @test "using .Exporter: exported_b" ∉ from_inner_file - - # This one shouldn't be there; we never use it, only explicitly import it. - # So actually it should be on a list of unnecessary imports. BUT it can show up - # because by importing it, we have the name in the file, so we used to detect it. - @test "using .Exporter: exported_c" ∉ from_inner_file - - @test from_inner_file == ["using .TestModA: TestModA", "using .TestModA: f"] - - ret = improper_explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl"; - allow_internal_imports=false) - - @test [(; row.name) for row in ret if row.stale] == - [(; name=:exported_c), (; name=:exported_d)] - - # Recursive version - lookup = Dict(improper_explicit_imports(TestModA, - "TestModA.jl"; allow_internal_imports=false)) - ret = lookup[TestModA.SubModB.TestModA.TestModC] - - @test [(; row.name) for row in ret if row.stale] == - [(; name=:exported_c), (; name=:exported_d)] - @test isempty((row for row in lookup[TestModA] if row.stale)) - - per_usage_info, _ = analyze_all_names("TestModC.jl") - testmodc = DataFrame(per_usage_info) - qualified_row = only(subset(testmodc, :name => ByRow(==(:exported_a)))) - @test qualified_row.analysis_code == ExplicitImports.IgnoredQualified - @test qualified_row.qualified_by == [:Exporter] - - qualified_row2 = only(subset(testmodc, :name => ByRow(==(:h)))) - @test qualified_row2.qualified_by == [:TestModA, :SubModB] - - @test using_statement.(explicit_imports_nonrecursive(TestMod1, - "test_mods.jl")) == - ["using ExplicitImports: print_explicit_imports"] - - # Recursion - nested = explicit_imports(TestModA, "TestModA.jl") - @test nested isa Vector{Pair{Module, - Vector{@NamedTuple{name::Symbol,source::Module, - exporters::Vector{Module},location::String}}}} - @test TestModA in first.(nested) - @test TestModA.SubModB in first.(nested) - @test TestModA.SubModB.TestModA in first.(nested) - @test TestModA.SubModB.TestModA.TestModC in first.(nested) - - # Printing - # should be no logs - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - allow_internal_imports=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "Module Main.TestModA is relying on implicit imports") - @test contains(str, "using .Exporter2: Exporter2, exported_a") - @test contains(str, - "However, module Main.TestModA.SubModB.TestModA.TestModC has stale explicit imports for these 2 unused names") - - # should be no logs - # try with linewidth tiny - should put one name per line - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - linewidth=0)) - @test contains(str, "using .Exporter2: Exporter2,\n exported_a") - - # test `show_locations=true` - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - show_locations=true, - allow_internal_imports=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "using .Exporter3: Exporter3 # used at TestModA.jl:") - @test contains(str, "is unused but it was imported from Main.Exporter at TestModC.jl") - - # test `separate_lines=true`` - str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; - separate_lines=true, - allow_internal_imports=false)) - str = replace(str, r"\s+" => " ") - @test contains(str, "using .Exporter3: Exporter3 using .Exporter3: exported_b") - - # `warn_improper_explicit_imports=false` does something (also still no logs) - str_no_warn = @test_logs sprint(io -> print_explicit_imports(io, TestModA, - "TestModA.jl"; - warn_improper_explicit_imports=false)) - str = replace(str, r"\s+" => " ") - @test length(str_no_warn) <= length(str) - - # in particular, this ensures we add `using Foo: Foo` as the first line - @test using_statement.(explicit_imports_nonrecursive(TestMod4, "test_mods.jl")) == - ["using .Exporter4: Exporter4" - "using .Exporter4: A" - "using .Exporter4: Z" - "using .Exporter4: a" - "using .Exporter4: z"] -end + @testset "has_ancestor" begin + @test has_ancestor(TestModA.SubModB, TestModA) + @test !has_ancestor(TestModA, TestModA.SubModB) -@testset "checks" begin - @test check_no_implicit_imports(TestModEmpty, "test_mods.jl") === nothing - @test check_no_stale_explicit_imports(TestModEmpty, "test_mods.jl") === nothing - @test check_no_stale_explicit_imports(TestMod1, "test_mods.jl") === nothing - - @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, - "test_mods.jl") - - # test name ignores - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - ignore=(:print_explicit_imports,)) === nothing - - # test name mod pair ignores - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - ignore=(:print_explicit_imports => ExplicitImports,)) === - nothing - - # if you pass the module in the pair, you must get the right one - @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, - "test_mods.jl"; - ignore=(:print_explicit_imports => TestModA,)) === - nothing - - # non-existent names are OK - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - ignore=(:print_explicit_imports => ExplicitImports, - :does_not_exist)) === nothing - - # you can use skip to skip whole modules - @test check_no_implicit_imports(TestMod1, "test_mods.jl"; - skip=(Base, Core, ExplicitImports)) === nothing - - @test_throws ImplicitImportsException check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl") - - # test submodule ignores - @test check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, "TestModC.jl"; - ignore=(TestModA.SubModB.TestModA.TestModC,)) === - nothing - - @test_throws StaleImportsException check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl") - - # make sure ignored names don't show up in error - e = try - check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl"; - ignore=(:exported_d,)) - @test false # should error before this - catch e - e + @test should_skip(Base.Iterators; skip=(Base, Core)) end - str = sprint(Base.showerror, e) - @test contains(str, "exported_c") - @test !contains(str, "exported_d") - - # ignore works: - @test check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl"; - ignore=(:exported_c, :exported_d)) === - nothing - - # Test the printing is hitting our formatted errors - str = exception_string() do - return check_no_implicit_imports(TestMod1, "test_mods.jl") - end - @test contains(str, "is relying on the following implicit imports") - str = exception_string() do - return check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, - "TestModC.jl") + function get_per_scope(per_usage_info) + per_usage_df = DataFrame(per_usage_info) + dropmissing!(per_usage_df, :external_global_name) + return per_usage_df end - @test contains(str, "has stale (unused) explicit imports for:") - @test check_all_explicit_imports_are_public(TestMod1, "test_mods.jl") === nothing - @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, - "imports.jl") - str = exception_string() do - return check_all_explicit_imports_are_public(ModImports, "imports.jl") - end - @test contains(str, "`_svd!` is not public in `LinearAlgebra` but it was imported") - @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; - ignore=(:_svd!, :exported_b, :f, :h, :map)) === - nothing - - @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; - ignore=(:_svd!, :exported_b, :f, :h), - skip=(LinearAlgebra => Base,)) === - nothing - - @testset "Tainted modules" begin - # 3 dynamic include statements - l = (:warn, r"Dynamic") - log = (l, l, l) - - @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl")) == - [DynMod => nothing, DynMod.Hidden => nothing] - @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl"; - strict=false)) == - [DynMod => [(; name=:print_explicit_imports, - source=ExplicitImports)], - # Wrong! Missing explicit export - DynMod.Hidden => []] - - @test_logs log... @test explicit_imports_nonrecursive(DynMod, "DynMod.jl") === - nothing - - @test_logs log... @test only_name_source(explicit_imports_nonrecursive(DynMod, - "DynMod.jl"; - strict=false)) == - [(; name=:print_explicit_imports, source=ExplicitImports)] - - @test @test_logs log... improper_explicit_imports(DynMod, - "DynMod.jl") == - [DynMod => nothing, - DynMod.Hidden => nothing] - - @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, - "DynMod.jl") === - nothing - - @test_logs log... @test improper_explicit_imports(DynMod, - "DynMod.jl"; - strict=false) == - [DynMod => [], - # Wrong! Missing stale explicit export - DynMod.Hidden => []] - - @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, - "DynMod.jl"; - strict=false) == - [] - - str = @test_logs log... sprint(print_explicit_imports, DynMod, "DynMod.jl") - @test contains(str, "DynMod could not be accurately analyzed") - - @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod, - DynMod.Hidden)) === - nothing - - # Ignore also works - @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod,), - ignore=(DynMod.Hidden,)) === - nothing - - e = UnanalyzableModuleException - @test_logs log... @test_throws e check_no_implicit_imports(DynMod, - "DynMod.jl") - - # Missed `Hidden` - @test_logs log... @test_throws e check_no_implicit_imports(DynMod, - "DynMod.jl"; - allow_unanalyzable=(DynMod,),) - - @test_logs log... @test check_no_stale_explicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod, - DynMod.Hidden)) === - nothing - - @test_logs log... @test_throws e check_no_stale_explicit_imports(DynMod, - "DynMod.jl") - - str = sprint(Base.showerror, UnanalyzableModuleException(DynMod)) - @test contains(str, "was found to be unanalyzable") - - @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod, - DynMod.Hidden)) === - nothing - - @test_logs log... @test_throws e check_no_implicit_imports(DynMod, "DynMod.jl"; - allow_unanalyzable=(DynMod,)) + @testset "file not found" begin + for f in (check_no_implicit_imports, check_no_stale_explicit_imports, + check_all_explicit_imports_via_owners, check_all_qualified_accesses_via_owners, + explicit_imports, + explicit_imports_nonrecursive, print_explicit_imports, + improper_explicit_imports, improper_explicit_imports_nonrecursive, + improper_qualified_accesses, improper_qualified_accesses_nonrecursive) + @test_throws FileNotFoundException f(TestModA) + end + str = sprint(Base.showerror, FileNotFoundException()) + @test contains(str, "module which is not top-level in a package") end -end -@testset "Aqua" begin - Aqua.test_all(ExplicitImports; ambiguities=false) -end + @testset "ExplicitImports.jl" begin + @test using_statement.(explicit_imports_nonrecursive(TestModA, "TestModA.jl")) == + ["using .Exporter: Exporter", "using .Exporter: @mac", + "using .Exporter2: Exporter2", + "using .Exporter2: exported_a", "using .Exporter3: Exporter3"] + + per_usage_info, _ = analyze_all_names("TestModA.jl") + df = get_per_scope(per_usage_info) + locals = contains.(string.(df.name), Ref("local")) + @test all(!, df.external_global_name[locals]) + + # we use `x` in two scopes + xs = subset(df, :name => ByRow(==(:x))) + @test !xs[1, :external_global_name] + @test !xs[2, :external_global_name] + @test xs[2, :analysis_code] == ExplicitImports.InternalAssignment + + # we use `exported_a` in two scopes; both times refer to the global name + exported_as = subset(df, :name => ByRow(==(:exported_a))) + @test exported_as[1, :external_global_name] + @test exported_as[2, :external_global_name] + @test !exported_as[2, :is_assignment] + + # Test submodules + @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB, + "TestModA.jl")) == + ["using .Exporter3: Exporter3", "using .Exporter3: exported_b", + "using .TestModA: f"] + + mod_path = module_path(TestModA.SubModB) + @test mod_path == [:SubModB, :TestModA, :Main] + sub_df = restrict_to_module(df, TestModA.SubModB) + + h = only(subset(sub_df, :name => ByRow(==(:h)))) + @test h.external_global_name + @test !h.is_assignment + + # Nested submodule with same name as outer module... + @test using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA, + "TestModA.jl")) == + ["using .Exporter3: Exporter3", "using .Exporter3: exported_b"] + + # Check we are getting innermost names and not outer ones + subsub_df = restrict_to_module(df, TestModA.SubModB.TestModA) + @test :inner_h in subsub_df.name + @test :h ∉ subsub_df.name + # ...we do currently get the outer ones when the module path prefixes collide + @test_broken :f ∉ subsub_df.name + @test_broken :func ∉ subsub_df.name + + # starts from innermost + @test module_path(TestModA.SubModB.TestModA.TestModC) == + [:TestModC, :TestModA, :SubModB, :TestModA, :Main] + + from_outer_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, + "TestModA.jl")) + from_inner_file = using_statement.(explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl")) + @test from_inner_file == from_outer_file + @test "using .TestModA: f" in from_inner_file + # This one isn't needed bc all usages are fully qualified + @test "using .Exporter: exported_a" ∉ from_inner_file + + # This one isn't needed; it is already explicitly imported + @test "using .Exporter: exported_b" ∉ from_inner_file + + # This one shouldn't be there; we never use it, only explicitly import it. + # So actually it should be on a list of unnecessary imports. BUT it can show up + # because by importing it, we have the name in the file, so we used to detect it. + @test "using .Exporter: exported_c" ∉ from_inner_file + + @test from_inner_file == ["using .TestModA: TestModA", "using .TestModA: f"] + + ret = improper_explicit_imports_nonrecursive(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl"; + allow_internal_imports=false) -@testset "`inspect_session`" begin - # We just want to make sure we are robust enough that this doesn't error - big_str = with_logger(Logging.NullLogger()) do - return sprint(inspect_session) - end -end + @test [(; row.name) for row in ret if row.stale] == + [(; name=:exported_c), (; name=:exported_d)] + + # Recursive version + lookup = Dict(improper_explicit_imports(TestModA, + "TestModA.jl"; + allow_internal_imports=false)) + ret = lookup[TestModA.SubModB.TestModA.TestModC] + + @test [(; row.name) for row in ret if row.stale] == + [(; name=:exported_c), (; name=:exported_d)] + @test isempty((row for row in lookup[TestModA] if row.stale)) + + per_usage_info, _ = analyze_all_names("TestModC.jl") + testmodc = DataFrame(per_usage_info) + qualified_row = only(subset(testmodc, :name => ByRow(==(:exported_a)))) + @test qualified_row.analysis_code == ExplicitImports.IgnoredQualified + @test qualified_row.qualified_by == [:Exporter] + + qualified_row2 = only(subset(testmodc, :name => ByRow(==(:h)))) + @test qualified_row2.qualified_by == [:TestModA, :SubModB] + + @test using_statement.(explicit_imports_nonrecursive(TestMod1, + "test_mods.jl")) == + ["using ExplicitImports: print_explicit_imports"] + + # Recursion + nested = explicit_imports(TestModA, "TestModA.jl") + @test nested isa Vector{Pair{Module, + Vector{@NamedTuple{name::Symbol,source::Module, + exporters::Vector{Module},location::String}}}} + @test TestModA in first.(nested) + @test TestModA.SubModB in first.(nested) + @test TestModA.SubModB.TestModA in first.(nested) + @test TestModA.SubModB.TestModA.TestModC in first.(nested) + + # Printing + # should be no logs + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + allow_internal_imports=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "Module Main.TestModA is relying on implicit imports") + @test contains(str, "using .Exporter2: Exporter2, exported_a") + @test contains(str, + "However, module Main.TestModA.SubModB.TestModA.TestModC has stale explicit imports for these 2 unused names") + + # should be no logs + # try with linewidth tiny - should put one name per line + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + linewidth=0)) + @test contains(str, "using .Exporter2: Exporter2,\n exported_a") + + # test `show_locations=true` + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + show_locations=true, + allow_internal_imports=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "using .Exporter3: Exporter3 # used at TestModA.jl:") + @test contains(str, + "is unused but it was imported from Main.Exporter at TestModC.jl") + + # test `separate_lines=true`` + str = @test_logs sprint(io -> print_explicit_imports(io, TestModA, "TestModA.jl"; + separate_lines=true, + allow_internal_imports=false)) + str = replace(str, r"\s+" => " ") + @test contains(str, "using .Exporter3: Exporter3 using .Exporter3: exported_b") -@testset "backtick modules and locations" begin - @testset "print_explicit_imports" begin - # Test that module names and file:line locations are surrounded by backticks - # and that underscores in module and file names are printed and do not cause italics. - str = sprint() do io - print_explicit_imports(io, Test_Mod_Underscores, "Test_Mod_Underscores.jl"; report_non_public=true) - end + # `warn_improper_explicit_imports=false` does something (also still no logs) + str_no_warn = @test_logs sprint(io -> print_explicit_imports(io, TestModA, + "TestModA.jl"; + warn_improper_explicit_imports=false)) str = replace(str, r"\s+" => " ") - # stale import - @test contains(str, "Test_Mod_Underscores has stale explicit imports") - @test contains(str, "svd is unused but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") - # non-owner module - @test contains(str, "Test_Mod_Underscores explicitly imports 1 name from non-owner module") - @test contains(str, "map has owner Base but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") - # non-public name - @test contains(str, "Test_Mod_Underscores explicitly imports 1 non-public name") - @test contains(str, "_svd! is not public in LinearAlgebra but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") - # self-qualified access - @test contains(str, "Test_Mod_Underscores has 1 self-qualified access") - @test contains(str, "foo was accessed as Main.Test_Mod_Underscores.foo inside Main.TestModUnderscores at Test_Mod_Underscores.jl") - # access non-owner module - @test contains(str, "Test_Mod_Underscores accesses 1 name from non-owner modules") - @test contains(str, "Number has owner Base but it was accessed from Base.Sys at Test_Mod_Underscores.jl") - # access non-public name - @test contains(str, "Test_Mod_Underscores accesses 1 non-public name") - @test contains(str, "__unsafe_string! is not public in Base but it was accessed via Base at Test_Mod_Underscores.jl") + @test length(str_no_warn) <= length(str) + + # in particular, this ensures we add `using Foo: Foo` as the first line + @test using_statement.(explicit_imports_nonrecursive(TestMod4, "test_mods.jl")) == + ["using .Exporter4: Exporter4" + "using .Exporter4: A" + "using .Exporter4: Z" + "using .Exporter4: a" + "using .Exporter4: z"] end - @testset "check_*" begin - str = exception_string() do - check_no_implicit_imports(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + + @testset "checks" begin + @test check_no_implicit_imports(TestModEmpty, "test_mods.jl") === nothing + @test check_no_stale_explicit_imports(TestModEmpty, "test_mods.jl") === nothing + @test check_no_stale_explicit_imports(TestMod1, "test_mods.jl") === nothing + + @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, + "test_mods.jl") + + # test name ignores + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + ignore=(:print_explicit_imports,)) === nothing + + # test name mod pair ignores + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + ignore=(:print_explicit_imports => ExplicitImports,)) === + nothing + + # if you pass the module in the pair, you must get the right one + @test_throws ImplicitImportsException check_no_implicit_imports(TestMod1, + "test_mods.jl"; + ignore=(:print_explicit_imports => TestModA,)) === + nothing + + # non-existent names are OK + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + ignore=(:print_explicit_imports => ExplicitImports, + :does_not_exist)) === nothing + + # you can use skip to skip whole modules + @test check_no_implicit_imports(TestMod1, "test_mods.jl"; + skip=(Base, Core, ExplicitImports)) === nothing + + @test_throws ImplicitImportsException check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl") + + # test submodule ignores + @test check_no_implicit_imports(TestModA.SubModB.TestModA.TestModC, "TestModC.jl"; + ignore=(TestModA.SubModB.TestModA.TestModC,)) === + nothing + + @test_throws StaleImportsException check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl") + + # make sure ignored names don't show up in error + e = try + check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl"; + ignore=(:exported_d,)) + @test false # should error before this + catch e + e end - @test contains(str, "`Main.Test_Mod_Underscores` is relying on") + str = sprint(Base.showerror, e) + @test contains(str, "exported_c") + @test !contains(str, "exported_d") + + # ignore works: + @test check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl"; + ignore=(:exported_c, :exported_d)) === + nothing + + # Test the printing is hitting our formatted errors str = exception_string() do - check_all_explicit_imports_via_owners(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + return check_no_implicit_imports(TestMod1, "test_mods.jl") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has explicit imports") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + @test contains(str, "is relying on the following implicit imports") + str = exception_string() do - check_all_explicit_imports_are_public(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + return check_no_stale_explicit_imports(TestModA.SubModB.TestModA.TestModC, + "TestModC.jl") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + @test contains(str, "has stale (unused) explicit imports for:") + + @test check_all_explicit_imports_are_public(TestMod1, "test_mods.jl") === nothing + @test_throws NonPublicExplicitImportsException check_all_explicit_imports_are_public(ModImports, + "imports.jl") str = exception_string() do - check_no_stale_explicit_imports(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + return check_all_explicit_imports_are_public(ModImports, "imports.jl") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has stale") - str = exception_string() do - check_all_qualified_accesses_via_owners(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + @test contains(str, "`_svd!` is not public in `LinearAlgebra` but it was imported") + @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; + ignore=(:_svd!, :exported_b, :f, :h, + :map)) === + nothing + + @test check_all_explicit_imports_are_public(ModImports, "imports.jl"; + ignore=(:_svd!, :exported_b, :f, :h), + skip=(LinearAlgebra => Base,)) === + nothing + + @testset "Tainted modules" begin + # 3 dynamic include statements + l = (:warn, r"Dynamic") + log = (l, l, l) + + @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl")) == + [DynMod => nothing, DynMod.Hidden => nothing] + @test_logs log... @test only_name_source(explicit_imports(DynMod, "DynMod.jl"; + strict=false)) == + [DynMod => [(; name=:print_explicit_imports, + source=ExplicitImports)], + # Wrong! Missing explicit export + DynMod.Hidden => []] + + @test_logs log... @test explicit_imports_nonrecursive(DynMod, "DynMod.jl") === + nothing + + @test_logs log... @test only_name_source(explicit_imports_nonrecursive(DynMod, + "DynMod.jl"; + strict=false)) == + [(; name=:print_explicit_imports, + source=ExplicitImports)] + + @test @test_logs log... improper_explicit_imports(DynMod, + "DynMod.jl") == + [DynMod => nothing, + DynMod.Hidden => nothing] + + @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, + "DynMod.jl") === + nothing + + @test_logs log... @test improper_explicit_imports(DynMod, + "DynMod.jl"; + strict=false) == + [DynMod => [], + # Wrong! Missing stale explicit export + DynMod.Hidden => []] + + @test_logs log... @test improper_explicit_imports_nonrecursive(DynMod, + "DynMod.jl"; + strict=false) == + [] + + str = @test_logs log... sprint(print_explicit_imports, DynMod, "DynMod.jl") + @test contains(str, "DynMod could not be accurately analyzed") + + @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod, + DynMod.Hidden)) === + nothing + + # Ignore also works + @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod,), + ignore=(DynMod.Hidden,)) === + nothing + + e = UnanalyzableModuleException + @test_logs log... @test_throws e check_no_implicit_imports(DynMod, + "DynMod.jl") + + # Missed `Hidden` + @test_logs log... @test_throws e check_no_implicit_imports(DynMod, + "DynMod.jl"; + allow_unanalyzable=(DynMod,),) + + @test_logs log... @test check_no_stale_explicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod, + DynMod.Hidden)) === + nothing + + @test_logs log... @test_throws e check_no_stale_explicit_imports(DynMod, + "DynMod.jl") + + str = sprint(Base.showerror, UnanalyzableModuleException(DynMod)) + @test contains(str, "was found to be unanalyzable") + + @test_logs log... @test check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod, + DynMod.Hidden)) === + nothing + + @test_logs log... @test_throws e check_no_implicit_imports(DynMod, "DynMod.jl"; + allow_unanalyzable=(DynMod,)) end - @test contains(str, "Module `Main.Test_Mod_Underscores` has qualified accesses") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") - str = exception_string() do - check_all_qualified_accesses_are_public(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + end + + # TODO reenable this + # @testset "Aqua" begin + # Aqua.test_all(ExplicitImports; ambiguities=false) + # end + + @testset "`inspect_session`" begin + # We just want to make sure we are robust enough that this doesn't error + big_str = with_logger(Logging.NullLogger()) do + return sprint(inspect_session) end - @test contains(str, "Module `Main.Test_Mod_Underscores` has explicit imports of") - @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") - str = exception_string() do - check_no_self_qualified_accesses(Test_Mod_Underscores, "Test_Mod_Underscores.jl") + end + + @testset "backtick modules and locations" begin + @testset "print_explicit_imports" begin + # Test that module names and file:line locations are surrounded by backticks + # and that underscores in module and file names are printed and do not cause italics. + str = sprint() do io + return print_explicit_imports(io, Test_Mod_Underscores, + "Test_Mod_Underscores.jl"; + report_non_public=true) + end + str = replace(str, r"\s+" => " ") + # stale import + @test contains(str, "Test_Mod_Underscores has stale explicit imports") + @test contains(str, + "svd is unused but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") + # non-owner module + @test contains(str, + "Test_Mod_Underscores explicitly imports 1 name from non-owner module") + @test contains(str, + "map has owner Base but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") + # non-public name + @test contains(str, "Test_Mod_Underscores explicitly imports 1 non-public name") + @test contains(str, + "_svd! is not public in LinearAlgebra but it was imported from LinearAlgebra at Test_Mod_Underscores.jl") + # self-qualified access + @test contains(str, "Test_Mod_Underscores has 1 self-qualified access") + @test contains(str, + "foo was accessed as Main.Test_Mod_Underscores.foo inside Main.TestModUnderscores at Test_Mod_Underscores.jl") + # access non-owner module + @test contains(str, + "Test_Mod_Underscores accesses 1 name from non-owner modules") + @test contains(str, + "Number has owner Base but it was accessed from Base.Sys at Test_Mod_Underscores.jl") + # access non-public name + @test contains(str, "Test_Mod_Underscores accesses 1 non-public name") + @test contains(str, + "__unsafe_string! is not public in Base but it was accessed via Base at Test_Mod_Underscores.jl") + end + @testset "check_*" begin + str = exception_string() do + return check_no_implicit_imports(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "`Main.Test_Mod_Underscores` is relying on") + str = exception_string() do + return check_all_explicit_imports_via_owners(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has explicit imports") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_all_explicit_imports_are_public(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_no_stale_explicit_imports(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has stale") + str = exception_string() do + return check_all_qualified_accesses_via_owners(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, "Module `Main.Test_Mod_Underscores` has qualified accesses") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_all_qualified_accesses_are_public(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, + "Module `Main.Test_Mod_Underscores` has explicit imports of") + @test contains(str, r"at `Test_Mod_Underscores.jl:\d+:\d+`") + str = exception_string() do + return check_no_self_qualified_accesses(Test_Mod_Underscores, + "Test_Mod_Underscores.jl") + end + @test contains(str, + "Module `Main.Test_Mod_Underscores` has self-qualified accesses") + @test contains(str, + r"accessed as `Main.Test_Mod_Underscores.foo` inside `Main.Test_Mod_Underscores` at `Test_Mod_Underscores.jl:10:40`") end - @test contains(str, "Module `Main.Test_Mod_Underscores` has self-qualified accesses") - @test contains(str, r"accessed as `Main.Test_Mod_Underscores.foo` inside `Main.Test_Mod_Underscores` at `Test_Mod_Underscores.jl:10:40`") end end From ebdf549be740bacddf3535fd47ecf7ef7fef03a8 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 22:44:31 +0200 Subject: [PATCH 18/35] re-enable aqua --- test/runtests.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1b6315fe..d1bbf65a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1062,12 +1062,6 @@ include("module_alias.jl") allow_unanalyzable=(DynMod,)) end end - - # TODO reenable this - # @testset "Aqua" begin - # Aqua.test_all(ExplicitImports; ambiguities=false) - # end - @testset "`inspect_session`" begin # We just want to make sure we are robust enough that this doesn't error big_str = with_logger(Logging.NullLogger()) do @@ -1158,4 +1152,9 @@ include("module_alias.jl") r"accessed as `Main.Test_Mod_Underscores.foo` inside `Main.Test_Mod_Underscores` at `Test_Mod_Underscores.jl:10:40`") end end + + + @testset "Aqua" begin + Aqua.test_all(ExplicitImports; ambiguities=false) + end end From 42eec55e6dcba56de4dca3ad542e095fcb33cfd4 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 22:46:24 +0200 Subject: [PATCH 19/35] support 1.12-beta --- src/find_implicit_imports.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/find_implicit_imports.jl b/src/find_implicit_imports.jl index 490f78e7..97696697 100644 --- a/src/find_implicit_imports.jl +++ b/src/find_implicit_imports.jl @@ -45,6 +45,7 @@ function find_implicit_imports(mod::Module; skip=(mod, Base, Core)) # `WARNING: both Exporter3 and Exporter2 export "exported_a"; uses of it in module TestModA must be qualified` # and there is an ambiguity, and the name is in fact not resolved in `mod` clash = (err == ErrorException("\"$name\" is not defined in module $mod"))::Bool + clash |= (err == ErrorException("Constant binding was imported from multiple modules"))::Bool # if it is something else, rethrow clash || rethrow() missing From 326f419c843c74f43c0572c6c1de7ad145e16cea Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Fri, 13 Jun 2025 22:56:36 +0200 Subject: [PATCH 20/35] format --- test/runtests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index d1bbf65a..7b49a8e5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1153,7 +1153,6 @@ include("module_alias.jl") end end - @testset "Aqua" begin Aqua.test_all(ExplicitImports; ambiguities=false) end From 9a81195fad4f140436872e906bca4dc8d9500429 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 21 Jul 2025 21:57:20 +0200 Subject: [PATCH 21/35] set up code to vendor JuliaLowering --- src/ExplicitImports.jl | 1 + vendor/run.jl | 42 +++++++++++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 0ba11f2c..2b58477a 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -7,6 +7,7 @@ module ExplicitImports module Vendored include(joinpath("vendored", "JuliaSyntax", "src", "JuliaSyntax.jl")) include(joinpath("vendored", "AbstractTrees", "src", "AbstractTrees.jl")) +include(joinpath("vendored", "JuliaLowering", "src", "JuliaLowering.jl")) end #! explicit-imports: on diff --git a/vendor/run.jl b/vendor/run.jl index 32c1fbd7..99b3311a 100644 --- a/vendor/run.jl +++ b/vendor/run.jl @@ -1,17 +1,37 @@ -using PackageAnalyzer +using PackageAnalyzer, UUIDs -deps = [("JuliaSyntax", v"1.0.2"), - ("AbstractTrees", v"0.4.5")] +deps = [find_package("JuliaSyntax"; version=v"1.0.2"), + find_package("AbstractTrees"; version=v"0.4.5"), + PackageAnalyzer.Added(; name="JuliaLowering", + uuid=UUID("f3c80556-a63f-4383-b822-37d64f81a311"), + path="", + repo_url="https://github.com/mlechu/JuliaLowering.jl", + tree_hash="2d3dfe83e9be4318c056ed9df2d3788f5723bb9d", + subdir="")] -for (name, version) in deps - pkg = find_package(name; version) - local_path, reachable, _ = PackageAnalyzer.obtain_code(pkg) +for pkg in deps + code_dir, reachable, _ = PackageAnalyzer.obtain_code(pkg) + name, _ = PackageAnalyzer.parse_project(code_dir) @assert reachable - p = mkpath(joinpath(@__DIR__, "..", "src", "vendored", name)) + vendor_dir = mkpath(joinpath(@__DIR__, "..", "src", "vendored", name)) # remove any existing files - if isdir(p) - rm(p; recursive=true, force=true) + if isdir(vendor_dir) + rm(vendor_dir; recursive=true, force=true) + end + mkpath(joinpath(vendor_dir, "src")) + cp(joinpath(code_dir, "src"), joinpath(vendor_dir, "src"); force=true) + + # patch `using JuliaSyntax` => `using ..JuliaSyntax` + for (root, dirs, files) in walkdir(joinpath(vendor_dir, "src")) + for file in files + endswith(file, ".jl") || continue + contents = replace(read(joinpath(root, file), String), + "using JuliaSyntax" => "using ..JuliaSyntax", + # remove unnecessary `using JuliaLowering` from src/hooks.jl + "using JuliaLowering" => "") + chmod(joinpath(root, file), 0o666) # make writable + write(abspath(joinpath(root, file)), contents) + chmod(joinpath(root, file), 0o444) # back to read-only + end end - mkpath(joinpath(p, "src")) - cp(joinpath(local_path, "src"), joinpath(p, "src"); force=true) end From 6ca7cc90f28bae28225d5764f7cd4f4612568ac4 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 21 Jul 2025 21:57:34 +0200 Subject: [PATCH 22/35] commit JuliaLowering --- .../JuliaLowering/src/JuliaLowering.jl | 42 + src/vendored/JuliaLowering/src/ast.jl | 705 +++ src/vendored/JuliaLowering/src/bindings.jl | 212 + .../JuliaLowering/src/closure_conversion.jl | 589 +++ src/vendored/JuliaLowering/src/desugaring.jl | 4552 +++++++++++++++++ src/vendored/JuliaLowering/src/eval.jl | 372 ++ src/vendored/JuliaLowering/src/hooks.jl | 153 + src/vendored/JuliaLowering/src/kinds.jl | 154 + src/vendored/JuliaLowering/src/linear_ir.jl | 1161 +++++ .../JuliaLowering/src/macro_expansion.jl | 340 ++ src/vendored/JuliaLowering/src/runtime.jl | 416 ++ .../JuliaLowering/src/scope_analysis.jl | 780 +++ .../JuliaLowering/src/syntax_graph.jl | 747 +++ .../JuliaLowering/src/syntax_macros.jl | 223 + src/vendored/JuliaLowering/src/utils.jl | 168 + 15 files changed, 10614 insertions(+) create mode 100644 src/vendored/JuliaLowering/src/JuliaLowering.jl create mode 100644 src/vendored/JuliaLowering/src/ast.jl create mode 100644 src/vendored/JuliaLowering/src/bindings.jl create mode 100644 src/vendored/JuliaLowering/src/closure_conversion.jl create mode 100644 src/vendored/JuliaLowering/src/desugaring.jl create mode 100644 src/vendored/JuliaLowering/src/eval.jl create mode 100644 src/vendored/JuliaLowering/src/hooks.jl create mode 100644 src/vendored/JuliaLowering/src/kinds.jl create mode 100644 src/vendored/JuliaLowering/src/linear_ir.jl create mode 100644 src/vendored/JuliaLowering/src/macro_expansion.jl create mode 100644 src/vendored/JuliaLowering/src/runtime.jl create mode 100644 src/vendored/JuliaLowering/src/scope_analysis.jl create mode 100644 src/vendored/JuliaLowering/src/syntax_graph.jl create mode 100644 src/vendored/JuliaLowering/src/syntax_macros.jl create mode 100644 src/vendored/JuliaLowering/src/utils.jl diff --git a/src/vendored/JuliaLowering/src/JuliaLowering.jl b/src/vendored/JuliaLowering/src/JuliaLowering.jl new file mode 100644 index 00000000..fba6ec19 --- /dev/null +++ b/src/vendored/JuliaLowering/src/JuliaLowering.jl @@ -0,0 +1,42 @@ +baremodule JuliaLowering + +# ^ Use baremodule because we're implementing `Base.include` and `Core.eval`. +using Base +# We define a separate _include() for use in this module to avoid mixing method +# tables with the public `JuliaLowering.include()` API +_include(path::AbstractString) = Base.include(JuliaLowering, path) +using Core: eval + +using ..JuliaSyntax + +using ..JuliaSyntax: highlight, Kind, @KSet_str +using ..JuliaSyntax: is_leaf, children, numchildren, head, kind, flags, has_flags, numeric_flags +using ..JuliaSyntax: filename, first_byte, last_byte, byte_range, sourcefile, source_location, span, sourcetext + +using ..JuliaSyntax: is_literal, is_number, is_operator, is_prec_assignment, is_prefix_call, is_infix_op_call, is_postfix_op_call, is_error, is_dotted + +_include("kinds.jl") +_register_kinds() + +_include("syntax_graph.jl") +_include("ast.jl") +_include("bindings.jl") +_include("utils.jl") + +_include("macro_expansion.jl") +_include("desugaring.jl") +_include("scope_analysis.jl") +_include("closure_conversion.jl") +_include("linear_ir.jl") +_include("runtime.jl") +_include("syntax_macros.jl") + +_include("eval.jl") + +_include("hooks.jl") + +function __init__() + _register_kinds() +end + +end diff --git a/src/vendored/JuliaLowering/src/ast.jl b/src/vendored/JuliaLowering/src/ast.jl new file mode 100644 index 00000000..7188eba4 --- /dev/null +++ b/src/vendored/JuliaLowering/src/ast.jl @@ -0,0 +1,705 @@ +#------------------------------------------------------------------------------- +# @chk: Basic AST structure checking tool +# +# Check a condition involving an expression, throwing a LoweringError if it +# doesn't evaluate to true. Does some very simple pattern matching to attempt +# to extract the expression variable from the left hand side. +# +# Forms: +# @chk pred(ex) +# @chk pred(ex) msg +# @chk pred(ex) (msg_display_ex, msg) +macro chk(cond, msg=nothing) + if Meta.isexpr(msg, :tuple) + ex = msg.args[1] + msg = msg.args[2] + else + ex = cond + while true + if ex isa Symbol + break + elseif ex.head == :call + ex = ex.args[2] + elseif ex.head == :ref + ex = ex.args[1] + elseif ex.head == :. + ex = ex.args[1] + elseif ex.head in (:(==), :(in), :<, :>) + ex = ex.args[1] + else + error("Can't analyze $cond") + end + end + end + quote + ex = $(esc(ex)) + @assert ex isa SyntaxTree + ok = try + $(esc(cond)) + catch + false + end + if !ok + throw(LoweringError(ex, $(isnothing(msg) ? "expected `$cond`" : esc(msg)))) + end + end +end + +#------------------------------------------------------------------------------- +abstract type AbstractLoweringContext end + +""" +Bindings for the current lambda being processed. + +Lowering passes prior to scope resolution return `nothing` and bindings are +collected later. +""" +current_lambda_bindings(ctx::AbstractLoweringContext) = nothing + +function syntax_graph(ctx::AbstractLoweringContext) + ctx.graph +end + +""" +Unique symbolic identity for a variable, constant, label, or other entity +""" +const IdTag = Int + +""" +Id for scope layers in macro expansion +""" +const LayerId = Int + +#------------------------------------------------------------------------------- +# AST creation utilities +_node_id(graph::SyntaxGraph, ex::SyntaxTree) = (check_compatible_graph(graph, ex); ex._id) +function _node_id(graph::SyntaxGraph, ex) + # Fallback to give a comprehensible error message for use with the @ast macro + error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`") +end +function _node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree}) + # Fallback to give a comprehensible error message for use with the @ast macro + error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)") +end + +_node_ids(graph::SyntaxGraph) = () +_node_ids(graph::SyntaxGraph, ::Nothing, cs...) = _node_ids(graph, cs...) +_node_ids(graph::SyntaxGraph, c, cs...) = (_node_id(graph, c), _node_ids(graph, cs...)...) +_node_ids(graph::SyntaxGraph, cs::SyntaxList, cs1...) = (_node_ids(graph, cs...)..., _node_ids(graph, cs1...)...) +function _node_ids(graph::SyntaxGraph, cs::SyntaxList) + check_compatible_graph(graph, cs) + cs.ids +end + +_unpack_srcref(graph, srcref::SyntaxTree) = _node_id(graph, srcref) +_unpack_srcref(graph, srcref::Tuple) = _node_ids(graph, srcref...) +_unpack_srcref(graph, srcref) = srcref + +function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val) + push!(ids, _node_id(graph, val)) +end +function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val::Nothing) + nothing +end +function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals) + for v in vals + _push_nodeid!(graph, ids, v) + end +end +function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals::SyntaxList) + check_compatible_graph(graph, vals) + append!(ids, vals.ids) +end + +function makeleaf(graph::SyntaxGraph, srcref, proto; attrs...) + id = newnode!(graph) + ex = SyntaxTree(graph, id) + copy_attrs!(ex, proto, true) + setattr!(graph, id; source=_unpack_srcref(graph, srcref), attrs...) + return ex +end + +function _makenode(graph::SyntaxGraph, srcref, proto, children; attrs...) + id = newnode!(graph) + setchildren!(graph, id, children) + ex = SyntaxTree(graph, id) + copy_attrs!(ex, proto, true) + setattr!(graph, id; source=_unpack_srcref(graph, srcref), attrs...) + return SyntaxTree(graph, id) +end +function _makenode(ctx, srcref, proto, children; attrs...) + _makenode(syntax_graph(ctx), srcref, proto, children; attrs...) +end + +function makenode(ctx, srcref, proto, children...; attrs...) + _makenode(ctx, srcref, proto, _node_ids(syntax_graph(ctx), children...); attrs...) +end + +function makeleaf(ctx, srcref, proto; kws...) + makeleaf(syntax_graph(ctx), srcref, proto; kws...) +end + +function makeleaf(ctx, srcref, k::Kind, value; kws...) + graph = syntax_graph(ctx) + if k == K"Identifier" || k == K"core" || k == K"top" || k == K"Symbol" || + k == K"globalref" || k == K"Placeholder" + makeleaf(graph, srcref, k; name_val=value, kws...) + elseif k == K"BindingId" + makeleaf(graph, srcref, k; var_id=value, kws...) + elseif k == K"label" + makeleaf(graph, srcref, k; id=value, kws...) + elseif k == K"symbolic_label" + makeleaf(graph, srcref, k; name_val=value, kws...) + elseif k == K"TOMBSTONE" || k == K"SourceLocation" + makeleaf(graph, srcref, k; kws...) + else + val = k == K"Integer" ? convert(Int, value) : + k == K"Float" ? convert(Float64, value) : + k == K"String" ? convert(String, value) : + k == K"Char" ? convert(Char, value) : + k == K"Value" ? value : + k == K"Bool" ? value : + error("Unexpected leaf kind `$k`") + makeleaf(graph, srcref, k; value=val, kws...) + end +end + +# TODO: Replace this with makeleaf variant? +function mapleaf(ctx, src, kind) + ex = makeleaf(syntax_graph(ctx), src, kind) + # TODO: Value coersion might be broken here due to use of `name_val` vs + # `value` vs ... ? + copy_attrs!(ex, src) + ex +end + +# Convenience functions to create leaf nodes referring to identifiers within +# the Core and Top modules. +core_ref(ctx, ex, name) = makeleaf(ctx, ex, K"core", name) +svec_type(ctx, ex) = core_ref(ctx, ex, "svec") +nothing_(ctx, ex) = core_ref(ctx, ex, "nothing") + +top_ref(ctx, ex, name) = makeleaf(ctx, ex, K"top", name) + +# Assign `ex` to an SSA variable. +# Return (variable, assignment_node) +function assign_tmp(ctx::AbstractLoweringContext, ex, name="tmp") + var = ssavar(ctx, ex, name) + assign_var = makenode(ctx, ex, K"=", var, ex) + var, assign_var +end + +function emit_assign_tmp(stmts::SyntaxList, ctx, ex, name="tmp") + if is_ssa(ctx, ex) + return ex + end + var = ssavar(ctx, ex, name) + push!(stmts, makenode(ctx, ex, K"=", var, ex)) + var +end + +#------------------------------------------------------------------------------- +# @ast macro +function _match_srcref(ex) + if Meta.isexpr(ex, :macrocall) && ex.args[1] == Symbol("@HERE") + QuoteNode(ex.args[2]) + else + esc(ex) + end +end + +function _match_kind(f::Function, srcref, ex) + kws = [] + if Meta.isexpr(ex, :call) + kind = esc(ex.args[1]) + args = ex.args[2:end] + if Meta.isexpr(args[1], :parameters) + kws = map(esc, args[1].args) + popfirst!(args) + end + while length(args) >= 1 && Meta.isexpr(args[end], :kw) + pushfirst!(kws, esc(pop!(args))) + end + if length(args) == 1 + srcref_tmp = gensym("srcref") + return quote + $srcref_tmp = $(_match_srcref(args[1])) + $(f(kind, srcref_tmp, kws)) + end + elseif length(args) > 1 + error("Unexpected: extra srcref argument in `$ex`?") + end + else + kind = esc(ex) + end + f(kind, srcref, kws) +end + +function _expand_ast_tree(ctx, srcref, tree) + if Meta.isexpr(tree, :(::)) + # Leaf node + if length(tree.args) == 2 + val = esc(tree.args[1]) + kindspec = tree.args[2] + else + val = nothing + kindspec = tree.args[1] + end + _match_kind(srcref, kindspec) do kind, srcref, kws + :(makeleaf($ctx, $srcref, $kind, $(val), $(kws...))) + end + elseif Meta.isexpr(tree, :call) && tree.args[1] === :(=>) + # Leaf node with copied attributes + kind = esc(tree.args[3]) + srcref = esc(tree.args[2]) + :(mapleaf($ctx, $srcref, $kind)) + elseif Meta.isexpr(tree, (:vcat, :hcat, :vect)) + # Interior node + flatargs = [] + for a in tree.args + if Meta.isexpr(a, :row) + append!(flatargs, a.args) + else + push!(flatargs, a) + end + end + children_ex = :(let child_ids = Vector{NodeId}(), graph = syntax_graph($ctx) + end) + child_stmts = children_ex.args[2].args + for a in flatargs[2:end] + child = _expand_ast_tree(ctx, srcref, a) + if Meta.isexpr(child, :(...)) + push!(child_stmts, :(_append_nodeids!(graph, child_ids, $(child.args[1])))) + else + push!(child_stmts, :(_push_nodeid!(graph, child_ids, $child))) + end + end + push!(child_stmts, :(child_ids)) + _match_kind(srcref, flatargs[1]) do kind, srcref, kws + :(_makenode($ctx, $srcref, $kind, $children_ex; $(kws...))) + end + elseif Meta.isexpr(tree, :(:=)) + lhs = tree.args[1] + rhs = _expand_ast_tree(ctx, srcref, tree.args[2]) + ssadef = gensym("ssadef") + quote + ($(esc(lhs)), $ssadef) = assign_tmp($ctx, $rhs, $(string(lhs))) + $ssadef + end + elseif Meta.isexpr(tree, :macrocall) + esc(tree) + elseif tree isa Expr + Expr(tree.head, map(a->_expand_ast_tree(ctx, srcref, a), tree.args)...) + else + esc(tree) + end +end + +""" + @ast ctx srcref tree + +Syntactic s-expression shorthand for constructing a `SyntaxTree` AST. + +* `ctx` - SyntaxGraph context +* `srcref` - Reference to the source code from which this AST was derived. + +The `tree` contains syntax of the following forms: +* `[kind child₁ child₂]` - construct an interior node with children +* `value :: kind` - construct a leaf node +* `ex => kind` - convert a leaf node to the given `kind`, copying attributes + from it and also using `ex` as the source reference. +* `var := ex` - Set `var=ssavar(...)` and return an assignment node `\$var=ex`. + `var` may be used outside `@ast` +* `cond ? ex1 : ex2` - Conditional; `ex1` and `ex2` will be recursively expanded. + `if ... end` and `if ... else ... end` also work with this. + +Any `kind` can be replaced with an expression of the form +* `kind(srcref)` - override the source reference for this node and its children +* `kind(attr=val)` - set an additional attribute +* `kind(srcref; attr₁=val₁, attr₂=val₂)` - the general form + +In any place `srcref` is used, the special form `@HERE()` can be used to instead +to indicate that the "primary" location of the source is the location where +`@HERE` occurs. + + +# Examples + +``` +@ast ctx srcref [ + K"toplevel" + [K"using" + [K"importpath" + "Base" ::K"Identifier"(src) + ] + ] + [K"function" + [K"call" + "eval" ::K"Identifier" + "x" ::K"Identifier" + ] + [K"call" + "eval" ::K"core" + mn =>K"Identifier" + "x" ::K"Identifier" + ] + ] +] +``` +""" +macro ast(ctx, srcref, tree) + quote + ctx = $(esc(ctx)) + srcref = $(_match_srcref(srcref)) + $(_expand_ast_tree(:ctx, :srcref, tree)) + end +end + +#------------------------------------------------------------------------------- +# Mapping and copying of AST nodes +function copy_attrs!(dest, src, all=false) + # TODO: Make this faster? + for (name, attr) in pairs(src._graph.attributes) + if (all || (name !== :source && name !== :kind && name !== :syntax_flags)) && + haskey(attr, src._id) + dest_attr = getattr(dest._graph, name, nothing) + if !isnothing(dest_attr) + dest_attr[dest._id] = attr[src._id] + end + end + end +end + +function copy_attrs!(dest, head::Union{Kind,JuliaSyntax.SyntaxHead}, all=false) + if all + sethead!(dest._graph, dest._id, head) + end +end + +function mapchildren(f::Function, ctx, ex::SyntaxTree, do_map_child::Function; + extra_attrs...) + if is_leaf(ex) + return ex + end + orig_children = children(ex) + cs = isempty(extra_attrs) ? nothing : SyntaxList(ctx) + for (i,e) in enumerate(orig_children) + newchild = do_map_child(i) ? f(e) : e + if isnothing(cs) + if newchild == e + continue + else + cs = SyntaxList(ctx) + append!(cs, orig_children[1:i-1]) + end + end + push!(cs::SyntaxList, newchild) + end + if isnothing(cs) + # This function should be allocation-free if no children were changed + # by the mapping and there's no extra_attrs + return ex + end + cs::SyntaxList + ex2 = makenode(ctx, ex, head(ex), cs) + copy_attrs!(ex2, ex) + setattr!(ex2; extra_attrs...) + return ex2 +end + +function mapchildren(f::Function, ctx, ex::SyntaxTree, mapped_children::AbstractVector{<:Integer}; + extra_attrs...) + j = Ref(firstindex(mapped_children)) + function do_map_child(i) + ind = j[] + if ind <= lastindex(mapped_children) && mapped_children[ind] == i + j[] += 1 + true + else + false + end + end + mapchildren(f, ctx, ex, do_map_child; extra_attrs...) +end + +function mapchildren(f::Function, ctx, ex::SyntaxTree; extra_attrs...) + mapchildren(f, ctx, ex, i->true; extra_attrs...) +end + + +""" +Copy AST `ex` into `ctx` +""" +function copy_ast(ctx, ex) + # TODO: Do we need to keep a mapping of node IDs to ensure we don't + # double-copy here in the case when some tree nodes are pointed to by + # multiple parents? (How much does this actually happen in practice?) + s = ex.source + # TODO: Figure out how to use provenance() here? + srcref = s isa NodeId ? copy_ast(ctx, SyntaxTree(ex._graph, s)) : + s isa Tuple ? map(i->copy_ast(ctx, SyntaxTree(ex._graph, i)), s) : + s + if !is_leaf(ex) + cs = SyntaxList(ctx) + for e in children(ex) + push!(cs, copy_ast(ctx, e)) + end + ex2 = makenode(ctx, srcref, ex, cs) + else + ex2 = makeleaf(ctx, srcref, ex) + end + return ex2 +end + +#------------------------------------------------------------------------------- +function set_scope_layer(ctx, ex, layer_id, force) + k = kind(ex) + scope_layer = force ? layer_id : get(ex, :scope_layer, layer_id) + if k == K"module" || k == K"toplevel" || k == K"inert" + makenode(ctx, ex, ex, children(ex); + scope_layer=scope_layer) + elseif k == K"." + makenode(ctx, ex, ex, set_scope_layer(ctx, ex[1], layer_id, force), ex[2], + scope_layer=scope_layer) + elseif !is_leaf(ex) + mapchildren(e->set_scope_layer(ctx, e, layer_id, force), ctx, ex; + scope_layer=scope_layer) + else + makeleaf(ctx, ex, ex; + scope_layer=scope_layer) + end +end + +""" + adopt_scope(ex, ref) + +Copy `ex`, adopting the scope layer of `ref`. +""" +function adopt_scope(ex::SyntaxTree, scope_layer::LayerId) + set_scope_layer(ex, ex, scope_layer, true) +end + +function adopt_scope(ex::SyntaxTree, ref::SyntaxTree) + adopt_scope(ex, ref.scope_layer) +end + +function adopt_scope(exs::SyntaxList, ref) + out = SyntaxList(syntax_graph(exs)) + for e in exs + push!(out, adopt_scope(e, ref)) + end + return out +end + +# Type for `meta` attribute, to replace `Expr(:meta)`. +# It's unclear how much flexibility we need here - is a dict good, or could we +# just use a struct? Likely this will be sparse. Alternatively we could just +# use individual attributes but those aren't easy to add on an ad-hoc basis in +# the middle of a pass. +const CompileHints = Base.ImmutableDict{Symbol,Any} + +function setmeta(ex::SyntaxTree; kws...) + @assert length(kws) == 1 # todo relax later ? + key = first(keys(kws)) + value = first(values(kws)) + meta = begin + m = get(ex, :meta, nothing) + isnothing(m) ? CompileHints(key, value) : CompileHints(m, key, value) + end + setattr(ex; meta=meta) +end + +function getmeta(ex::SyntaxTree, name::Symbol, default) + meta = get(ex, :meta, nothing) + isnothing(meta) ? default : get(meta, name, default) +end + +#------------------------------------------------------------------------------- +# Predicates and accessors working on expression trees + +# For historical reasons, `cglobal` and `ccall` are their own special +# quasi-identifier-like syntax but with special handling inside lowering which +# means they can't be used as normal identifiers. +function is_ccall_or_cglobal(name::AbstractString) + return name == "ccall" || name == "cglobal" +end + +function is_quoted(ex) + kind(ex) in KSet"Symbol quote top core globalref break inert + meta inbounds inline noinline loopinfo" +end + +function extension_type(ex) + @assert kind(ex) == K"extension" || kind(ex) == K"assert" + @chk numchildren(ex) >= 1 + @chk kind(ex[1]) == K"Symbol" + ex[1].name_val +end + +function is_sym_decl(x) + k = kind(x) + k == K"Identifier" || k == K"::" +end + +function is_identifier(x) + k = kind(x) + k == K"Identifier" || k == K"var" || is_operator(k) || is_macro_name(k) +end + +function is_eventually_call(ex::SyntaxTree) + k = kind(ex) + return k == K"call" || ((k == K"where" || k == K"::") && is_eventually_call(ex[1])) +end + +function is_function_def(ex) + k = kind(ex) + return k == K"function" || k == K"->" +end + +function find_parameters_ind(exs) + i = length(exs) + while i >= 1 + k = kind(exs[i]) + if k == K"parameters" + return i + elseif k != K"do" + break + end + i -= 1 + end + return 0 +end + +function has_parameters(ex::SyntaxTree) + find_parameters_ind(children(ex)) != 0 +end + +function has_parameters(args::AbstractVector) + find_parameters_ind(args) != 0 +end + +function any_assignment(exs) + any(kind(e) == K"=" for e in exs) +end + +function is_valid_modref(ex) + return kind(ex) == K"." && kind(ex[2]) == K"Symbol" && + (kind(ex[1]) == K"Identifier" || is_valid_modref(ex[1])) +end + +function is_core_ref(ex, name) + kind(ex) == K"core" && ex.name_val == name +end + +function is_core_nothing(ex) + is_core_ref(ex, "nothing") +end + +function is_core_Any(ex) + is_core_ref(ex, "Any") +end + +function is_simple_atom(ctx, ex) + k = kind(ex) + # TODO thismodule + is_literal(k) || k == K"Symbol" || k == K"Value" || is_ssa(ctx, ex) || is_core_nothing(ex) +end + +function is_identifier_like(ex) + k = kind(ex) + k == K"Identifier" || k == K"BindingId" || k == K"Placeholder" +end + +function decl_var(ex) + kind(ex) == K"::" ? ex[1] : ex +end + +# Given the signature of a `function`, return the symbol that will ultimately +# be assigned to in local/global scope, if any. +function assigned_function_name(ex) + while kind(ex) == K"where" + # f() where T + ex = ex[1] + end + if kind(ex) == K"::" && numchildren(ex) == 2 + # f()::T + ex = ex[1] + end + if kind(ex) != K"call" + throw(LoweringError(ex, "Expected call syntax in function signature")) + end + ex = ex[1] + if kind(ex) == K"curly" + # f{T}() + ex = ex[1] + end + if kind(ex) == K"::" || kind(ex) == K"." + # (obj::CallableType)(args) + # A.b.c(args) + nothing + elseif is_identifier_like(ex) + ex + else + throw(LoweringError(ex, "Unexpected name in function signature")) + end +end + +# Remove empty parameters block, eg, in the arg list of `f(x, y;)` +function remove_empty_parameters(args) + i = length(args) + while i > 0 && kind(args[i]) == K"parameters" && numchildren(args[i]) == 0 + i -= 1 + end + args[1:i] +end + +function to_symbol(ctx, ex) + @ast ctx ex ex=>K"Symbol" +end + +function new_scope_layer(ctx, mod_ref::Module=ctx.mod; is_macro_expansion=true) + new_layer = ScopeLayer(length(ctx.scope_layers)+1, ctx.mod, is_macro_expansion) + push!(ctx.scope_layers, new_layer) + new_layer.id +end + +function new_scope_layer(ctx, mod_ref::SyntaxTree) + @assert kind(mod_ref) == K"Identifier" + new_scope_layer(ctx, ctx.scope_layers[mod_ref.scope_layer].mod) +end + +#------------------------------------------------------------------------------- +# Context wrapper which helps to construct a list of statements to be executed +# prior to some expression. Useful when we need to use subexpressions multiple +# times. +struct StatementListCtx{Ctx, GraphType} <: AbstractLoweringContext + ctx::Ctx + stmts::SyntaxList{GraphType} +end + +function Base.getproperty(ctx::StatementListCtx, field::Symbol) + if field === :ctx + getfield(ctx, :ctx) + elseif field === :stmts + getfield(ctx, :stmts) + else + getproperty(getfield(ctx, :ctx), field) + end +end + +function emit(ctx::StatementListCtx, ex) + push!(ctx.stmts, ex) +end + +function emit_assign_tmp(ctx::StatementListCtx, ex, name="tmp") + emit_assign_tmp(ctx.stmts, ctx.ctx, ex, name) +end + +with_stmts(ctx, stmts) = StatementListCtx(ctx, stmts) +with_stmts(ctx::StatementListCtx, stmts) = StatementListCtx(ctx.ctx, stmts) + +function with_stmts(ctx) + StatementListCtx(ctx, SyntaxList(ctx)) +end + +with_stmts(ctx::StatementListCtx) = StatementListCtx(ctx.ctx) diff --git a/src/vendored/JuliaLowering/src/bindings.jl b/src/vendored/JuliaLowering/src/bindings.jl new file mode 100644 index 00000000..e6fda3c2 --- /dev/null +++ b/src/vendored/JuliaLowering/src/bindings.jl @@ -0,0 +1,212 @@ +""" +Metadata about a binding +""" +struct BindingInfo + id::IdTag # Unique integer identifying this binding + name::String + kind::Symbol # :local :global :argument :static_parameter + node_id::Int # ID of associated K"BindingId" node in the syntax graph + mod::Union{Nothing,Module} # Set when `kind === :global` + type::Union{Nothing,SyntaxTree} # Type, for bindings declared like x::T = 10 + n_assigned::Int32 # Number of times variable is assigned to + is_const::Bool # Constant, cannot be reassigned + is_ssa::Bool # Single assignment, defined before use + is_captured::Bool # Variable is captured by some lambda + is_always_defined::Bool # A local that we know has an assignment that dominates all usages (is never undef) + is_internal::Bool # True for internal bindings generated by the compiler + is_ambiguous_local::Bool # Local, but would be global in soft scope (ie, the REPL) + is_nospecialize::Bool # @nospecialize on this argument (only valid for kind == :argument) +end + +function BindingInfo(id::IdTag, name::AbstractString, kind::Symbol, node_id::Integer; + mod::Union{Nothing,Module} = nothing, + type::Union{Nothing,SyntaxTree} = nothing, + n_assigned::Integer = 0, + is_const::Bool = false, + is_ssa::Bool = false, + is_captured::Bool = false, + is_always_defined::Bool = is_ssa, + is_internal::Bool = false, + is_ambiguous_local::Bool = false, + is_nospecialize::Bool = false) + BindingInfo(id, name, kind, node_id, mod, type, n_assigned, is_const, + is_ssa, is_captured, is_always_defined, + is_internal, is_ambiguous_local, is_nospecialize) +end + +""" +Metadata about "entities" (variables, constants, etc) in the program. Each +entity is associated to a unique integer id, the BindingId. A binding will be +inferred for each *name* in the user's source program by symbolic analysis of +the source. + +However, bindings can also be introduced programmatically during lowering or +macro expansion: the primary key for bindings is the `BindingId` integer, not +a name. +""" +struct Bindings + info::Vector{BindingInfo} +end + +Bindings() = Bindings(Vector{BindingInfo}()) + +next_binding_id(bindings::Bindings) = length(bindings.info) + 1 + +function add_binding(bindings::Bindings, binding) + if next_binding_id(bindings) != binding.id + error("Use next_binding_id() to create a valid binding id") + end + push!(bindings.info, binding) +end + +function _binding_id(id::Integer) + id +end + +function _binding_id(ex::SyntaxTree) + @chk kind(ex) == K"BindingId" + ex.var_id +end + +function update_binding!(bindings::Bindings, x; + type=nothing, is_const=nothing, add_assigned=0, + is_always_defined=nothing, is_captured=nothing) + id = _binding_id(x) + b = lookup_binding(bindings, id) + bindings.info[id] = BindingInfo( + b.id, + b.name, + b.kind, + b.node_id, + b.mod, + isnothing(type) ? b.type : type, + b.n_assigned + add_assigned, + isnothing(is_const) ? b.is_const : is_const, + b.is_ssa, + isnothing(is_captured) ? b.is_captured : is_captured, + isnothing(is_always_defined) ? b.is_always_defined : is_always_defined, + b.is_internal, + b.is_ambiguous_local, + b.is_nospecialize + ) +end + +function lookup_binding(bindings::Bindings, x) + bindings.info[_binding_id(x)] +end + +function lookup_binding(ctx::AbstractLoweringContext, x) + lookup_binding(ctx.bindings, x) +end + +function update_binding!(ctx::AbstractLoweringContext, x; kws...) + update_binding!(ctx.bindings, x; kws...) +end + +function new_binding(ctx::AbstractLoweringContext, srcref::SyntaxTree, + name::AbstractString, kind::Symbol; kws...) + binding_id = next_binding_id(ctx.bindings) + ex = @ast ctx srcref binding_id::K"BindingId" + add_binding(ctx.bindings, BindingInfo(binding_id, name, kind, ex._id; kws...)) + ex +end + +# Create a new SSA binding +function ssavar(ctx::AbstractLoweringContext, srcref, name="tmp") + nameref = makeleaf(ctx, srcref, K"Identifier", name_val=name) + new_binding(ctx, nameref, name, :local; is_ssa=true, is_internal=true) +end + +# Create a new local mutable binding or lambda argument +function new_local_binding(ctx::AbstractLoweringContext, srcref, name; kind=:local, kws...) + @assert kind === :local || kind === :argument + nameref = makeleaf(ctx, srcref, K"Identifier", name_val=name) + ex = new_binding(ctx, nameref, name, kind; is_internal=true, kws...) + lbindings = current_lambda_bindings(ctx) + if !isnothing(lbindings) + init_lambda_binding(lbindings, ex.var_id) + end + ex +end + +function new_global_binding(ctx::AbstractLoweringContext, srcref, name, mod; kws...) + nameref = makeleaf(ctx, srcref, K"Identifier", name_val=name) + new_binding(ctx, nameref, name, :global; is_internal=true, mod=mod, kws...) +end + +function binding_ex(ctx::AbstractLoweringContext, id::IdTag) + # Reconstruct the SyntaxTree for this binding. We keep only the node_id + # here, because that's got a concrete type. Whereas if we stored SyntaxTree + # that would contain the type of the graph used in the pass where the + # bindings were created and we'd need to call reparent(), etc. + SyntaxTree(syntax_graph(ctx), lookup_binding(ctx, id).node_id) +end + + +#------------------------------------------------------------------------------- +""" +Metadata about how a binding is used within some enclosing lambda +""" +struct LambdaBindingInfo + is_captured::Bool + is_read::Bool + is_assigned::Bool + # Binding was the function name in a call. Used for specialization + # heuristics in the optimizer. + is_called::Bool +end + +LambdaBindingInfo() = LambdaBindingInfo(false, false, false, false) + +function LambdaBindingInfo(parent::LambdaBindingInfo; + is_captured = nothing, + is_read = nothing, + is_assigned = nothing, + is_called = nothing) + LambdaBindingInfo( + isnothing(is_captured) ? parent.is_captured : is_captured, + isnothing(is_read) ? parent.is_read : is_read, + isnothing(is_assigned) ? parent.is_assigned : is_assigned, + isnothing(is_called) ? parent.is_called : is_called, + ) +end + +struct LambdaBindings + # Bindings used within the lambda + self::IdTag + bindings::Dict{IdTag,LambdaBindingInfo} +end + +LambdaBindings(self::IdTag = 0) = LambdaBindings(self, Dict{IdTag,LambdaBindings}()) + +function init_lambda_binding(bindings::LambdaBindings, id; kws...) + @assert !haskey(bindings.bindings, id) + bindings.bindings[id] = LambdaBindingInfo(LambdaBindingInfo(); kws...) +end + +function update_lambda_binding!(bindings::LambdaBindings, x; kws...) + id = _binding_id(x) + binfo = bindings.bindings[id] + bindings.bindings[id] = LambdaBindingInfo(binfo; kws...) +end + +function update_lambda_binding!(ctx::AbstractLoweringContext, x; kws...) + update_lambda_binding!(current_lambda_bindings(ctx), x; kws...) +end + +function lookup_lambda_binding(bindings::LambdaBindings, x) + get(bindings.bindings, _binding_id(x), nothing) +end + +function lookup_lambda_binding(ctx::AbstractLoweringContext, x) + lookup_lambda_binding(current_lambda_bindings(ctx), x) +end + +function has_lambda_binding(bindings::LambdaBindings, x) + haskey(bindings.bindings, _binding_id(x)) +end + +function has_lambda_binding(ctx::AbstractLoweringContext, x) + has_lambda_binding(current_lambda_bindings(ctx), x) +end + diff --git a/src/vendored/JuliaLowering/src/closure_conversion.jl b/src/vendored/JuliaLowering/src/closure_conversion.jl new file mode 100644 index 00000000..cbcdf16b --- /dev/null +++ b/src/vendored/JuliaLowering/src/closure_conversion.jl @@ -0,0 +1,589 @@ +struct ClosureInfo{GraphType} + # Global name of the type of the closure + type_name::SyntaxTree{GraphType} + # Names of fields for use with getfield, in order + field_names::SyntaxList{GraphType} + # Map from the original BindingId of closed-over vars to the index of the + # associated field in the closure type. + field_inds::Dict{IdTag,Int} +end + +struct ClosureConversionCtx{GraphType} <: AbstractLoweringContext + graph::GraphType + bindings::Bindings + mod::Module + closure_bindings::Dict{IdTag,ClosureBindings} + capture_rewriting::Union{Nothing,ClosureInfo{GraphType},SyntaxList{GraphType}} + lambda_bindings::LambdaBindings + # True if we're in a section of code which preserves top-level sequencing + # such that closure types can be emitted inline with other code. + is_toplevel_seq_point::Bool + toplevel_stmts::SyntaxList{GraphType} + closure_infos::Dict{IdTag,ClosureInfo{GraphType}} +end + +function ClosureConversionCtx(graph::GraphType, bindings::Bindings, + mod::Module, closure_bindings::Dict{IdTag,ClosureBindings}, + lambda_bindings::LambdaBindings) where {GraphType} + ClosureConversionCtx{GraphType}( + graph, bindings, mod, closure_bindings, nothing, + lambda_bindings, false, SyntaxList(graph), Dict{IdTag,ClosureInfo{GraphType}}()) +end + +function current_lambda_bindings(ctx::ClosureConversionCtx) + ctx.lambda_bindings +end + +# Access captured variable from inside a closure +function captured_var_access(ctx, ex) + cap_rewrite = ctx.capture_rewriting + if cap_rewrite isa ClosureInfo + field_sym = cap_rewrite.field_names[cap_rewrite.field_inds[ex.var_id]] + @ast ctx ex [K"call" + "getfield"::K"core" + binding_ex(ctx, current_lambda_bindings(ctx).self) + field_sym + ] + else + interpolations = cap_rewrite + @assert !isnothing(cap_rewrite) + if isempty(interpolations) || !is_same_identifier_like(interpolations[end], ex) + push!(interpolations, ex) + end + @ast ctx ex [K"captured_local" length(interpolations)::K"Integer"] + end +end + +function get_box_contents(ctx::ClosureConversionCtx, var, box_ex) + undef_var = new_local_binding(ctx, var, lookup_binding(ctx, var.var_id).name) + @ast ctx var [K"block" + box := box_ex + # Lower in an UndefVar check to a similarly named variable + # (ref #20016) so that closure lowering Box introduction + # doesn't impact the error message and the compiler is expected + # to fold away the extraneous null check + # + # TODO: Ideally the runtime would rely on provenance info for + # this error and we can remove the isdefined check. + [K"if" [K"call" + "isdefined"::K"core" + box + "contents"::K"Symbol" + ] + ::K"TOMBSTONE" + [K"block" + [K"newvar" undef_var] + undef_var + ] + ] + [K"call" + "getfield"::K"core" + box + "contents"::K"Symbol" + ] + ] +end + +# Convert `ex` to `type` by calling `convert(type, ex)` when necessary. +# +# Used for converting the right hand side of an assignment to a typed local or +# global and for converting the return value of a function call to the declared +# return type. +function convert_for_type_decl(ctx, srcref, ex, type, do_typeassert) + # Use a slot to permit union-splitting this in inference + tmp = new_local_binding(ctx, srcref, "tmp", is_always_defined=true) + + @ast ctx srcref [K"block" + type_tmp := type + # [K"=" type_ssa renumber_assigned_ssavalues(type)] + [K"=" tmp ex] + [K"if" + [K"call" "isa"::K"core" tmp type_tmp] + "nothing"::K"core" + [K"=" + tmp + if do_typeassert + [K"call" + "typeassert"::K"core" + [K"call" "convert"::K"top" type_tmp tmp] + type_tmp + ] + else + [K"call" "convert"::K"top" type_tmp tmp] + end + ] + ] + tmp + ] +end + +function convert_global_assignment(ctx, ex, var, rhs0) + binfo = lookup_binding(ctx, var) + @assert binfo.kind == :global + stmts = SyntaxList(ctx) + rhs1 = if is_simple_atom(ctx, rhs0) + rhs0 + else + tmp = ssavar(ctx, rhs0) + push!(stmts, @ast ctx rhs0 [K"=" tmp rhs0]) + tmp + end + rhs = if binfo.is_const && isnothing(binfo.type) + # const global assignments without a type declaration don't need us to + # deal with the binding type at all. + rhs1 + else + type_var = ssavar(ctx, ex, "binding_type") + push!(stmts, @ast ctx ex [K"=" + type_var + [K"call" + "get_binding_type"::K"core" + binfo.mod::K"Value" + binfo.name::K"Symbol" + ] + ]) + do_typeassert = false # Global assignment type checking is done by the runtime + convert_for_type_decl(ctx, ex, rhs1, type_var, do_typeassert) + end + push!(stmts, @ast ctx ex [K"=" var rhs]) + @ast ctx ex [K"block" + [K"globaldecl" var] + stmts... + rhs1 + ] +end + +# Convert assignment to a closed variable to a `setfield!` call and generate +# `convert` calls for variables with declared types. +# +# When doing this, the original value needs to be preserved, to ensure the +# expression `a=b` always returns exactly `b`. +function convert_assignment(ctx, ex) + var = ex[1] + rhs0 = _convert_closures(ctx, ex[2]) + if kind(var) == K"Placeholder" + return @ast ctx ex [K"=" var rhs0] + end + @chk kind(var) == K"BindingId" + binfo = lookup_binding(ctx, var) + if binfo.kind == :global + convert_global_assignment(ctx, ex, var, rhs0) + else + @assert binfo.kind == :local || binfo.kind == :argument + boxed = is_boxed(binfo) + if isnothing(binfo.type) && !boxed + @ast ctx ex [K"=" var rhs0] + else + # Typed local + tmp_rhs0 = ssavar(ctx, rhs0) + rhs = isnothing(binfo.type) ? tmp_rhs0 : + convert_for_type_decl(ctx, ex, tmp_rhs0, _convert_closures(ctx, binfo.type), true) + assignment = if boxed + @ast ctx ex [K"call" + "setfield!"::K"core" + is_self_captured(ctx, var) ? captured_var_access(ctx, var) : var + "contents"::K"Symbol" + rhs + ] + else + @ast ctx ex [K"=" var rhs] + end + @ast ctx ex [K"block" + [K"=" tmp_rhs0 rhs0] + assignment + tmp_rhs0 + ] + end + end +end + +# Compute fields for a closure type, one field for each captured variable. +function closure_type_fields(ctx, srcref, closure_binds, is_opaque) + capture_ids = Vector{IdTag}() + for lambda_bindings in closure_binds.lambdas + for (id, lbinfo) in lambda_bindings.bindings + if lbinfo.is_captured + push!(capture_ids, id) + end + end + end + # sort here to avoid depending on undefined Dict iteration order. + capture_ids = sort!(unique(capture_ids)) + + field_syms = SyntaxList(ctx) + if is_opaque + field_orig_bindings = capture_ids + # For opaque closures we don't try to generate sensible names for the + # fields as there's no closure type to generate. + for (i,id) in enumerate(field_orig_bindings) + push!(field_syms, @ast ctx srcref i::K"Integer") + end + else + field_names = Dict{String,IdTag}() + for id in capture_ids + binfo = lookup_binding(ctx, id) + # We name each field of the closure after the variable which was closed + # over, for clarity. Adding a suffix can be necessary when collisions + # occur due to macro expansion and generated bindings + name0 = binfo.name + name = name0 + i = 1 + while haskey(field_names, name) + name = "$name0#$i" + i += 1 + end + field_names[name] = id + end + field_orig_bindings = Vector{IdTag}() + for (name,id) in sort!(collect(field_names)) + push!(field_syms, @ast ctx srcref name::K"Symbol") + push!(field_orig_bindings, id) + end + end + field_inds = Dict{IdTag,Int}() + field_is_box = Vector{Bool}() + for (i,id) in enumerate(field_orig_bindings) + push!(field_is_box, is_boxed(ctx, id)) + field_inds[id] = i + end + + return field_syms, field_orig_bindings, field_inds, field_is_box +end + +# Return a thunk which creates a new type for a closure with `field_syms` named +# fields. The new type will be named `name_str` which must be an unassigned +# name in the module. +function type_for_closure(ctx::ClosureConversionCtx, srcref, name_str, field_syms, field_is_box) + # New closure types always belong to the module we're expanding into - they + # need to be serialized there during precompile. + mod = ctx.mod + type_binding = new_global_binding(ctx, srcref, name_str, mod) + type_ex = @ast ctx srcref [K"call" + #"_call_latest"::K"core" + eval_closure_type::K"Value" + ctx.mod::K"Value" + name_str::K"Symbol" + [K"call" "svec"::K"core" field_syms...] + [K"call" "svec"::K"core" [f::K"Bool" for f in field_is_box]...] + ] + type_ex, type_binding +end + +function is_boxed(binfo::BindingInfo) + # True for + # * :argument when it's not reassigned + # * :static_parameter (these can't be reassigned) + defined_but_not_assigned = binfo.is_always_defined && binfo.n_assigned == 0 + # For now, we box almost everything but later we'll want to do dominance + # analysis on the untyped IR. + return binfo.is_captured && !defined_but_not_assigned +end + +function is_boxed(ctx, x) + is_boxed(lookup_binding(ctx, x)) +end + +# Is captured in the closure's `self` argument +function is_self_captured(ctx, x) + lbinfo = lookup_lambda_binding(ctx, x) + !isnothing(lbinfo) && lbinfo.is_captured +end + +# Map the children of `ex` through _convert_closures, lifting any toplevel +# closure definition statements to occur before the other content of `ex`. +function map_cl_convert(ctx::ClosureConversionCtx, ex, toplevel_preserving) + if ctx.is_toplevel_seq_point && !toplevel_preserving + toplevel_stmts = SyntaxList(ctx) + ctx2 = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod, + ctx.closure_bindings, ctx.capture_rewriting, ctx.lambda_bindings, + false, toplevel_stmts, ctx.closure_infos) + res = mapchildren(e->_convert_closures(ctx2, e), ctx2, ex) + if isempty(toplevel_stmts) + res + else + @ast ctx ex [K"block" + toplevel_stmts... + res + ] + end + else + mapchildren(e->_convert_closures(ctx, e), ctx, ex) + end +end + +function _convert_closures(ctx::ClosureConversionCtx, ex) + k = kind(ex) + if k == K"BindingId" + access = is_self_captured(ctx, ex) ? captured_var_access(ctx, ex) : ex + if is_boxed(ctx, ex) + get_box_contents(ctx, ex, access) + else + access + end + elseif is_leaf(ex) || k == K"inert" + ex + elseif k == K"=" + convert_assignment(ctx, ex) + elseif k == K"isdefined" + # Convert isdefined expr to function for closure converted variables + var = ex[1] + binfo = lookup_binding(ctx, var) + if is_boxed(binfo) + access = is_self_captured(ctx, var) ? captured_var_access(ctx, var) : var + @ast ctx ex [K"call" + "isdefined"::K"core" + access + "contents"::K"Symbol" + ] + elseif binfo.is_always_defined || is_self_captured(ctx, var) + # Captured but unboxed vars are always defined + @ast ctx ex true::K"Bool" + elseif binfo.kind == :global + # Normal isdefined won't work for globals (#56985) + @ast ctx ex [K"call" + "isdefinedglobal"::K"core" + ctx.mod::K"Value" + binfo.name::K"Symbol" + false::K"Bool"] + else + ex + end + elseif k == K"decl" + @assert kind(ex[1]) == K"BindingId" + binfo = lookup_binding(ctx, ex[1]) + if binfo.kind == :global + @ast ctx ex [ + K"globaldecl" + ex[1] + _convert_closures(ctx, ex[2]) + ] + else + makeleaf(ctx, ex, K"TOMBSTONE") + end + elseif k == K"local" + var = ex[1] + binfo = lookup_binding(ctx, var) + if binfo.is_captured + @ast ctx ex [K"=" var [K"call" "Box"::K"core"]] + elseif !binfo.is_always_defined + @ast ctx ex [K"newvar" var] + else + makeleaf(ctx, ex, K"TOMBSTONE") + end + elseif k == K"lambda" + closure_convert_lambda(ctx, ex) + elseif k == K"function_decl" + func_name = ex[1] + @assert kind(func_name) == K"BindingId" + func_name_id = func_name.var_id + if haskey(ctx.closure_bindings, func_name_id) + closure_info = get(ctx.closure_infos, func_name_id, nothing) + needs_def = isnothing(closure_info) + if needs_def + closure_binds = ctx.closure_bindings[func_name_id] + field_syms, field_orig_bindings, field_inds, field_is_box = + closure_type_fields(ctx, ex, closure_binds, false) + name_str = reserve_module_binding_i(ctx.mod, + "#$(join(closure_binds.name_stack, "#"))##") + closure_type_def, closure_type_ = + type_for_closure(ctx, ex, name_str, field_syms, field_is_box) + if !ctx.is_toplevel_seq_point + push!(ctx.toplevel_stmts, closure_type_def) + push!(ctx.toplevel_stmts, @ast ctx ex [K"latestworld_if_toplevel"]) + closure_type_def = nothing + end + closure_info = ClosureInfo(closure_type_, field_syms, field_inds) + ctx.closure_infos[func_name_id] = closure_info + type_params = SyntaxList(ctx) + init_closure_args = SyntaxList(ctx) + for (id, boxed) in zip(field_orig_bindings, field_is_box) + field_val = binding_ex(ctx, id) + if is_self_captured(ctx, field_val) + # Access from outer closure if necessary but do not + # unbox to feed into the inner nested closure. + field_val = captured_var_access(ctx, field_val) + end + push!(init_closure_args, field_val) + if !boxed + push!(type_params, @ast ctx ex [K"call" + # TODO: Update to use _typeof_captured_variable (#40985) + #"_typeof_captured_variable"::K"core" + "typeof"::K"core" + field_val]) + end + end + @ast ctx ex [K"block" + closure_type_def + [K"latestworld_if_toplevel"] + closure_type := if isempty(type_params) + closure_type_ + else + [K"call" "apply_type"::K"core" closure_type_ type_params...] + end + closure_val := [K"new" + closure_type + init_closure_args... + ] + convert_assignment(ctx, [K"=" func_name closure_val]) + ::K"TOMBSTONE" + ] + else + @ast ctx ex (::K"TOMBSTONE") + end + else + # Single-arg K"method" has the side effect of creating a global + # binding for `func_name` if it doesn't exist. + @ast ctx ex [K"block" + [K"method" func_name] + ::K"TOMBSTONE" # <- function_decl should not be used in value position + ] + end + elseif k == K"function_type" + func_name = ex[1] + if kind(func_name) == K"BindingId" && lookup_binding(ctx, func_name).kind === :local + ctx.closure_infos[func_name.var_id].type_name + else + @ast ctx ex [K"call" "Typeof"::K"core" func_name] + end + elseif k == K"method_defs" + name = ex[1] + is_closure = kind(name) == K"BindingId" && lookup_binding(ctx, name).kind === :local + cap_rewrite = is_closure ? ctx.closure_infos[name.var_id] : nothing + ctx2 = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod, + ctx.closure_bindings, cap_rewrite, ctx.lambda_bindings, + ctx.is_toplevel_seq_point, ctx.toplevel_stmts, ctx.closure_infos) + body = map_cl_convert(ctx2, ex[2], false) + if is_closure + if ctx.is_toplevel_seq_point + body + else + # Move methods out to a top-level sequence point. + push!(ctx.toplevel_stmts, body) + @ast ctx ex (::K"TOMBSTONE") + end + else + @ast ctx ex [K"block" + body + ::K"TOMBSTONE" + ] + end + elseif k == K"_opaque_closure" + closure_binds = ctx.closure_bindings[ex[1].var_id] + field_syms, field_orig_bindings, field_inds, field_is_box = + closure_type_fields(ctx, ex, closure_binds, true) + + capture_rewrites = ClosureInfo(ex #=unused=#, field_syms, field_inds) + + ctx2 = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod, + ctx.closure_bindings, capture_rewrites, ctx.lambda_bindings, + false, ctx.toplevel_stmts, ctx.closure_infos) + + init_closure_args = SyntaxList(ctx) + for id in field_orig_bindings + push!(init_closure_args, binding_ex(ctx, id)) + end + @ast ctx ex [K"new_opaque_closure" + ex[2] # arg type tuple + ex[3] # return_lower_bound + ex[4] # return_upper_bound + ex[5] # allow_partial + [K"opaque_closure_method" + "nothing"::K"core" + ex[6] # nargs + ex[7] # is_va + ex[8] # functionloc + closure_convert_lambda(ctx2, ex[9]) + ] + init_closure_args... + ] + else + # A small number of kinds are toplevel-preserving in terms of closure + # closure definitions will be lifted out into `toplevel_stmts` if they + # occur inside `ex`. + toplevel_seq_preserving = k == K"if" || k == K"elseif" || k == K"block" || + k == K"tryfinally" || k == K"trycatchelse" + map_cl_convert(ctx, ex, toplevel_seq_preserving) + end +end + +function closure_convert_lambda(ctx, ex) + @assert kind(ex) == K"lambda" + lambda_bindings = ex.lambda_bindings + interpolations = nothing + if isnothing(ctx.capture_rewriting) + # Global method which may capture locals + interpolations = SyntaxList(ctx) + cap_rewrite = interpolations + else + cap_rewrite = ctx.capture_rewriting + end + ctx2 = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod, + ctx.closure_bindings, cap_rewrite, lambda_bindings, + ex.is_toplevel_thunk, ctx.toplevel_stmts, ctx.closure_infos) + lambda_children = SyntaxList(ctx) + args = ex[1] + push!(lambda_children, args) + push!(lambda_children, ex[2]) + + # Add box initializations for arguments which are captured by an inner lambda + body_stmts = SyntaxList(ctx) + for arg in children(args) + kind(arg) != K"Placeholder" || continue + if is_boxed(ctx, arg) + push!(body_stmts, @ast ctx arg [K"=" + arg + [K"call" "Box"::K"core" arg] + ]) + end + end + # Convert body. + input_body_stmts = kind(ex[3]) != K"block" ? ex[3:3] : ex[3][1:end] + for e in input_body_stmts + push!(body_stmts, _convert_closures(ctx2, e)) + end + push!(lambda_children, @ast ctx2 ex[3] [K"block" body_stmts...]) + + if numchildren(ex) > 3 + # Convert return type + @assert numchildren(ex) == 4 + push!(lambda_children, _convert_closures(ctx2, ex[4])) + end + + lam = makenode(ctx, ex, ex, lambda_children; lambda_bindings=lambda_bindings) + if !isnothing(interpolations) && !isempty(interpolations) + @ast ctx ex [K"call" + replace_captured_locals!::K"Value" + lam + [K"call" + "svec"::K"core" + interpolations... + ] + ] + else + lam + end +end + + +""" +Closure conversion and lowering of bindings + +This pass does a few things things: +* Deal with typed variables (K"decl") and their assignments +* Deal with const and non-const global assignments +* Convert closures into types +* Lower variables captured by closures into boxes, etc, as necessary + +Invariants: +* This pass must not introduce new K"Identifier" - only K"BindingId". +* Any new binding IDs must be added to the enclosing lambda locals +""" +function convert_closures(ctx::VariableAnalysisContext, ex) + ctx = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod, + ctx.closure_bindings, ex.lambda_bindings) + ex1 = closure_convert_lambda(ctx, ex) + if !isempty(ctx.toplevel_stmts) + throw(LoweringError(first(ctx.toplevel_stmts), "Top level code was found outside any top level context. `@generated` functions may not contain closures, including `do` syntax and generators/comprehension")) + end + ctx, ex1 +end diff --git a/src/vendored/JuliaLowering/src/desugaring.jl b/src/vendored/JuliaLowering/src/desugaring.jl new file mode 100644 index 00000000..9d9a3c03 --- /dev/null +++ b/src/vendored/JuliaLowering/src/desugaring.jl @@ -0,0 +1,4552 @@ +# Lowering Pass 2 - syntax desugaring + +struct DesugaringContext{GraphType} <: AbstractLoweringContext + graph::GraphType + bindings::Bindings + scope_layers::Vector{ScopeLayer} + mod::Module +end + +function DesugaringContext(ctx) + graph = ensure_attributes(syntax_graph(ctx), + kind=Kind, syntax_flags=UInt16, + source=SourceAttrType, + value=Any, name_val=String, + scope_type=Symbol, # :hard or :soft + var_id=IdTag, + is_toplevel_thunk=Bool) + DesugaringContext(graph, ctx.bindings, ctx.scope_layers, ctx.current_layer.mod) +end + +#------------------------------------------------------------------------------- + +# Return true when `x` and `y` are "the same identifier", but also works with +# bindings (and hence ssa vars). See also `is_identifier_like()` +function is_same_identifier_like(ex::SyntaxTree, y::SyntaxTree) + return (kind(ex) == K"Identifier" && kind(y) == K"Identifier" && NameKey(ex) == NameKey(y)) || + (kind(ex) == K"BindingId" && kind(y) == K"BindingId" && ex.var_id == y.var_id) +end + +function is_same_identifier_like(ex::SyntaxTree, name::AbstractString) + return kind(ex) == K"Identifier" && ex.name_val == name +end + +function contains_identifier(ex::SyntaxTree, idents::AbstractVector{<:SyntaxTree}) + contains_unquoted(ex) do e + any(is_same_identifier_like(e, id) for id in idents) + end +end + +function contains_identifier(ex::SyntaxTree, idents...) + contains_unquoted(ex) do e + any(is_same_identifier_like(e, id) for id in idents) + end +end + +function contains_ssa_binding(ctx, ex) + contains_unquoted(ex) do e + kind(e) == K"BindingId" && lookup_binding(ctx, e).is_ssa + end +end + +# Return true if `f(e)` is true for any unquoted child of `ex`, recursively. +function contains_unquoted(f::Function, ex::SyntaxTree) + if f(ex) + return true + elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert meta") + return any(contains_unquoted(f, e) for e in children(ex)) + else + return false + end +end + +# Identify some expressions that are safe to repeat +# +# TODO: Can we use this in more places? +function is_effect_free(ex) + k = kind(ex) + # TODO: metas + is_literal(k) || is_identifier_like(ex) || k == K"Symbol" || + k == K"inert" || k == K"top" || k == K"core" || k == K"Value" + # flisp also includes `a.b` with simple `a`, but this seems like a bug + # because this calls the user-defined getproperty? +end + +function check_no_parameters(ex::SyntaxTree, msg) + i = find_parameters_ind(children(ex)) + if i > 0 + throw(LoweringError(ex[i], msg)) + end +end + +function check_no_assignment(exs, msg="misplaced assignment statement in `[ ... ]`") + i = findfirst(kind(e) == K"=" for e in exs) + if !isnothing(i) + throw(LoweringError(exs[i], msg)) + end +end + +#------------------------------------------------------------------------------- +# Destructuring + +# Convert things like `(x,y,z) = (a,b,c)` to assignments, eliminating the +# tuple. Includes support for slurping/splatting. This function assumes that +# `_tuple_sides_match` returns true, so the following have already been +# checked: +# * There's max one `...` on the left hand side +# * There's max one `...` on the right hand side, in the last place, or +# matched with an lhs... in the last place. (required so that +# pairwise-matching terms from the right is valid) +# * Neither side has any key=val terms or parameter blocks +# +# Tuple elimination must act /as if/ the right hand side tuple was first +# constructed followed by destructuring. In particular, any side effects due to +# evaluating the individual terms in the right hand side tuple must happen in +# order. +function tuple_to_assignments(ctx, ex) + lhs = ex[1] + rhs = ex[2] + + # Tuple elimination aims to turn assignments between tuples into lists of assignments. + # + # However, there's a complex interplay of side effects due to the + # individual assignments and these can be surprisingly complicated to + # model. For example `(x[i], y) = (f(), g)` can contain the following + # surprises: + # * `tmp = f()` calls `f` which might throw, or modify the bindings for + # `x` or `y`. + # * `x[i] = tmp` is lowered to `setindex!` which might throw or modify the + # bindings for `x` or `y`. + # * `g` might throw an `UndefVarError` + # + # Thus for correctness we introduce temporaries for all right hand sides + # with observable side effects and ensure they're evaluated in order. + n_lhs = numchildren(lhs) + n_rhs = numchildren(rhs) + stmts = SyntaxList(ctx) + rhs_tmps = SyntaxList(ctx) + for i in 1:n_rhs + rh = rhs[i] + r = if kind(rh) == K"..." + rh[1] + else + rh + end + k = kind(r) + if is_literal(k) || k == K"Symbol" || k == K"inert" || k == K"top" || + k == K"core" || k == K"Value" + # Effect-free and nothrow right hand sides do not need a temporary + # (we require nothrow because the order of rhs terms is observable + # due to sequencing, thus identifiers are not allowed) + else + # Example rhs which need a temporary + # * `f()` - arbitrary side effects to any binding + # * `z` - might throw UndefVarError + tmp = emit_assign_tmp(stmts, ctx, r) + rh = kind(rh) == K"..." ? @ast(ctx, rh, [K"..." tmp]) : tmp + end + push!(rhs_tmps, rh) + end + + il = 0 + ir = 0 + while il < n_lhs + il += 1 + ir += 1 + lh = lhs[il] + if kind(lh) == K"..." + # Exactly one lhs `...` occurs in the middle somewhere, with a + # general rhs which has at least as many non-`...` terms or one + # `...` term at the end. + # Examples: + # (x, ys..., z) = (a, b, c, d) + # (x, ys..., z) = (a, bs...) + # (xs..., y) = (a, bs...) + # (xs...) = (a, b, c) + # in this case we can pairwise-match arguments from the end + # backward and emit a general tuple assignment for the middle. + jl = n_lhs + jr = n_rhs + while jl > il && jr > ir + if kind(lhs[jl]) == K"..." || kind(rhs_tmps[jr]) == K"..." + break + end + jl -= 1 + jr -= 1 + end + middle = emit_assign_tmp(stmts, ctx, + @ast(ctx, rhs, [K"tuple" rhs_tmps[ir:jr]...]), + "rhs_tmp" + ) + if il == jl + # (x, ys...) = (a,b,c) + # (x, ys...) = (a,bs...) + # (ys...) = () + push!(stmts, @ast ctx ex [K"=" lh[1] middle]) + else + # (x, ys..., z) = (a, b, c, d) + # (x, ys..., z) = (a, bs...) + # (xs..., y) = (a, bs...) + push!(stmts, @ast ctx ex [K"=" [K"tuple" lhs[il:jl]...] middle]) + end + # Continue with the remainder of the list of non-splat terms + il = jl + ir = jr + else + rh = rhs_tmps[ir] + if kind(rh) == K"..." + push!(stmts, @ast ctx ex [K"=" [K"tuple" lhs[il:end]...] rh[1]]) + break + else + push!(stmts, @ast ctx ex [K"=" lh rh]) + end + end + end + + @ast ctx ex [K"block" + stmts... + [K"removable" [K"tuple" rhs_tmps...]] + ] +end + +# Create an assignment `$lhs = $rhs` where `lhs` must be "simple". If `rhs` is +# a block, sink the assignment into the last statement of the block to keep +# more expressions at top level. `rhs` should already be expanded. +# +# flisp: sink-assignment +function sink_assignment(ctx, srcref, lhs, rhs) + @assert is_identifier_like(lhs) + if kind(rhs) == K"block" + @ast ctx srcref [K"block" + rhs[1:end-1]... + [K"=" lhs rhs[end]] + ] + else + @ast ctx srcref [K"=" lhs rhs] + end +end + +function _tuple_sides_match(lhs, rhs) + N = max(length(lhs), length(rhs)) + for i = 1:N+1 + if i > length(lhs) + # (x, y) = (a, b) # match + # (x,) = (a, b) # no match + return i > length(rhs) + elseif kind(lhs[i]) == K"..." + # (x, ys..., z) = (a, b) # match + # (x, ys...) = (a,) # match + return true + elseif i > length(rhs) + # (x, y) = (a,) # no match + # (x, y, zs...) = (a,) # no match + return false + elseif kind(rhs[i]) == K"..." + # (x, y) = (as...,) # match + # (x, y, z) = (a, bs...) # match + # (x, y) = (as..., b) # no match + return i == length(rhs) + end + end +end + +# Lower `(lhss...) = rhs` in contexts where `rhs` must be a tuple at runtime +# by assuming that `getfield(rhs, i)` works and is efficient. +function lower_tuple_assignment(ctx, assignment_srcref, lhss, rhs) + stmts = SyntaxList(ctx) + tmp = emit_assign_tmp(stmts, ctx, rhs, "rhs_tmp") + for (i, lh) in enumerate(lhss) + push!(stmts, @ast ctx assignment_srcref [K"=" + lh + [K"call" "getfield"::K"core" tmp i::K"Integer"] + ]) + end + makenode(ctx, assignment_srcref, K"block", stmts) +end + +# Implement destructuring with `lhs` a tuple expression (possibly with +# slurping) and `rhs` a general expression. +# +# Destructuring in this context is done via the iteration interface, though +# calls `Base.indexed_iterate()` to allow for a fast path in cases where the +# right hand side is directly indexable. +function _destructure(ctx, assignment_srcref, stmts, lhs, rhs) + n_lhs = numchildren(lhs) + if n_lhs > 0 + iterstate = new_local_binding(ctx, rhs, "iterstate") + end + + end_stmts = SyntaxList(ctx) + + i = 0 + for lh in children(lhs) + i += 1 + if kind(lh) == K"..." + lh1 = if is_identifier_like(lh[1]) + lh[1] + else + lhs_tmp = ssavar(ctx, lh[1], "lhs_tmp") + push!(end_stmts, expand_forms_2(ctx, @ast ctx lh[1] [K"=" lh[1] lhs_tmp])) + lhs_tmp + end + if i == n_lhs + # Slurping as last lhs, eg, for `zs` in + # (x, y, zs...) = rhs + if kind(lh1) != K"Placeholder" + push!(stmts, expand_forms_2(ctx, + @ast ctx assignment_srcref [K"=" + lh1 + [K"call" + "rest"::K"top" + rhs + if i > 1 + iterstate + end + ] + ] + )) + end + else + # Slurping before last lhs. Eg, for `xs` in + # (xs..., y, z) = rhs + # For this we call + # (xs, tail) = Base.split_rest(...) + # then continue iteration with `tail` as new rhs. + tail = ssavar(ctx, lh, "tail") + push!(stmts, + expand_forms_2(ctx, + lower_tuple_assignment(ctx, + assignment_srcref, + (lh1, tail), + @ast ctx assignment_srcref [K"call" + "split_rest"::K"top" + rhs + (n_lhs - i)::K"Integer" + if i > 1 + iterstate + end + ] + ) + ) + ) + rhs = tail + n_lhs = n_lhs - i + i = 0 + end + else + # Normal case, eg, for `y` in + # (x, y, z) = rhs + lh1 = if is_identifier_like(lh) + lh + # elseif is_eventually_call(lh) (TODO??) + else + lhs_tmp = ssavar(ctx, lh, "lhs_tmp") + push!(end_stmts, expand_forms_2(ctx, @ast ctx lh [K"=" lh lhs_tmp])) + lhs_tmp + end + push!(stmts, + expand_forms_2(ctx, + lower_tuple_assignment(ctx, + assignment_srcref, + i == n_lhs ? (lh1,) : (lh1, iterstate), + @ast ctx assignment_srcref [K"call" + "indexed_iterate"::K"top" + rhs + i::K"Integer" + if i > 1 + iterstate + end + ] + ) + ) + ) + end + end + # Actual assignments must happen after the whole iterator is desctructured + # (https://github.com/JuliaLang/julia/issues/40574) + append!(stmts, end_stmts) + stmts +end + +# Expands cases of property destructuring +function expand_property_destruct(ctx, ex) + @assert numchildren(ex) == 2 + lhs = ex[1] + @assert kind(lhs) == K"tuple" + if numchildren(lhs) != 1 + throw(LoweringError(lhs, "Property destructuring must use a single `;` before the property names, eg `(; a, b) = rhs`")) + end + params = lhs[1] + @assert kind(params) == K"parameters" + rhs = ex[2] + stmts = SyntaxList(ctx) + rhs1 = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, rhs)) + for prop in children(params) + propname = kind(prop) == K"Identifier" ? prop : + kind(prop) == K"::" && kind(prop[1]) == K"Identifier" ? prop[1] : + throw(LoweringError(prop, "invalid assignment location")) + push!(stmts, expand_forms_2(ctx, @ast ctx rhs1 [K"=" + prop + [K"call" + "getproperty"::K"top" + rhs1 + propname=>K"Symbol" + ] + ])) + end + push!(stmts, @ast ctx rhs1 [K"removable" rhs1]) + makenode(ctx, ex, K"block", stmts) +end + +# Expands all cases of general tuple destructuring, eg +# (x,y) = (a,b) +function expand_tuple_destruct(ctx, ex) + lhs = ex[1] + @assert kind(lhs) == K"tuple" + rhs = ex[2] + + num_slurp = 0 + for lh in children(lhs) + num_slurp += (kind(lh) == K"...") + if num_slurp > 1 + throw(LoweringError(lh, "multiple `...` in destructuring assignment are ambiguous")) + end + end + + if kind(rhs) == K"tuple" + num_splat = sum(kind(rh) == K"..." for rh in children(rhs)) + if num_splat == 0 && (numchildren(lhs) - num_slurp) > numchildren(rhs) + throw(LoweringError(ex, "More variables on left hand side than right hand in tuple assignment")) + end + + if !any_assignment(children(rhs)) && !has_parameters(rhs) && + _tuple_sides_match(children(lhs), children(rhs)) + return expand_forms_2(ctx, tuple_to_assignments(ctx, ex)) + end + end + + stmts = SyntaxList(ctx) + rhs1 = if is_ssa(ctx, rhs) || + (is_identifier_like(rhs) && + !any(is_same_identifier_like(kind(l) == K"..." ? l[1] : l, rhs) + for l in children(lhs))) + rhs + else + emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, rhs)) + end + _destructure(ctx, ex, stmts, lhs, rhs1) + push!(stmts, @ast ctx rhs1 [K"removable" rhs1]) + makenode(ctx, ex, K"block", stmts) +end + +#------------------------------------------------------------------------------- +# Expand comparison chains + +function expand_scalar_compare_chain(ctx, srcref, terms, i) + comparisons = nothing + while i + 2 <= length(terms) + lhs = terms[i] + op = terms[i+1] + rhs = terms[i+2] + if kind(op) == K"." + break + end + comp = @ast ctx op [K"call" + op + lhs + rhs + ] + if isnothing(comparisons) + comparisons = comp + else + comparisons = @ast ctx srcref [K"&&" + comparisons + comp + ] + end + i += 2 + end + (comparisons, i) +end + +# Expanding comparison chains: (comparison a op b op c ...) +# +# We use && to combine pairs of adjacent scalar comparisons and .& to combine +# vector-vector and vector-scalar comparisons. Combining scalar comparisons are +# treated as having higher precedence than vector comparisons, thus: +# +# a < b < c ==> (a < b) && (b < c) +# a .< b .< c ==> (a .< b) .& (b .< c) +# a < b < c .< d .< e ==> (a < b && b < c) .& (c .< d) .& (d .< e) +# a .< b .< c < d < e ==> (a .< b) .& (b .< c) .& (c < d && d < e) +function expand_compare_chain(ctx, ex) + @assert kind(ex) == K"comparison" + terms = children(ex) + @chk numchildren(ex) >= 3 + @chk isodd(numchildren(ex)) + i = 1 + comparisons = nothing + # Combine any number of dotted comparisons + while i + 2 <= length(terms) + if kind(terms[i+1]) != K"." + (comp, i) = expand_scalar_compare_chain(ctx, ex, terms, i) + else + lhs = terms[i] + op = terms[i+1] + rhs = terms[i+2] + i += 2 + comp = @ast ctx op [K"dotcall" + op[1] + lhs + rhs + ] + end + if isnothing(comparisons) + comparisons = comp + else + comparisons = @ast ctx ex [K"dotcall" + "&"::K"top" + # ^^ NB: Flisp bug. Flisp lowering essentially does + # adopt_scope("&"::K"Identifier", ctx.mod) + # here which seems wrong if the comparison chain arose from + # a macro in a different module. One fix would be to use + # adopt_scope("&"::K"Identifier", ex) + # to get the module of the comparison expression for the + # `&` operator. But a simpler option is probably to always + # use `Base.&` so we do that. + comparisons + comp + ] + end + end + comparisons +end + +#------------------------------------------------------------------------------- +# Expansion of array indexing +function _arg_to_temp(ctx, stmts, ex, eq_is_kw=false) + k = kind(ex) + if is_effect_free(ex) + ex + elseif k == K"..." + @ast ctx ex [k _arg_to_temp(ctx, stmts, ex[1])] + elseif k == K"=" && eq_is_kw + @ast ctx ex [K"=" ex[1] _arg_to_temp(ex[2])] + else + emit_assign_tmp(stmts, ctx, ex) + end +end + +# Make the *arguments* of an expression safe for multiple evaluation, for +# example +# +# a[f(x)] => (temp=f(x); a[temp]) +# +# Any assignments are added to `stmts` and a result expression returned which +# may be used in further desugaring. +function remove_argument_side_effects(ctx, stmts, ex) + if is_literal(ex) || is_identifier_like(ex) + ex + else + k = kind(ex) + if k == K"let" + emit_assign_tmp(stmts, ctx, ex) + else + args = SyntaxList(ctx) + eq_is_kw = ((k == K"call" || k == K"dotcall") && is_prefix_call(ex)) || k == K"ref" + for (i,e) in enumerate(children(ex)) + push!(args, _arg_to_temp(ctx, stmts, e, eq_is_kw && i > 1)) + end + # TODO: Copy attributes? + @ast ctx ex [k args...] + end + end +end + +# Replace any `begin` or `end` symbols with an expression indexing the array +# `arr` in the `n`th index. `splats` are a list of the splatted arguments that +# precede index `n` `is_last` is true when this is this +# last index +function replace_beginend(ctx, ex, arr, n, splats, is_last) + k = kind(ex) + if k == K"Identifier" && ex.name_val in ("begin", "end") + indexfunc = @ast ctx ex (ex.name_val == "begin" ? "firstindex" : "lastindex")::K"top" + if length(splats) == 0 + if is_last && n == 1 + @ast ctx ex [K"call" indexfunc arr] + else + @ast ctx ex [K"call" indexfunc arr n::K"Integer"] + end + else + splat_lengths = SyntaxList(ctx) + for splat in splats + push!(splat_lengths, @ast ctx ex [K"call" "length"::K"top" splat]) + end + @ast ctx ex [K"call" + indexfunc + arr + [K"call" + "+"::K"top" + (n - length(splats))::K"Integer" + splat_lengths... + ] + ] + end + elseif is_leaf(ex) || is_quoted(ex) + ex + elseif k == K"ref" + # inside ref, only replace within the first argument + @ast ctx ex [k + replace_beginend(ctx, ex[1], arr, n, splats, is_last) + ex[2:end]... + ] + # elseif k == K"kw" - keyword args - what does this mean here? + # # note from flisp + # # TODO: this probably should not be allowed since keyword args aren't + # # positional, but in this context we have just used their positions anyway + else + mapchildren(e->replace_beginend(ctx, e, arr, n, splats, is_last), ctx, ex) + end +end + +# Go through indices and replace the `begin` or `end` symbol +# `arr` - array being indexed +# `idxs` - list of indices +# returns the expanded indices. Any statements that need to execute first are +# added to ctx.stmts. +function process_indices(sctx::StatementListCtx, arr, idxs) + has_splats = any(kind(i) == K"..." for i in idxs) + idxs_out = SyntaxList(sctx) + splats = SyntaxList(sctx) + for (n, idx0) in enumerate(idxs) + is_splat = kind(idx0) == K"..." + val = replace_beginend(sctx, is_splat ? idx0[1] : idx0, + arr, n, splats, n == length(idxs)) + # TODO: kwarg? + idx = !has_splats || is_simple_atom(sctx, val) ? val : emit_assign_tmp(sctx, val) + if is_splat + push!(splats, idx) + end + push!(idxs_out, is_splat ? @ast(sctx, idx0, [K"..." idx]) : idx) + end + return idxs_out +end + +# Expand things like `f()[i,end]`, add to `sctx.stmts` (temporaries for +# computing indices) and return +# * `arr` - The array (may be a temporary ssa value) +# * `idxs` - List of indices +function expand_ref_components(sctx::StatementListCtx, ex) + check_no_parameters(ex, "unexpected semicolon in array expression") + @assert kind(ex) == K"ref" + @chk numchildren(ex) >= 1 + arr = ex[1] + idxs = ex[2:end] + if any(contains_identifier(e, "begin", "end") for e in idxs) + arr = emit_assign_tmp(sctx, arr) + end + new_idxs = process_indices(sctx, arr, idxs) + return (arr, new_idxs) +end + +function expand_setindex(ctx, ex) + @assert kind(ex) == K"=" && numchildren(ex) == 2 + lhs = ex[1] + sctx = with_stmts(ctx) + (arr, idxs) = expand_ref_components(sctx, lhs) + rhs = emit_assign_tmp(sctx, ex[2]) + @ast ctx ex [K"block" + sctx.stmts... + expand_forms_2(ctx, [K"call" + "setindex!"::K"top" + arr + rhs + idxs... + ]) + [K"removable" rhs] + ] +end + +#------------------------------------------------------------------------------- +# Expansion of broadcast notation `f.(x .+ y)` + +function expand_dotcall(ctx, ex) + k = kind(ex) + if k == K"dotcall" + @chk numchildren(ex) >= 1 + farg = ex[1] + args = SyntaxList(ctx) + append!(args, ex[2:end]) + kws = remove_kw_args!(ctx, args) + @ast ctx ex [K"call" + (isnothing(kws) ? "broadcasted" : "broadcasted_kwsyntax")::K"top" + farg # todo: What about (z=f).(x,y) ? + (expand_dotcall(ctx, arg) for arg in args)... + if !isnothing(kws) + [K"parameters" + kws... + ] + end + ] + elseif k == K"comparison" + expand_dotcall(ctx, expand_compare_chain(ctx, ex)) + elseif (k == K"&&" || k == K"||") && is_dotted(ex) + @ast ctx ex [K"call" + "broadcasted"::K"top" + (k == K"&&" ? "andand" : "oror")::K"top" + (expand_dotcall(ctx, arg) for arg in children(ex))... + ] + else + ex + end +end + +function expand_fuse_broadcast(ctx, ex) + if kind(ex) == K"=" + @assert is_dotted(ex) + @chk numchildren(ex) == 2 + lhs = ex[1] + kl = kind(lhs) + rhs = expand_dotcall(ctx, ex[2]) + @ast ctx ex [K"call" + "materialize!"::K"top" + if kl == K"ref" + sctx = with_stmts(ctx) + (arr, idxs) = expand_ref_components(sctx, lhs) + [K"block" + sctx.stmts... + [K"call" + "dotview"::K"top" + arr + idxs... + ] + ] + elseif kl == K"." && numchildren(lhs) == 2 + [K"call" + "dotgetproperty"::K"top" + children(lhs)... + ] + else + lhs + end + if !(kind(rhs) == K"call" && kind(rhs[1]) == K"top" && rhs[1].name_val == "broadcasted") + # Ensure the rhs of .= is always wrapped in a call to `broadcasted()` + [K"call"(rhs) + "broadcasted"::K"top" + "identity"::K"top" + rhs + ] + else + rhs + end + ] + else + @ast ctx ex [K"call" + "materialize"::K"top" + expand_dotcall(ctx, ex) + ] + end +end + +#------------------------------------------------------------------------------- +# Expansion of generators and comprehensions + +# Return any subexpression which is a 'return` statement, not including any +# inside quoted sections or method bodies. +function find_return(ex::SyntaxTree) + if kind(ex) == K"return" + return ex + elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert meta function ->") + for e in children(ex) + r = find_return(e) + if !isnothing(r) + return r + end + end + else + return nothing + end +end + +function check_no_return(ex) + r = find_return(ex) + if !isnothing(r) + throw(LoweringError(r, "`return` not allowed inside comprehension or generator")) + end +end + +# Return true for nested tuples of the same identifiers +function similar_tuples_or_identifiers(a, b) + if kind(a) == K"tuple" && kind(b) == K"tuple" + return numchildren(a) == numchildren(b) && + all( ((x,y),)->similar_tuples_or_identifiers(x,y), + zip(children(a), children(b))) + else + is_same_identifier_like(a,b) + end +end + +# Return the anonymous function taking an iterated value, for use with the +# first agument to `Base.Generator` +function func_for_generator(ctx, body, iter_value_destructuring) + if similar_tuples_or_identifiers(iter_value_destructuring, body) + # Use Base.identity for generators which are filters such as + # `(x for x in xs if f(x))`. This avoids creating a new type. + @ast ctx body "identity"::K"top" + else + @ast ctx body [K"->" + [K"tuple" + iter_value_destructuring + ] + [K"block" + body + ] + ] + end +end + +function expand_generator(ctx, ex) + @chk numchildren(ex) >= 2 + body = ex[1] + check_no_return(body) + if numchildren(ex) > 2 + # Uniquify outer vars by NameKey + outervars_by_key = Dict{NameKey,typeof(ex)}() + for iterspecs in ex[2:end-1] + for iterspec in children(iterspecs) + lhs = iterspec[1] + foreach_lhs_var(lhs) do var + @assert kind(var) == K"Identifier" # Todo: K"BindingId"? + outervars_by_key[NameKey(var)] = var + end + end + end + outervar_assignments = SyntaxList(ctx) + for (k,v) in sort(collect(pairs(outervars_by_key)), by=first) + push!(outervar_assignments, @ast ctx v [K"=" v v]) + end + body = @ast ctx ex [K"let" + [K"block" + outervar_assignments... + ] + [K"block" + body + ] + ] + end + for iterspecs_ind in numchildren(ex):-1:2 + iterspecs = ex[iterspecs_ind] + filter_test = nothing + if kind(iterspecs) == K"filter" + filter_test = iterspecs[2] + iterspecs = iterspecs[1] + end + if kind(iterspecs) != K"iteration" + throw(LoweringError("""Expected `K"iteration"` iteration specification in generator""")) + end + iter_ranges = SyntaxList(ctx) + iter_lhss = SyntaxList(ctx) + for iterspec in children(iterspecs) + @chk kind(iterspec) == K"in" + @chk numchildren(iterspec) == 2 + push!(iter_lhss, iterspec[1]) + push!(iter_ranges, iterspec[2]) + end + iter_value_destructuring = if numchildren(iterspecs) == 1 + iterspecs[1][1] + else + iter_lhss = SyntaxList(ctx) + for iterspec in children(iterspecs) + push!(iter_lhss, iterspec[1]) + end + @ast ctx iterspecs [K"tuple" iter_lhss...] + end + iter = if length(iter_ranges) > 1 + @ast ctx iterspecs [K"call" + "product"::K"top" + iter_ranges... + ] + else + iter_ranges[1] + end + if !isnothing(filter_test) + iter = @ast ctx ex [K"call" + "Filter"::K"top" + func_for_generator(ctx, filter_test, iter_value_destructuring) + iter + ] + end + body = @ast ctx ex [K"call" + "Generator"::K"top" + func_for_generator(ctx, body, iter_value_destructuring) + iter + ] + if iterspecs_ind < numchildren(ex) + body = @ast ctx ex [K"call" + "Flatten"::K"top" + body + ] + end + end + body +end + +function expand_comprehension_to_loops(ctx, ex) + @assert kind(ex) == K"typed_comprehension" + element_type = ex[1] + gen = ex[2] + @assert kind(gen) == K"generator" + body = gen[1] + check_no_return(body) + # TODO: check_no_break_continue + iterspecs = gen[2] + @assert kind(iterspecs) == K"iteration" + new_iterspecs = SyntaxList(ctx) + iters = SyntaxList(ctx) + iter_defs = SyntaxList(ctx) + for iterspec in children(iterspecs) + iter = emit_assign_tmp(iter_defs, ctx, iterspec[2], "iter") + push!(iters, iter) + push!(new_iterspecs, @ast ctx iterspec [K"in" iterspec[1] iter]) + end + # Lower to nested for loops + idx = new_local_binding(ctx, iterspecs, "idx") + @ast ctx ex [K"block" + iter_defs... + full_iter := if length(iters) == 1 + iters[1] + else + [K"call" + "product"::K"top" + iters... + ] + end + iter_size := [K"call" "IteratorSize"::K"top" full_iter] + size_unknown := [K"call" "isa"::K"core" iter_size "SizeUnknown"::K"top"] + result := [K"call" "_array_for"::K"top" element_type full_iter iter_size] + [K"=" idx [K"call" "first"::K"top" [K"call" "LinearIndices"::K"top" result]]] + [K"for" [K"iteration" Iterators.reverse(new_iterspecs)...] + [K"block" + val := body + # TODO: inbounds setindex + [K"if" size_unknown + [K"call" "push!"::K"top" result val] + [K"call" "setindex!"::K"top" result val idx] + ] + #[K"call" "println"::K"top" [K"call" "typeof"::K"core" idx]] + [K"=" idx [K"call" "add_int"::K"top" idx 1::K"Integer"]] + ] + ] + result + ] +end + +#------------------------------------------------------------------------------- +# Expansion of array concatenation notation `[a b ; c d]` etc + +function expand_vcat(ctx, ex) + check_no_parameters(ex, "unexpected semicolon in array expression") + check_no_assignment(children(ex)) + had_row = false + had_row_splat = false + is_typed = kind(ex) == K"typed_vcat" + eltype = is_typed ? ex[1] : nothing + elements = is_typed ? ex[2:end] : ex[1:end] + for e in elements + k = kind(e) + if k == K"row" + had_row = true + had_row_splat = had_row_splat || any(kind(e1) == K"..." for e1 in children(e)) + end + end + if had_row_splat + # In case there is splatting inside `hvcat`, collect each row as a + # separate tuple and pass those to `hvcat_rows` instead (ref #38844) + rows = SyntaxList(ctx) + for e in elements + if kind(e) == K"row" + push!(rows, @ast ctx e [K"tuple" children(e)...]) + else + push!(rows, @ast ctx e [K"tuple" e]) + end + end + fname = is_typed ? "typed_hvcat_rows" : "hvcat_rows" + @ast ctx ex [K"call" + fname::K"top" + eltype + rows... + ] + else + row_sizes = SyntaxList(ctx) + flat_elems = SyntaxList(ctx) + for e in elements + if kind(e) == K"row" + rowsize = numchildren(e) + append!(flat_elems, children(e)) + else + rowsize = 1 + push!(flat_elems, e) + end + push!(row_sizes, @ast ctx e rowsize::K"Integer") + end + if had_row + fname = is_typed ? "typed_hvcat" : "hvcat" + @ast ctx ex [K"call" + fname::K"top" + eltype + [K"tuple" row_sizes...] + flat_elems... + ] + else + fname = is_typed ? "typed_vcat" : "vcat" + @ast ctx ex [K"call" + fname::K"top" + eltype + flat_elems... + ] + end + end +end + +function ncat_contains_row(ex) + k = kind(ex) + if k == K"row" + return true + elseif k == K"nrow" + return any(ncat_contains_row(e) for e in children(ex)) + else + return false + end +end + +# flip first and second dimension for row major layouts +function nrow_flipdim(row_major, d) + return !row_major ? d : + d == 1 ? 2 : + d == 2 ? 1 : d +end + +function flatten_ncat_rows!(flat_elems, nrow_spans, row_major, parent_layout_dim, ex) + # Note that most of the checks for valid nesting here are also checked in + # the parser - they can only fail when nrcat is constructed + # programmatically (eg, by a macro). + k = kind(ex) + if k == K"row" + layout_dim = 1 + @chk parent_layout_dim != 1 (ex,"Badly nested rows in `ncat`") + elseif k == K"nrow" + dim = numeric_flags(ex) + @chk dim > 0 (ex,"Unsupported dimension $dim in ncat") + @chk !row_major || dim != 2 (ex,"2D `nrow` cannot be mixed with `row` in `ncat`") + layout_dim = nrow_flipdim(row_major, dim) + elseif kind(ex) == K"..." + throw(LoweringError(ex, "Splatting ... in an `ncat` with multiple dimensions is not supported")) + else + push!(flat_elems, ex) + for ld in parent_layout_dim-1:-1:1 + push!(nrow_spans, (ld, 1)) + end + return + end + row_start = length(flat_elems) + @chk parent_layout_dim > layout_dim (ex, "Badly nested rows in `ncat`") + for e in children(ex) + if layout_dim == 1 + @chk kind(e) ∉ KSet"nrow row" (e,"Badly nested rows in `ncat`") + end + flatten_ncat_rows!(flat_elems, nrow_spans, row_major, layout_dim, e) + end + n_elems_in_row = length(flat_elems) - row_start + for ld in parent_layout_dim-1:-1:layout_dim + push!(nrow_spans, (ld, n_elems_in_row)) + end +end + +# ncat comes in various layouts which we need to lower to special cases +# - one dimensional along some dimension +# - balanced column first or row first +# - ragged colum first or row first +function expand_ncat(ctx, ex) + is_typed = kind(ex) == K"typed_ncat" + outer_dim = numeric_flags(ex) + @chk outer_dim > 0 (ex,"Unsupported dimension in ncat") + eltype = is_typed ? ex[1] : nothing + elements = is_typed ? ex[2:end] : ex[1:end] + hvncat_name = is_typed ? "typed_hvncat" : "hvncat" + if !any(kind(e) in KSet"row nrow" for e in elements) + # One-dimensional ncat along some dimension + # [a ;;; b ;;; c] + return @ast ctx ex [K"call" + hvncat_name::K"top" + eltype + outer_dim::K"Integer" + elements... + ] + end + # N-dimensional case. May be + # * column first or row first: + # [a;b ;;; c;d] + # [a b ;;; c d] + # * balanced or ragged: + # [a ; b ;;; c ; d] + # [a ; b ;;; c] + row_major = any(ncat_contains_row, elements) + @chk !row_major || outer_dim != 2 (ex,"2D `nrow` cannot be mixed with `row` in `ncat`") + flat_elems = SyntaxList(ctx) + # `ncat` syntax nests lower dimensional `nrow` inside higher dimensional + # ones (with the exception of K"row" when `row_major` is true). Each nrow + # spans a number of elements and we first extract that. + nrow_spans = Vector{Tuple{Int,Int}}() + for e in elements + flatten_ncat_rows!(flat_elems, nrow_spans, row_major, + nrow_flipdim(row_major, outer_dim), e) + end + push!(nrow_spans, (outer_dim, length(flat_elems))) + # Construct the shape specification by postprocessing the flat list of + # spans. + sort!(nrow_spans, by=first) # depends on a stable sort + is_balanced = true + i = 1 + dim_lengths = zeros(outer_dim) + prev_dimspan = 1 + while i <= length(nrow_spans) + layout_dim, dimspan = nrow_spans[i] + while i <= length(nrow_spans) && nrow_spans[i][1] == layout_dim + if dimspan != nrow_spans[i][2] + is_balanced = false + break + end + i += 1 + end + is_balanced || break + @assert dimspan % prev_dimspan == 0 + dim_lengths[layout_dim] = dimspan ÷ prev_dimspan + prev_dimspan = dimspan + end + shape_spec = SyntaxList(ctx) + if is_balanced + if row_major + dim_lengths[1], dim_lengths[2] = dim_lengths[2], dim_lengths[1] + end + # For balanced concatenations, the shape is specified by the length + # along each dimension. + for dl in dim_lengths + push!(shape_spec, @ast ctx ex dl::K"Integer") + end + else + # For unbalanced/ragged concatenations, the shape is specified by the + # number of elements in each ND slice of the array, from layout + # dimension 1 to N. See the documentation for `hvncat` for details. + i = 1 + while i <= length(nrow_spans) + groups_for_dim = Int[] + layout_dim = nrow_spans[i][1] + while i <= length(nrow_spans) && nrow_spans[i][1] == layout_dim + push!(groups_for_dim, nrow_spans[i][2]) + i += 1 + end + push!(shape_spec, + @ast ctx ex [K"tuple" + [i::K"Integer" for i in groups_for_dim]... + ] + ) + end + end + @ast ctx ex [K"call" + hvncat_name::K"top" + eltype + [K"tuple" shape_spec...] + row_major::K"Bool" + flat_elems... + ] +end + +#------------------------------------------------------------------------------- +# Expand assignments + +# Expand UnionAll definitions, eg `X{T} = Y{T,T}` +function expand_unionall_def(ctx, srcref, lhs, rhs, is_const=true) + if numchildren(lhs) <= 1 + throw(LoweringError(lhs, "empty type parameter list in type alias")) + end + name = lhs[1] + rr = ssavar(ctx, srcref) + expand_forms_2( + ctx, + @ast ctx srcref [ + K"block" + [K"=" rr [K"where" rhs lhs[2:end]...]] + [is_const ? K"constdecl" : K"assign_const_if_global" name rr] + [K"latestworld_if_toplevel"] + rr + ] + ) +end + +# Expand general assignment syntax, including +# * UnionAll definitions +# * Chained assignments +# * Setting of structure fields +# * Assignments to array elements +# * Destructuring +# * Typed variable declarations +function expand_assignment(ctx, ex, is_const=false) + @chk numchildren(ex) == 2 + lhs = ex[1] + rhs = ex[2] + kl = kind(lhs) + if kl == K"curly" + expand_unionall_def(ctx, ex, lhs, rhs, is_const) + elseif kind(rhs) == K"=" + # Expand chains of assignments + # a = b = c ==> b=c; a=c + stmts = SyntaxList(ctx) + push!(stmts, lhs) + while kind(rhs) == K"=" + push!(stmts, rhs[1]) + rhs = rhs[2] + end + if is_identifier_like(rhs) + tmp_rhs = nothing + rr = rhs + else + tmp_rhs = ssavar(ctx, rhs, "rhs") + rr = tmp_rhs + end + # In const a = b = c, only a is const + stmts[1] = @ast ctx ex [(is_const ? K"constdecl" : K"=") stmts[1] rr] + for i in 2:length(stmts) + stmts[i] = @ast ctx ex [K"=" stmts[i] rr] + end + if !isnothing(tmp_rhs) + pushfirst!(stmts, @ast ctx ex [K"=" tmp_rhs rhs]) + end + expand_forms_2(ctx, + @ast ctx ex [K"block" + stmts... + [K"removable" rr] + ] + ) + elseif is_identifier_like(lhs) + if is_const + rr = ssavar(ctx, rhs) + @ast ctx ex [ + K"block" + sink_assignment(ctx, ex, rr, expand_forms_2(ctx, rhs)) + [K"constdecl" lhs rr] + [K"latestworld"] + [K"removable" rr] + ] + else + sink_assignment(ctx, ex, lhs, expand_forms_2(ctx, rhs)) + end + elseif kl == K"." + # a.b = rhs ==> setproperty!(a, :b, rhs) + @chk !is_const (ex, "cannot declare `.` form const") + @chk numchildren(lhs) == 2 + a = lhs[1] + b = lhs[2] + stmts = SyntaxList(ctx) + # TODO: Do we need these first two temporaries? + if !is_identifier_like(a) + a = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, a), "a_tmp") + end + if kind(b) != K"Symbol" + b = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, b), "b_tmp") + end + if !is_identifier_like(rhs) && !is_literal(rhs) + rhs = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, rhs), "rhs_tmp") + end + @ast ctx ex [K"block" + stmts... + [K"call" "setproperty!"::K"top" a b rhs] + [K"removable" rhs] + ] + elseif kl == K"tuple" + if has_parameters(lhs) + expand_property_destruct(ctx, ex) + else + expand_tuple_destruct(ctx, ex) + end + elseif kl == K"ref" + # a[i1, i2] = rhs + @chk !is_const (ex, "cannot declare ref form const") + expand_forms_2(ctx, expand_setindex(ctx, ex)) + elseif kl == K"::" && numchildren(lhs) == 2 + x = lhs[1] + T = lhs[2] + res = if is_const + expand_forms_2(ctx, @ast ctx ex [ + K"const" + [K"=" + lhs[1] + convert_for_type_decl(ctx, ex, rhs, T, true) + ]]) + elseif is_identifier_like(x) + # Identifer in lhs[1] is a variable type declaration, eg + # x::T = rhs + @ast ctx ex [K"block" + [K"decl" lhs[1] lhs[2]] + is_const ? [K"const" [K"=" lhs[1] rhs]] : [K"=" lhs[1] rhs] + ] + else + # Otherwise just a type assertion, eg + # a[i]::T = rhs ==> (a[i]::T; a[i] = rhs) + # a[f(x)]::T = rhs ==> (tmp = f(x); a[tmp]::T; a[tmp] = rhs) + stmts = SyntaxList(ctx) + l1 = remove_argument_side_effects(ctx, stmts, lhs[1]) + # TODO: What about (f(z),y)::T = rhs? That's broken syntax and + # needs to be detected somewhere but won't be detected here. Maybe + # it shows that remove_argument_side_effects() is not the ideal + # solution here? + # TODO: handle underscore? + @ast ctx ex [K"block" + stmts... + [K"::" l1 lhs[2]] + [K"=" l1 rhs] + ] + end + expand_forms_2(ctx, res) + elseif kl == K"dotcall" + throw(LoweringError(lhs, "invalid dot call syntax on left hand side of assignment")) + elseif kl == K"typed_hcat" + throw(LoweringError(lhs, "invalid spacing in left side of indexed assignment")) + elseif kl == K"typed_vcat" || kl == K"typed_ncat" + throw(LoweringError(lhs, "unexpected `;` in left side of indexed assignment")) + elseif kl == K"vect" || kl == K"hcat" || kl == K"vcat" || kl == K"ncat" + throw(LoweringError(lhs, "use `(a, b) = ...` to assign multiple values")) + else + throw(LoweringError(lhs, "invalid assignment location")) + end +end + +function expand_update_operator(ctx, ex) + k = kind(ex) + dotted = is_dotted(ex) + + @chk numchildren(ex) == 3 + lhs = ex[1] + op = ex[2] + rhs = ex[3] + + stmts = SyntaxList(ctx) + + declT = nothing + if kind(lhs) == K"::" + # eg `a[i]::T += 1` + declT = lhs[2] + decl_lhs = lhs + lhs = lhs[1] + end + + if kind(lhs) == K"ref" + # eg `a[end] = rhs` + sctx = with_stmts(ctx, stmts) + (arr, idxs) = expand_ref_components(sctx, lhs) + lhs = @ast ctx lhs [K"ref" arr idxs...] + end + + lhs = remove_argument_side_effects(ctx, stmts, lhs) + + if dotted + if !(kind(lhs) == K"ref" || (kind(lhs) == K"." && numchildren(lhs) == 2)) + # `f() .+= rhs` + lhs = emit_assign_tmp(stmts, ctx, lhs) + end + else + if kind(lhs) == K"tuple" && contains_ssa_binding(ctx, lhs) + # If remove_argument_side_effects needed to replace an expression + # with an ssavalue, then it can't be updated by assignment + # (JuliaLang/julia#30062) + throw(LoweringError(lhs, "invalid multiple assignment location")) + end + end + + @ast ctx ex [K"block" + stmts... + [K"="(syntax_flags=(dotted ? JuliaSyntax.DOTOP_FLAG : nothing)) + lhs + [(dotted ? K"dotcall" : K"call") + op + if isnothing(declT) + lhs + else + [K"::"(decl_lhs) lhs declT] + end + rhs + ] + ] + ] +end + +#------------------------------------------------------------------------------- +# Expand logical conditional statements + +# Flatten nested && or || nodes and expand their children +function expand_cond_children(ctx, ex, cond_kind=kind(ex), flat_children=SyntaxList(ctx)) + for e in children(ex) + if kind(e) == cond_kind + expand_cond_children(ctx, e, cond_kind, flat_children) + else + push!(flat_children, expand_forms_2(ctx, e)) + end + end + flat_children +end + +# Expand condition in, eg, `if` or `while` +function expand_condition(ctx, ex) + isblock = kind(ex) == K"block" + test = isblock ? ex[end] : ex + k = kind(test) + if k == K"&&" || k == K"||" + # `||` and `&&` get special lowering so that they compile directly to + # jumps rather than first computing a bool and then jumping. + cs = expand_cond_children(ctx, test) + @assert length(cs) > 1 + test = makenode(ctx, test, k, cs) + else + test = expand_forms_2(ctx, test) + end + if isblock + # Special handling so that the rules for `&&` and `||` can be applied + # to the last statement of a block + @ast ctx ex [K"block" ex[1:end-1]... test] + else + test + end +end + +#------------------------------------------------------------------------------- +# Expand let blocks + +function expand_let(ctx, ex) + @chk numchildren(ex) == 2 + bindings = ex[1] + @chk kind(bindings) == K"block" + blk = ex[2] + scope_type = get(ex, :scope_type, :hard) + if numchildren(bindings) == 0 + return @ast ctx ex [K"scope_block"(scope_type=scope_type) blk] + end + for binding in Iterators.reverse(children(bindings)) + kb = kind(binding) + if is_sym_decl(kb) + blk = @ast ctx ex [K"scope_block"(scope_type=scope_type) + [K"local" binding] + blk + ] + elseif kb == K"=" && numchildren(binding) == 2 + lhs = binding[1] + rhs = binding[2] + kl = kind(lhs) + if kl == K"Identifier" || kl == K"BindingId" + blk = @ast ctx binding [K"block" + tmp := rhs + [K"scope_block"(ex, scope_type=scope_type) + [K"local"(lhs) lhs] + [K"always_defined" lhs] + [K"="(binding) lhs tmp] + blk + ] + ] + elseif kl == K"::" + var = lhs[1] + if !(kind(var) in KSet"Identifier BindingId") + throw(LoweringError(var, "Invalid assignment location in let syntax")) + end + blk = @ast ctx binding [K"block" + tmp := rhs + type := lhs[2] + [K"scope_block"(ex, scope_type=scope_type) + [K"local"(lhs) [K"::" var type]] + [K"always_defined" var] + [K"="(binding) var tmp] + blk + ] + ] + elseif kind(lhs) == K"tuple" + lhs_locals = SyntaxList(ctx) + foreach_lhs_var(lhs) do var + push!(lhs_locals, @ast ctx var [K"local" var]) + push!(lhs_locals, @ast ctx var [K"always_defined" var]) + end + blk = @ast ctx binding [K"block" + tmp := rhs + [K"scope_block"(ex, scope_type=scope_type) + lhs_locals... + [K"="(binding) lhs tmp] + blk + ] + ] + else + throw(LoweringError(lhs, "Invalid assignment location in let syntax")) + end + elseif kind(binding) == K"function" + sig = binding[1] + func_name = assigned_function_name(sig) + if isnothing(func_name) + # Some valid function syntaxes define methods on existing types and + # don't really make sense with let: + # let A.f() = 1 ... end + # let (obj::Callable)() = 1 ... end + throw(LoweringError(sig, "Function signature does not define a local function name")) + end + blk = @ast ctx binding [K"block" + [K"scope_block"(ex, scope_type=scope_type) + [K"local"(func_name) func_name] + [K"always_defined" func_name] + binding + [K"scope_block"(ex, scope_type=scope_type) + # The inside of the block is isolated from the closure, + # which itself can only capture values from the outside. + blk + ] + ] + ] + else + throw(LoweringError(binding, "Invalid binding in let")) + continue + end + end + return blk +end + +#------------------------------------------------------------------------------- +# Expand named tuples + +function _named_tuple_expr(ctx, srcref, names, values) + if isempty(names) + @ast ctx srcref [K"call" "NamedTuple"::K"core"] + else + @ast ctx srcref [K"call" + [K"curly" "NamedTuple"::K"core" [K"tuple" names...]] + # NOTE: don't use `tuple` head, so an assignment expression as a value + # doesn't turn this into another named tuple. + [K"call" "tuple"::K"core" values...] + ] + end +end + +function _merge_named_tuple(ctx, srcref, old, new) + if isnothing(old) + new + else + @ast ctx srcref [K"call" "merge"::K"top" old new] + end +end + +function expand_named_tuple(ctx, ex, kws; + field_name="named tuple field", + element_name="named tuple element") + name_strs = Set{String}() + names = SyntaxList(ctx) + values = SyntaxList(ctx) + current_nt = nothing + for (i,kw) in enumerate(kws) + k = kind(kw) + appended_nt = nothing + name = nothing + if kind(k) == K"Identifier" + # x ==> x = x + name = to_symbol(ctx, kw) + value = kw + elseif k == K"=" + # x = a + if kind(kw[1]) != K"Identifier" && kind(kw[1]) != K"Placeholder" + throw(LoweringError(kw[1], "invalid $field_name name")) + end + if kind(kw[2]) == K"..." + throw(LoweringError(kw[2], "`...` cannot be used in a value for a $field_name")) + end + name = to_symbol(ctx, kw[1]) + value = kw[2] + elseif k == K"." + # a.x ==> x=a.x + if kind(kw[2]) != K"Symbol" + throw(LoweringError(kw, "invalid $element_name")) + end + name = to_symbol(ctx, kw[2]) + value = kw + elseif k == K"call" && is_infix_op_call(kw) && numchildren(kw) == 3 && + is_same_identifier_like(kw[1], "=>") + # a=>b ==> $a=b + appended_nt = _named_tuple_expr(ctx, kw, (kw[2],), (kw[3],)) + nothing, nothing + elseif k == K"..." + # args... ==> splat pairs + appended_nt = kw[1] + if isnothing(current_nt) && isempty(names) + # Must call merge to create NT from an initial splat + current_nt = _named_tuple_expr(ctx, ex, (), ()) + end + nothing, nothing + else + throw(LoweringError(kw, "Invalid $element_name")) + end + if !isnothing(name) + if kind(name) == K"Symbol" + name_str = name.name_val + if name_str in name_strs + throw(LoweringError(name, "Repeated $field_name name")) + end + push!(name_strs, name_str) + end + push!(names, name) + push!(values, value) + end + if !isnothing(appended_nt) + if !isempty(names) + current_nt = _merge_named_tuple(ctx, ex, current_nt, + _named_tuple_expr(ctx, ex, names, values)) + empty!(names) + empty!(values) + end + current_nt = _merge_named_tuple(ctx, ex, current_nt, appended_nt) + end + end + if !isempty(names) || isnothing(current_nt) + current_nt = _merge_named_tuple(ctx, ex, current_nt, + _named_tuple_expr(ctx, ex, names, values)) + end + @assert !isnothing(current_nt) + current_nt +end + +#------------------------------------------------------------------------------- +# Call expansion + +function expand_kw_call(ctx, srcref, farg, args, kws) + @ast ctx srcref [K"block" + func := farg + kw_container := expand_named_tuple(ctx, srcref, kws; + field_name="keyword argument", + element_name="keyword argument") + if all(kind(kw) == K"..." for kw in kws) + # In this case need to check kws nonempty at runtime + [K"if" + [K"call" "isempty"::K"top" kw_container] + [K"call" func args...] + [K"call" "kwcall"::K"core" kw_container func args...] + ] + else + [K"call" "kwcall"::K"core" kw_container func args...] + end + ] +end + +function expand_ccall(ctx, ex) + @assert kind(ex) == K"call" && is_core_ref(ex[1], "ccall") + if numchildren(ex) < 4 + throw(LoweringError(ex, "too few arguments to ccall")) + end + cfunc_name = ex[2] + # Detect calling convention if present. + # + # Note `@ccall` also emits `Expr(:cconv, convention, nreq)`, but this is a + # somewhat undocumented performance workaround. Instead we should just make + # sure @ccall can emit foreigncall directly and efficiently. + known_conventions = ("cdecl", "stdcall", "fastcall", "thiscall", "llvmcall") + cconv = if any(is_same_identifier_like(ex[3], id) for id in known_conventions) + ex[3] + end + if isnothing(cconv) + rt_idx = 3 + else + rt_idx = 4 + if numchildren(ex) < 5 + throw(LoweringError(ex, "too few arguments to ccall with calling convention specified")) + end + end + return_type = ex[rt_idx] + arg_type_tuple = ex[rt_idx+1] + args = ex[rt_idx+2:end] + if kind(arg_type_tuple) != K"tuple" + msg = "ccall argument types must be a tuple; try `(T,)`" + if kind(return_type) == K"tuple" + throw(LoweringError(return_type, msg*" and check if you specified a correct return type")) + else + throw(LoweringError(arg_type_tuple, msg)) + end + end + arg_types = children(arg_type_tuple) + vararg_type = nothing + if length(arg_types) >= 1 + va = arg_types[end] + if kind(va) == K"..." + @chk numchildren(va) == 1 + # Ok: vararg function + vararg_type = va + end + end + # todo: use multi-range errors here + if length(args) < length(arg_types) + throw(LoweringError(ex, "Too few arguments in ccall compared to argument types")) + elseif length(args) > length(arg_types) && isnothing(vararg_type) + throw(LoweringError(ex, "More arguments than types in ccall")) + end + if isnothing(vararg_type) + num_required_args = 0 + else + num_required_args = length(arg_types) - 1 + if num_required_args < 1 + throw(LoweringError(vararg_type, "C ABI prohibits vararg without one required argument")) + end + end + sctx = with_stmts(ctx) + expanded_types = SyntaxList(ctx) + for (i, argt) in enumerate(arg_types) + if kind(argt) == K"..." + if i == length(arg_types) + argt = argt[1] + else + throw(LoweringError(argt, "only the trailing ccall argument type should have `...`")) + end + end + if is_same_identifier_like(argt, "Any") + # Special rule: Any becomes core.Any regardless of the module + # scope, and don't need GC roots. + argt = @ast ctx argt "Any"::K"core" + end + push!(expanded_types, expand_forms_2(ctx, argt)) + end + # + # An improvement might be wrap the use of types in cconvert in a special + # K"global_scope" expression which modifies the scope resolution. This + # would at least make the rules self consistent if not pretty. + # + # One small improvement we make here is to emit temporaries for all the + # types used during expansion so at least we don't have their side effects + # more than once. + types_for_conv = SyntaxList(ctx) + for argt in expanded_types + push!(types_for_conv, emit_assign_tmp(sctx, argt)) + end + gc_roots = SyntaxList(ctx) + unsafe_args = SyntaxList(ctx) + for (i,arg) in enumerate(args) + if i > length(expanded_types) + raw_argt = expanded_types[end] + push!(expanded_types, raw_argt) + argt = types_for_conv[end] + else + raw_argt = expanded_types[i] + argt = types_for_conv[i] + end + exarg = expand_forms_2(ctx, arg) + if is_core_Any(raw_argt) + push!(unsafe_args, exarg) + else + cconverted_arg = emit_assign_tmp(sctx, + @ast ctx argt [K"call" + "cconvert"::K"top" + argt + exarg + ] + ) + push!(gc_roots, cconverted_arg) + push!(unsafe_args, + @ast ctx argt [K"call" + "unsafe_convert"::K"top" + argt + cconverted_arg + ] + ) + end + end + @ast ctx ex [K"block" + sctx.stmts... + [K"foreigncall" + expand_forms_2(ctx, cfunc_name) + expand_forms_2(ctx, return_type) + [K"call" + "svec"::K"core" + expanded_types... + ] + num_required_args::K"Integer" + if isnothing(cconv) + "ccall"::K"Symbol" + else + cconv=>K"Symbol" + end + unsafe_args... + gc_roots... # GC roots + ] + ] +end + +# Wrap unsplatted arguments in `tuple`: +# `[a, b, xs..., c]` -> `[(a, b), xs, (c,)]` +function _wrap_unsplatted_args(ctx, call_ex, args) + wrapped = SyntaxList(ctx) + i = 1 + while i <= length(args) + if kind(args[i]) == K"..." + splatarg = args[i] + @chk numchildren(splatarg) == 1 + push!(wrapped, splatarg[1]) + else + i1 = i + # Find range of non-splatted args + while i < length(args) && kind(args[i+1]) != K"..." + i += 1 + end + push!(wrapped, @ast ctx call_ex [K"call" "tuple"::K"core" args[i1:i]...]) + end + i += 1 + end + wrapped +end + +function remove_kw_args!(ctx, args::SyntaxList) + kws = nothing + j = 0 + num_parameter_blocks = 0 + for i in 1:length(args) + arg = args[i] + k = kind(arg) + if k == K"=" + if isnothing(kws) + kws = SyntaxList(ctx) + end + push!(kws, arg) + elseif k == K"parameters" + num_parameter_blocks += 1 + if num_parameter_blocks > 1 + throw(LoweringError(arg, "Cannot have more than one group of keyword arguments separated with `;`")) + end + if numchildren(arg) == 0 + continue # ignore empty parameters (issue #18845) + end + if isnothing(kws) + kws = SyntaxList(ctx) + end + append!(kws, children(arg)) + else + j += 1 + if j < i + args[j] = args[i] + end + end + end + resize!(args, j) + return kws +end + +function expand_call(ctx, ex) + farg = ex[1] + if is_core_ref(farg, "ccall") + return expand_ccall(ctx, ex) + end + args = copy(ex[2:end]) + kws = remove_kw_args!(ctx, args) + if !isnothing(kws) + return expand_forms_2(ctx, expand_kw_call(ctx, ex, farg, args, kws)) + end + if any(kind(arg) == K"..." for arg in args) + # Splatting, eg, `f(a, xs..., b)` + @ast ctx ex [K"call" + "_apply_iterate"::K"core" + "iterate"::K"top" + expand_forms_2(ctx, farg) + expand_forms_2(ctx, _wrap_unsplatted_args(ctx, ex, args))... + ] + elseif kind(farg) == K"Identifier" && farg.name_val == "include" + # world age special case + r = ssavar(ctx, ex) + @ast ctx ex [K"block" + [K"=" r [K"call" + expand_forms_2(ctx, farg) + expand_forms_2(ctx, args)... + ]] + [K"latestworld_if_toplevel"] + r + ] + else + @ast ctx ex [K"call" + expand_forms_2(ctx, farg) + expand_forms_2(ctx, args)... + ] + end +end + +#------------------------------------------------------------------------------- + +function expand_dot(ctx, ex) + @chk numchildren(ex) in (1,2) (ex, "`.` form requires either one or two children") + + if numchildren(ex) == 1 + # eg, `f = .+` + # Upstream TODO: Remove the (. +) representation and replace with use + # of DOTOP_FLAG? This way, `K"."` will be exclusively used for + # getproperty. + @ast ctx ex [K"call" + "BroadcastFunction"::K"top" + ex[1] + ] + elseif numchildren(ex) == 2 + # eg, `x.a` syntax + rhs = ex[2] + # Required to support the possibly dubious syntax `a."b"`. See + # https://github.com/JuliaLang/julia/issues/26873 + # Syntax edition TODO: reconsider this; possibly restrict to only K"String"? + if !(kind(rhs) == K"string" || is_leaf(rhs)) + throw(LoweringError(rhs, "Unrecognized field access syntax")) + end + @ast ctx ex [K"call" + "getproperty"::K"top" + ex[1] + rhs + ] + end +end + +#------------------------------------------------------------------------------- +# Expand for loops + +# Extract the variable names assigned to from a "fancy assignment left hand +# side" such as nested tuple destructuring. +function foreach_lhs_var(f::Function, ex) + k = kind(ex) + if k == K"Identifier" || k == K"BindingId" + f(ex) + elseif k == K"::" && numchildren(ex) == 2 + foreach_lhs_var(f, ex[1]) + elseif k == K"tuple" || k == K"parameters" + for e in children(ex) + foreach_lhs_var(f, e) + end + end + # k == K"Placeholder" ignored, along with everything else - we assume + # validation is done elsewhere. +end + +function expand_for(ctx, ex) + iterspecs = ex[1] + + @chk kind(iterspecs) == K"iteration" + + # Loop variables not declared `outer` are reassigned for each iteration of + # the innermost loop in case the user assigns them to something else. + # (Maybe we should filter these to remove vars not assigned in the loop? + # But that would ideally happen after the variable analysis pass, not + # during desugaring.) + copied_vars = SyntaxList(ctx) + for iterspec in iterspecs[1:end-1] + @chk kind(iterspec) == K"in" + lhs = iterspec[1] + if kind(lhs) != K"outer" + foreach_lhs_var(lhs) do var + push!(copied_vars, @ast ctx var [K"=" var var]) + end + end + end + + loop = ex[2] + for i in numchildren(iterspecs):-1:1 + iterspec = iterspecs[i] + lhs = iterspec[1] + + outer = kind(lhs) == K"outer" + lhs_local_defs = SyntaxList(ctx) + lhs_outer_defs = SyntaxList(ctx) + if outer + lhs = lhs[1] + end + foreach_lhs_var(lhs) do var + if outer + push!(lhs_outer_defs, @ast ctx var var) + else + push!(lhs_local_defs, @ast ctx var [K"local" var]) + end + end + + iter_ex = iterspec[2] + next = new_local_binding(ctx, iterspec, "next") + state = ssavar(ctx, iterspec, "state") + collection = ssavar(ctx, iter_ex, "collection") + + # Assign iteration vars and next state + body = @ast ctx iterspec [K"block" + lhs_local_defs... + lower_tuple_assignment(ctx, iterspec, (lhs, state), next) + loop + ] + + body = if i == numchildren(iterspecs) + # Innermost loop gets the continue label and copied vars + @ast ctx ex [K"break_block" + "loop_cont"::K"symbolic_label" + [K"let"(scope_type=:neutral) + [K"block" + copied_vars... + ] + body + ] + ] + else + # Outer loops get a scope block to contain the iteration vars + @ast ctx ex [K"scope_block"(scope_type=:neutral) + body + ] + end + + loop = @ast ctx ex [K"block" + if outer + [K"assert" + "require_existing_locals"::K"Symbol" + lhs_outer_defs... + ] + end + [K"="(iter_ex) collection iter_ex] + # First call to iterate is unrolled + # next = top.iterate(collection) + [K"="(iterspec) next [K"call" "iterate"::K"top" collection]] + [K"if"(iterspec) # if next !== nothing + [K"call"(iterspec) + "not_int"::K"top" + [K"call" "==="::K"core" next "nothing"::K"core"] + ] + [K"_do_while"(ex) + [K"block" + body + # Advance iterator + [K"="(iterspec) next [K"call" "iterate"::K"top" collection state]] + ] + [K"call"(iterspec) + "not_int"::K"top" + [K"call" "==="::K"core" next "nothing"::K"core"] + ] + ] + ] + ] + end + + @ast ctx ex [K"break_block" "loop_exit"::K"symbolic_label" + loop + ] +end + +#------------------------------------------------------------------------------- +# Expand try/catch/finally + +function match_try(ex) + @chk numchildren(ex) > 1 "Invalid `try` form" + try_ = ex[1] + catch_ = nothing + finally_ = nothing + else_ = nothing + for e in ex[2:end] + k = kind(e) + if k == K"catch" && isnothing(catch_) + @chk numchildren(e) == 2 "Invalid `catch` form" + catch_ = e + elseif k == K"else" && isnothing(else_) + @chk numchildren(e) == 1 + else_ = e[1] + elseif k == K"finally" && isnothing(finally_) + @chk numchildren(e) == 1 + finally_ = e[1] + else + throw(LoweringError(ex, "Invalid clause in `try` form")) + end + end + (try_, catch_, else_, finally_) +end + +function expand_try(ctx, ex) + (try_, catch_, else_, finally_) = match_try(ex) + + if !isnothing(finally_) + # TODO: check unmatched symbolic gotos in try. + end + + try_body = @ast ctx try_ [K"scope_block"(scope_type=:neutral) try_] + + if isnothing(catch_) + try_block = try_body + else + exc_var = catch_[1] + catch_block = catch_[2] + if !is_identifier_like(exc_var) + throw(LoweringError(exc_var, "Expected an identifier as exception variable")) + end + try_block = @ast ctx ex [K"trycatchelse" + try_body + [K"scope_block"(catch_, scope_type=:neutral) + if kind(exc_var) != K"Placeholder" + [K"block" + [K"="(exc_var) exc_var [K"call" current_exception::K"Value"]] + catch_block + ] + else + catch_block + end + ] + if !isnothing(else_) + [K"scope_block"(else_, scope_type=:neutral) else_] + end + ] + end + + if isnothing(finally_) + try_block + else + @ast ctx ex [K"tryfinally" + try_block + [K"scope_block"(finally_, scope_type=:neutral) finally_] + ] + end +end + +#------------------------------------------------------------------------------- +# Expand local/global/const declarations + +# Strip variable type declarations from within a `local` or `global`, returning +# the stripped expression. Works recursively with complex left hand side +# assignments containing tuple destructuring. Eg, given +# (x::T, (y::U, z)) +# strip out stmts = (local x) (decl x T) (local x) (decl y U) (local z) +# and return (x, (y, z)) +function strip_decls!(ctx, stmts, declkind, declkind2, declmeta, ex) + k = kind(ex) + if k == K"Identifier" + if !isnothing(declmeta) + push!(stmts, makenode(ctx, ex, declkind, ex; meta=declmeta)) + else + push!(stmts, makenode(ctx, ex, declkind, ex)) + end + if !isnothing(declkind2) + push!(stmts, makenode(ctx, ex, declkind2, ex)) + end + ex + elseif k == K"Placeholder" + ex + elseif k == K"::" + @chk numchildren(ex) == 2 + name = ex[1] + @chk kind(name) == K"Identifier" + push!(stmts, makenode(ctx, ex, K"decl", name, ex[2])) + strip_decls!(ctx, stmts, declkind, declkind2, declmeta, ex[1]) + elseif k == K"tuple" || k == K"parameters" + cs = SyntaxList(ctx) + for e in children(ex) + push!(cs, strip_decls!(ctx, stmts, declkind, declkind2, declmeta, e)) + end + makenode(ctx, ex, k, cs) + end +end + +# local x, (y=2), z ==> local x; local z; y = 2 +# Note there are differences from lisp (evaluation order?) +function expand_decls(ctx, ex) + declkind = kind(ex) + declmeta = get(ex, :meta, nothing) + if numchildren(ex) == 1 && kind(ex[1]) ∈ KSet"const global local" + declkind2 = kind(ex[1]) + bindings = children(ex[1]) + else + declkind2 = nothing + bindings = children(ex) + end + stmts = SyntaxList(ctx) + for binding in bindings + kb = kind(binding) + if is_prec_assignment(kb) + @chk numchildren(binding) == 2 + lhs = strip_decls!(ctx, stmts, declkind, declkind2, declmeta, binding[1]) + push!(stmts, @ast ctx binding [kb lhs binding[2]]) + elseif is_sym_decl(binding) + if declkind == K"const" || declkind2 == K"const" + throw(LoweringError(ex, "expected assignment after `const`")) + end + strip_decls!(ctx, stmts, declkind, declkind2, declmeta, binding) + else + throw(LoweringError(ex, "invalid syntax in variable declaration")) + end + end + makenode(ctx, ex, K"block", stmts) +end + +# Return all the names that will be bound by the assignment LHS, including +# curlies and calls. +function lhs_bound_names(ex) + k = kind(ex) + if k == K"Placeholder" + [] + elseif is_identifier_like(ex) + [ex] + elseif k in KSet"call curly where ::" + lhs_bound_names(ex[1]) + elseif k in KSet"tuple parameters" + vcat(map(lhs_bound_names, children(ex))...) + else + [] + end +end + +function expand_const_decl(ctx, ex) + function check_assignment(asgn) + @chk (kind(asgn) == K"=") (ex, "expected assignment after `const`") + end + + k = kind(ex[1]) + if numchildren(ex) == 2 + @ast ctx ex [ + K"constdecl" + ex[1] + expand_forms_2(ctx, ex[2]) + ] + elseif k == K"global" + asgn = ex[1][1] + check_assignment(asgn) + globals = map(lhs_bound_names(asgn[1])) do x + @ast ctx ex [K"global" x] + end + @ast ctx ex [ + K"block" + globals... + expand_assignment(ctx, ex[1], true) + ] + elseif k == K"=" + if numchildren(ex[1]) >= 1 && kind(ex[1][1]) == K"tuple" + throw(LoweringError(ex[1][1], "unsupported `const` tuple")) + end + expand_assignment(ctx, ex[1], true) + elseif k == K"local" + throw(LoweringError(ex, "unsupported `const local` declaration")) + else + throw(LoweringError(ex, "expected assignment after `const`")) + end +end + +#------------------------------------------------------------------------------- +# Expansion of function definitions + +function expand_function_arg(ctx, body_stmts, arg, is_last_arg, is_kw) + ex = arg + + if kind(ex) == K"=" + default = ex[2] + ex = ex[1] + else + default = nothing + end + + if kind(ex) == K"..." + if !is_last_arg + typmsg = is_kw ? "keyword" : "positional" + throw(LoweringError(arg, "`...` may only be used for the last $typmsg argument")) + end + @chk numchildren(ex) == 1 + slurp_ex = ex + ex = ex[1] + else + slurp_ex = nothing + end + + if kind(ex) == K"::" + @chk numchildren(ex) in (1,2) + if numchildren(ex) == 1 + type = ex[1] + ex = @ast ctx ex "_"::K"Placeholder" + else + type = ex[2] + ex = ex[1] + end + if is_kw && !isnothing(slurp_ex) + throw(LoweringError(slurp_ex, "keyword argument with `...` may not be given a type")) + end + else + type = @ast ctx ex "Any"::K"core" + end + if !isnothing(slurp_ex) + type = @ast ctx slurp_ex [K"curly" "Vararg"::K"core" type] + end + + k = kind(ex) + if k == K"tuple" && !is_kw + # Argument destructuring + is_nospecialize = getmeta(arg, :nospecialize, false) + name = new_local_binding(ctx, ex, "destructured_arg"; + kind=:argument, is_nospecialize=is_nospecialize) + push!(body_stmts, @ast ctx ex [ + K"local"(meta=CompileHints(:is_destructured_arg, true)) + [K"=" ex name] + ]) + elseif k == K"Identifier" || k == K"Placeholder" + name = ex + else + throw(LoweringError(ex, is_kw ? "Invalid keyword name" : "Invalid function argument")) + end + + return (name, type, default, !isnothing(slurp_ex)) +end + +# Expand `where` clause(s) of a function into (typevar_names, typevar_stmts) where +# - `typevar_names` are the names of the type's type parameters +# - `typevar_stmts` are a list of statements to define a `TypeVar` for each parameter +# name in `typevar_names`, with exactly one per `typevar_name`. Some of these +# may already have been emitted. +# - `new_typevar_stmts` is the list of statements which needs to to be emitted +# prior to uses of `typevar_names`. +function _split_wheres!(ctx, typevar_names, typevar_stmts, new_typevar_stmts, ex) + if kind(ex) == K"where" && numchildren(ex) == 2 + vars_kind = kind(ex[2]) + if vars_kind == K"_typevars" + append!(typevar_names, children(ex[2][1])) + append!(typevar_stmts, children(ex[2][2])) + else + params = vars_kind == K"braces" ? ex[2][1:end] : ex[2:2] + n_existing = length(new_typevar_stmts) + expand_typevars!(ctx, typevar_names, new_typevar_stmts, params) + append!(typevar_stmts, view(new_typevar_stmts, n_existing+1:length(new_typevar_stmts))) + end + _split_wheres!(ctx, typevar_names, typevar_stmts, new_typevar_stmts, ex[1]) + else + ex + end +end + +function method_def_expr(ctx, srcref, callex_srcref, method_table, + typevar_names, arg_names, arg_types, body, ret_var=nothing) + @ast ctx srcref [K"block" + # metadata contains svec(types, sparms, location) + method_metadata := [K"call"(callex_srcref) + "svec" ::K"core" + [K"call" + "svec" ::K"core" + arg_types... + ] + [K"call" + "svec" ::K"core" + typevar_names... + ] + ::K"SourceLocation"(callex_srcref) + ] + [K"method" + isnothing(method_table) ? "nothing"::K"core" : method_table + method_metadata + [K"lambda"(body, is_toplevel_thunk=false) + [K"block" arg_names...] + [K"block" typevar_names...] + body + ret_var # might be `nothing` and hence removed + ] + ] + [K"latestworld"] + [K"removable" method_metadata] + ] +end + +# Select static parameters which are used in function arguments `arg_types`, or +# transitively used. +# +# The transitive usage check probably doesn't guarentee that the types are +# inferrable during dispatch as they may only be part of the bounds of another +# type. Thus we might get false positives here but we shouldn't get false +# negatives. +function select_used_typevars(arg_types, typevar_names, typevar_stmts) + n_typevars = length(typevar_names) + @assert n_typevars == length(typevar_stmts) + # Filter typevar names down to those which are directly used in the arg list + typevar_used = Bool[any(contains_identifier(argtype, tn) for argtype in arg_types) + for tn in typevar_names] + # _Or_ used transitively via other typevars. The following code + # computes this by incrementally coloring the graph of dependencies + # between type vars. + found_used = true + while found_used + found_used = false + for (i,tn) in enumerate(typevar_names) + if typevar_used[i] + continue + end + for j = i+1:n_typevars + if typevar_used[j] && contains_identifier(typevar_stmts[j], tn) + found_used = true + typevar_used[i] = true + break + end + end + end + end + typevar_used +end + +function check_all_typevars_used(arg_types, typevar_names, typevar_stmts) + selected = select_used_typevars(arg_types, typevar_names, typevar_stmts) + unused_typevar = findfirst(s->!s, selected) + if !isnothing(unused_typevar) + # Type variables which may be statically determined to be unused in + # any function argument and therefore can't be inferred during + # dispatch. + throw(LoweringError(typevar_names[unused_typevar], + "Method definition declares type variable but does not use it in the type of any function parameter")) + end +end + +# Return `typevar_names` which are used directly or indirectly in `arg_types`. +function trim_used_typevars(ctx, arg_types, typevar_names, typevar_stmts) + typevar_used = select_used_typevars(arg_types, typevar_names, typevar_stmts) + trimmed_typevar_names = SyntaxList(ctx) + for (used,tn) in zip(typevar_used, typevar_names) + if used + push!(trimmed_typevar_names, tn) + end + end + return trimmed_typevar_names +end + +function is_if_generated(ex) + kind(ex) == K"if" && kind(ex[1]) == K"generated" +end + +# Return true if a function body contains a code generator from `@generated` in +# the form `[K"if" [K"generated"] ...]` +function is_generated(ex) + if is_if_generated(ex) + return true + elseif is_quoted(ex) || kind(ex) == K"function" + return false + else + return any(is_generated, children(ex)) + end +end + +function split_generated(ctx, ex, gen_part) + if is_leaf(ex) + ex + elseif is_if_generated(ex) + gen_part ? @ast(ctx, ex, [K"$" ex[2]]) : ex[3] + else + mapchildren(e->split_generated(ctx, e, gen_part), ctx, ex) + end +end + +# Split @generated function body into two parts: +# * The code generator +# * The non-generated function body +function expand_function_generator(ctx, srcref, callex_srcref, func_name, func_name_str, body, arg_names, typevar_names) + gen_body = if is_if_generated(body) + body[2] # Simple case - don't need interpolation when the whole body is generated + else + expand_quote(ctx, @ast ctx body [K"block" split_generated(ctx, body, true)]) + end + gen_name_str = reserve_module_binding_i(ctx.mod, + "#$(isnothing(func_name_str) ? "_" : func_name_str)@generator#") + gen_name = new_global_binding(ctx, body, gen_name_str, ctx.mod) + + # Set up the arguments for the code generator + gen_arg_names = SyntaxList(ctx) + gen_arg_types = SyntaxList(ctx) + # Self arg + push!(gen_arg_names, new_local_binding(ctx, callex_srcref, "#self#"; kind=:argument)) + push!(gen_arg_types, @ast ctx callex_srcref [K"function_type" gen_name]) + # Macro expansion context arg + if kind(func_name) != K"Identifier" + TODO(func_name, "Which scope do we adopt for @generated generator `__context__` in this case?") + end + push!(gen_arg_names, adopt_scope(@ast(ctx, callex_srcref, "__context__"::K"Identifier"), func_name)) + push!(gen_arg_types, @ast(ctx, callex_srcref, MacroContext::K"Value")) + # Trailing arguments to the generator are provided by the Julia runtime. They are: + # static_parameters... parent_function arg_types... + first_trailing_arg = length(gen_arg_names) + 1 + append!(gen_arg_names, typevar_names) + append!(gen_arg_names, arg_names) + # Apply nospecialize to all arguments to prevent so much codegen and add + # Core.Any type for them + for i in first_trailing_arg:length(gen_arg_names) + gen_arg_names[i] = setmeta(gen_arg_names[i]; nospecialize=true) + push!(gen_arg_types, @ast ctx gen_arg_names[i] "Any"::K"core") + end + # Code generator definition + gen_func_method_defs = @ast ctx srcref [K"block" + [K"function_decl" gen_name] + [K"latestworld_if_toplevel"] + [K"scope_block"(scope_type=:hard) + [K"method_defs" + gen_name + [K"block" + [K"latestworld_if_toplevel"] + method_def_expr(ctx, srcref, callex_srcref, nothing, SyntaxList(ctx), + gen_arg_names, gen_arg_types, gen_body, nothing) + ] + ] + ] + ] + + # Extract non-generated body + nongen_body = @ast ctx body [K"block" + # The Julia runtime associates the code generator with the + # non-generated method by adding this meta to the body. This feels like + # a hack though since the generator ultimately gets attached to the + # method rather than the CodeInfo which we're putting it inside. + [K"meta" + "generated"::K"Symbol" + # The following is code to be evaluated at top level and will wrap + # whatever code comes from the user's generator into an appropriate + # K"lambda" (+ K"with_static_parameters") suitable for lowering + # into a CodeInfo. + # + # todo: As isolated top-level code, we don't actually want to apply + # the normal scope rules of the surrounding function ... it should + # technically have scope resolved at top level. + [K"new" + GeneratedFunctionStub::K"Value" # Use stub type from JuliaLowering + gen_name + # Truncate provenance to just the source file range, as this + # will live permanently in the IR and we probably don't want + # the full provenance tree and intermediate expressions + # (TODO: More truncation. We certainly don't want to store the + # source file either.) + sourceref(srcref)::K"Value" + [K"call" + "svec"::K"core" + "#self#"::K"Symbol" + (n.name_val::K"Symbol"(n) for n in arg_names[2:end])... + ] + [K"call" + "svec"::K"core" + (n.name_val::K"Symbol"(n) for n in typevar_names)... + ] + ] + ] + split_generated(ctx, body, false) + ] + + return gen_func_method_defs, nongen_body +end + +# Generate a method for every number of allowed optional arguments +# For example for `f(x, y=1, z=2)` we generate two additional methods +# f(x) = f(x, 1, 2) +# f(x, y) = f(x, y, 2) +function optional_positional_defs!(ctx, method_stmts, srcref, callex, + method_table, typevar_names, typevar_stmts, + arg_names, arg_types, first_default, + arg_defaults) + # Replace placeholder arguments with variables - we need to pass them to + # the inner method for dispatch even when unused in the inner method body + def_arg_names = map(arg_names) do arg + kind(arg) == K"Placeholder" ? + new_local_binding(ctx, arg, arg.name_val; kind=:argument) : + arg + end + for def_idx = 1:length(arg_defaults) + first_omitted = first_default + def_idx - 1 + trimmed_arg_names = def_arg_names[1:first_omitted-1] + # Call the full method directly if no arguments are reused in + # subsequent defaults. Otherwise conservatively call the function with + # only one additional default argument supplied and let the chain of + # function calls eventually lead to the full method. + any_args_in_trailing_defaults = + any(arg_defaults[def_idx+1:end]) do defaultval + contains_identifier(defaultval, def_arg_names[first_omitted:end]) + end + last_used_default = any_args_in_trailing_defaults ? + def_idx : lastindex(arg_defaults) + body = @ast ctx callex [K"block" + [K"call" + trimmed_arg_names... + arg_defaults[def_idx:last_used_default]... + ] + ] + trimmed_arg_types = arg_types[1:first_omitted-1] + trimmed_typevar_names = trim_used_typevars(ctx, trimmed_arg_types, + typevar_names, typevar_stmts) + # TODO: Ensure we preserve @nospecialize metadata in args + push!(method_stmts, + method_def_expr(ctx, srcref, callex, method_table, + trimmed_typevar_names, trimmed_arg_names, trimmed_arg_types, + body)) + end +end + +function scope_nest(ctx, names, values, body) + for (name, value) in Iterators.reverse(zip(names, values)) + body = @ast ctx name [K"let" [K"block" [K"=" name value]] + body + ] + end + body +end + +# Generate body function and `Core.kwcall` overloads for functions taking keywords. +function keyword_function_defs(ctx, srcref, callex_srcref, name_str, typevar_names, + typevar_stmts, new_typevar_stmts, arg_names, + arg_types, has_slurp, first_default, arg_defaults, + keywords, body, ret_var) + mangled_name = let n = isnothing(name_str) ? "_" : name_str + reserve_module_binding_i(ctx.mod, string(startswith(n, '#') ? "" : "#", n, "#")) + end + # TODO: Is the layer correct here? Which module should be the parent module + # of this body function? + layer = new_scope_layer(ctx; is_macro_expansion=false) + body_func_name = adopt_scope(@ast(ctx, callex_srcref, mangled_name::K"Identifier"), layer) + + kwcall_arg_names = SyntaxList(ctx) + kwcall_arg_types = SyntaxList(ctx) + + push!(kwcall_arg_names, new_local_binding(ctx, callex_srcref, "#self#"; kind=:argument)) + push!(kwcall_arg_types, + @ast ctx callex_srcref [K"call" + "typeof"::K"core" + "kwcall"::K"core" + ] + ) + kws_arg = new_local_binding(ctx, keywords, "kws"; kind=:argument) + push!(kwcall_arg_names, kws_arg) + push!(kwcall_arg_types, @ast ctx keywords "NamedTuple"::K"core") + + body_arg_names = SyntaxList(ctx) + body_arg_types = SyntaxList(ctx) + push!(body_arg_names, new_local_binding(ctx, body_func_name, "#self#"; kind=:argument)) + push!(body_arg_types, @ast ctx body_func_name [K"function_type" body_func_name]) + + non_positional_typevars = typevar_names[map(!, + select_used_typevars(arg_types, typevar_names, typevar_stmts))] + + kw_values = SyntaxList(ctx) + kw_defaults = SyntaxList(ctx) + kw_names = SyntaxList(ctx) + kw_name_syms = SyntaxList(ctx) + has_kw_slurp = false + kwtmp = new_local_binding(ctx, keywords, "kwtmp") + for (i,arg) in enumerate(children(keywords)) + (aname, atype, default, is_slurp) = + expand_function_arg(ctx, nothing, arg, i == numchildren(keywords), true) + push!(kw_names, aname) + name_sym = @ast ctx aname aname=>K"Symbol" + push!(body_arg_names, aname) + + if is_slurp + if !isnothing(default) + throw(LoweringError(arg, "keyword argument with `...` cannot have a default value")) + end + has_kw_slurp = true + push!(body_arg_types, @ast ctx arg [K"call" "pairs"::K"top" "NamedTuple"::K"core"]) + push!(kw_defaults, @ast ctx arg [K"call" "pairs"::K"top" [K"call" "NamedTuple"::K"core"]]) + continue + else + push!(body_arg_types, atype) + end + + if isnothing(default) + default = @ast ctx arg [K"call" + "throw"::K"core" + [K"call" + "UndefKeywordError"::K"core" + name_sym + ] + ] + end + push!(kw_defaults, default) + + # Extract the keyword argument value and check the type + push!(kw_values, @ast ctx arg [K"block" + [K"if" + [K"call" "isdefined"::K"core" kws_arg name_sym] + [K"block" + kwval := [K"call" "getfield"::K"core" kws_arg name_sym] + if is_core_Any(atype) || contains_identifier(atype, non_positional_typevars) + # <- Do nothing in this branch because `atype` includes + # something from the typevars and those static + # parameters don't have values yet. Instead, the type + # will be picked up when the body method is called and + # result in a MethodError during dispatch rather than + # the `TypeError` below. + # + # In principle we could probably construct the + # appropriate UnionAll here in some simple cases but + # the fully general case probably requires simulating + # the runtime's dispatch machinery. + else + [K"if" [K"call" "isa"::K"core" kwval atype] + "nothing"::K"core" + [K"call" + "throw"::K"core" + [K"new" "TypeError"::K"core" + "keyword argument"::K"Symbol" + name_sym + atype + kwval + ] + ] + ] + end + # Compiler performance hack: we reuse the kwtmp slot in all + # keyword if blocks rather than using the if block in value + # position. This cuts down on the number of slots required + # https://github.com/JuliaLang/julia/pull/44333 + [K"=" kwtmp kwval] + ] + [K"=" kwtmp default] + ] + kwtmp + ]) + + push!(kw_name_syms, name_sym) + end + append!(body_arg_names, arg_names) + append!(body_arg_types, arg_types) + + first_default += length(kwcall_arg_names) + append!(kwcall_arg_names, arg_names) + append!(kwcall_arg_types, arg_types) + + kwcall_mtable = @ast(ctx, srcref, "nothing"::K"core") + + kwcall_method_defs = SyntaxList(ctx) + if !isempty(arg_defaults) + # Construct kwcall overloads which forward default positional args on + # to the main kwcall overload. + optional_positional_defs!(ctx, kwcall_method_defs, srcref, callex_srcref, + kwcall_mtable, typevar_names, typevar_stmts, + kwcall_arg_names, kwcall_arg_types, first_default, arg_defaults) + end + + positional_forwarding_args = if has_slurp + a = copy(arg_names) + a[end] = @ast ctx a[end] [K"..." a[end]] + a + else + arg_names + end + + #-------------------------------------------------- + # Construct the "main kwcall overload" which unpacks keywords and checks + # their consistency before dispatching to the user's code in the body + # method. + defaults_depend_on_kw_names = any(val->contains_identifier(val, kw_names), kw_defaults) + defaults_have_assign = any(val->contains_unquoted(e->kind(e) == K"=", val), kw_defaults) + use_ssa_kw_temps = !defaults_depend_on_kw_names && !defaults_have_assign + + if use_ssa_kw_temps + kw_val_stmts = SyntaxList(ctx) + for n in kw_names + # If not using slots for the keyword argument values, still declare + # them for reflection purposes. + push!(kw_val_stmts, @ast ctx n [K"local" n]) + end + kw_val_vars = SyntaxList(ctx) + for val in kw_values + v = emit_assign_tmp(kw_val_stmts, ctx, val, "kwval") + push!(kw_val_vars, v) + end + else + kw_val_vars = kw_names + end + + kwcall_body_tail = @ast ctx keywords [K"block" + if has_kw_slurp + # Slurp remaining keywords into last arg + remaining_kws := [K"call" + "pairs"::K"top" + if isempty(kw_name_syms) + kws_arg + else + [K"call" + "structdiff"::K"top" + kws_arg + [K"curly" + "NamedTuple"::K"core" + [K"tuple" kw_name_syms...] + ] + ] + end + ] + else + # Check that there's no unexpected keywords + [K"if" + [K"call" + "isempty"::K"top" + [K"call" + "diff_names"::K"top" + [K"call" "keys"::K"top" kws_arg] + [K"tuple" kw_name_syms...] + ] + ] + "nothing"::K"core" + [K"call" + "kwerr"::K"top" + kws_arg + positional_forwarding_args... + ] + ] + end + [K"call" + body_func_name + kw_val_vars... + if has_kw_slurp + remaining_kws + end + positional_forwarding_args... + ] + ] + kwcall_body = if use_ssa_kw_temps + @ast ctx keywords [K"block" + kw_val_stmts... + kwcall_body_tail + ] + else + scope_nest(ctx, kw_names, kw_values, kwcall_body_tail) + end + main_kwcall_typevars = trim_used_typevars(ctx, kwcall_arg_types, typevar_names, typevar_stmts) + push!(kwcall_method_defs, + method_def_expr(ctx, srcref, callex_srcref, kwcall_mtable, + main_kwcall_typevars, kwcall_arg_names, kwcall_arg_types, kwcall_body)) + + # Check kws of body method + check_all_typevars_used(body_arg_types, typevar_names, typevar_stmts) + + kw_func_method_defs = @ast ctx srcref [K"block" + [K"function_decl" body_func_name] + [K"latestworld"] + [K"scope_block"(scope_type=:hard) + [K"method_defs" + body_func_name + [K"block" + new_typevar_stmts... + method_def_expr(ctx, srcref, callex_srcref, "nothing"::K"core", + typevar_names, body_arg_names, body_arg_types, + [K"block" + [K"meta" "nkw"::K"Symbol" numchildren(keywords)::K"Integer"] + body + ], + ret_var) + ] + ] + ] + [K"scope_block"(scope_type=:hard) + [K"method_defs" + "nothing"::K"core" + [K"block" + new_typevar_stmts... + kwcall_method_defs... + ] + ] + ] + ] + + #-------------------------------------------------- + # Body for call with no keywords + body_for_positional_args_only = if defaults_depend_on_kw_names + scope_nest(ctx, kw_names, kw_defaults, + @ast ctx srcref [K"call" body_func_name + kw_names... + positional_forwarding_args... + ] + ) + else + @ast ctx srcref [K"call" body_func_name + kw_defaults... + positional_forwarding_args... + ] + end + + kw_func_method_defs, body_for_positional_args_only +end + +# Check valid identifier/function names +function is_invalid_func_name(ex) + k = kind(ex) + if k == K"Identifier" + name = ex.name_val + elseif k == K"." && numchildren(ex) == 2 && kind(ex[2]) == K"Symbol" + # `function A.f(x,y) ...` + name = ex[2].name_val + else + return true + end + return is_ccall_or_cglobal(name) +end + +function expand_function_def(ctx, ex, docs, rewrite_call=identity, rewrite_body=identity) + @chk numchildren(ex) in (1,2) + name = ex[1] + if numchildren(ex) == 1 && is_identifier_like(name) + # Function declaration with no methods + if is_invalid_func_name(name) + throw(LoweringError(name, "Invalid function name")) + end + return @ast ctx ex [K"block" + [K"function_decl" name] + name + ] + end + + typevar_names = SyntaxList(ctx) + typevar_stmts = SyntaxList(ctx) + new_typevar_stmts = SyntaxList(ctx) + if kind(name) == K"where" + # `where` vars end up in two places + # 1. Argument types - the `T` in `x::T` becomes a `TypeVar` parameter in + # the method sig, eg, `function f(x::T) where T ...`. These define the + # static parameters of the method. + # 2. In the method body - either explicitly or implicitly via the method + # return type or default arguments - where `T` turns up as the *name* of + # a special slot of kind ":static_parameter" + name = _split_wheres!(ctx, typevar_names, typevar_stmts, new_typevar_stmts, name) + end + + return_type = nothing + if kind(name) == K"::" + @chk numchildren(name) == 2 + return_type = name[2] + name = name[1] + end + + callex = if kind(name) == K"call" + name + elseif kind(name) == K"tuple" + # Anonymous function syntax `function (x,y) ... end` + @ast ctx name [K"call" + "#anon#"::K"Placeholder" + children(name)... + ] + elseif kind(name) == K"dotcall" + throw(LoweringError(name, "Cannot define function using `.` broadcast syntax")) + else + throw(LoweringError(name, "Bad function definition")) + end + + # Fixup for `new` constructor sigs if necessary + callex = rewrite_call(callex) + + # Construct method argument lists of names and types. + # + # First, match the "self" argument: In the method signature, each function + # gets a self argument name+type. For normal generic functions, this is a + # singleton and subtype of `Function`. But objects of any type can be made + # callable when the self argument is explicitly given using `::` syntax in + # the function name. + name = callex[1] + bare_func_name = nothing + name_str = nothing + doc_obj = nothing + self_name = nothing + if kind(name) == K"::" + # Self argument is specified by user + if numchildren(name) == 1 + # function (::T)() ... + self_type = name[1] + else + # function (f::T)() ... + @chk numchildren(name) == 2 + self_name = name[1] + self_type = name[2] + end + doc_obj = self_type + else + if kind(name) == K"Placeholder" + # Anonymous function. In this case we may use an ssavar for the + # closure's value. + name_str = name.name_val + name = ssavar(ctx, name, name.name_val) + bare_func_name = name + elseif is_invalid_func_name(name) + throw(LoweringError(name, "Invalid function name")) + elseif is_identifier_like(name) + # Add methods to a global `Function` object, or local closure + # type function f() ... + name_str = name.name_val + bare_func_name = name + else + # Add methods to an existing Function + # function A.B.f() ... + if kind(name) == K"." && kind(name[2]) == K"Symbol" + name_str = name[2].name_val + end + end + doc_obj = name # todo: can closures be documented? + self_type = @ast ctx name [K"function_type" name] + end + # Add self argument + if isnothing(self_name) + # TODO: #self# should be symbolic rather than a binding for the cases + # where it's reused in `optional_positional_defs!` because it's + # probably unsafe to reuse bindings for multiple different methods in + # the presence of closure captures or other global binding properties. + # + # This is reminiscent of the need to renumber SSA vars in certain cases + # in the flisp implementation. + self_name = new_local_binding(ctx, name, "#self#"; kind=:argument) + end + + # Expand remaining argument names and types + arg_names = SyntaxList(ctx) + arg_types = SyntaxList(ctx) + push!(arg_names, self_name) + push!(arg_types, self_type) + args = callex[2:end] + keywords = nothing + if !isempty(args) && kind(args[end]) == K"parameters" + keywords = args[end] + args = args[1:end-1] + if numchildren(keywords) == 0 + keywords = nothing + end + end + body_stmts = SyntaxList(ctx) + has_slurp = false + first_default = 0 # index into arg_names/arg_types + arg_defaults = SyntaxList(ctx) + for (i,arg) in enumerate(args) + (aname, atype, default, is_slurp) = expand_function_arg(ctx, body_stmts, arg, + i == length(args), false) + has_slurp |= is_slurp + push!(arg_names, aname) + + # TODO: Ideally, ensure side effects of evaluating arg_types only + # happen once - we should create an ssavar if there's any following + # defaults. (flisp lowering doesn't ensure this either). Beware if + # fixing this that optional_positional_defs! depends on filtering the + # *symbolic* representation of arg_types. + push!(arg_types, atype) + + if isnothing(default) + if !isempty(arg_defaults) && !is_slurp + # TODO: Referring to multiple pieces of syntax in one error message is necessary. + # TODO: Poison ASTs with error nodes and continue rather than immediately throwing. + # + # We should make something like the following kind of thing work! + # arg_defaults[1] = @ast_error ctx arg_defaults[1] """ + # Positional arguments with defaults must occur at the end. + # + # We found a [non-optional position argument]($arg) *after* + # one with a [default value]($(first(arg_defaults))) + # """ + # + throw(LoweringError(args[first_default-1], "optional positional arguments must occur at end")) + end + else + if isempty(arg_defaults) + first_default = i + 1 # Offset for self argument + end + push!(arg_defaults, default) + end + end + + if !isnothing(return_type) + ret_var = ssavar(ctx, return_type, "return_type") + push!(body_stmts, @ast ctx return_type [K"=" ret_var return_type]) + else + ret_var = nothing + end + + body = rewrite_body(ex[2]) + if !isempty(body_stmts) + body = @ast ctx body [ + K"block" + body_stmts... + body + ] + end + + gen_func_method_defs = nothing + if is_generated(body) + gen_func_method_defs, body = + expand_function_generator(ctx, ex, callex, name, name_str, body, arg_names, typevar_names) + + end + + if isnothing(keywords) + kw_func_method_defs = nothing + # NB: The following check seems good as it statically catches any useless + # static parameters which can't be bound during method invocation. + # However it wasn't previously an error so we might need to reduce it + # to a warning? + check_all_typevars_used(arg_types, typevar_names, typevar_stmts) + main_typevar_names = typevar_names + else + # Rewrite `body` here so that the positional-only versions dispatch there. + kw_func_method_defs, body = + keyword_function_defs(ctx, ex, callex, name_str, typevar_names, typevar_stmts, + new_typevar_stmts, arg_names, arg_types, has_slurp, + first_default, arg_defaults, keywords, body, ret_var) + # The main function (but without keywords) needs its typevars trimmed, + # as some of them may be for the keywords only. + main_typevar_names = trim_used_typevars(ctx, arg_types, typevar_names, typevar_stmts) + # ret_var is used only in the body method + ret_var = nothing + end + + method_table_val = nothing # TODO: method overlays + method_table = isnothing(method_table_val) ? + @ast(ctx, callex, "nothing"::K"core") : + ssavar(ctx, ex, "method_table") + method_stmts = SyntaxList(ctx) + + if !isempty(arg_defaults) + optional_positional_defs!(ctx, method_stmts, ex, callex, + method_table, typevar_names, typevar_stmts, + arg_names, arg_types, first_default, arg_defaults) + end + + # The method with all non-default arguments + push!(method_stmts, + method_def_expr(ctx, ex, callex, method_table, main_typevar_names, arg_names, + arg_types, body, ret_var)) + if !isnothing(docs) + method_stmts[end] = @ast ctx docs [K"block" + method_metadata := method_stmts[end] + @ast ctx docs [K"call" + bind_docs!::K"Value" + doc_obj + docs[1] + method_metadata + ] + ] + end + + @ast ctx ex [K"block" + if !isnothing(bare_func_name) + # Need the main function type created here before running any code + # in kw_func_method_defs + [K"function_decl"(bare_func_name) bare_func_name] + end + gen_func_method_defs + kw_func_method_defs + [K"latestworld_if_toplevel"] + [K"scope_block"(scope_type=:hard) + [K"method_defs" + isnothing(bare_func_name) ? "nothing"::K"core" : bare_func_name + [K"block" + new_typevar_stmts... + if !isnothing(method_table_val) + [K"=" method_table method_table_val] + end + method_stmts... + ] + ] + ] + [K"removable" + isnothing(bare_func_name) ? "nothing"::K"core" : bare_func_name + ] + ] +end + +#------------------------------------------------------------------------------- +# Anon function syntax +function expand_arrow_arglist(ctx, arglist, arrowname) + k = kind(arglist) + if k == K"where" + @ast ctx arglist [K"where" + expand_arrow_arglist(ctx, arglist[1], arrowname) + argslist[2] + ] + else + # The arglist can sometimes be parsed as a block, or something else, and + # fixing this is extremely awkward when nested inside `where`. See + # https://github.com/JuliaLang/JuliaSyntax.jl/pull/522 + if k == K"block" + @chk numchildren(arglist) == 2 + arglist = @ast ctx arglist [K"tuple" + ex[1] + [K"parameters" ex[2]] + ] + elseif k != K"tuple" + # `x::Int -> body` + arglist = @ast ctx arglist [K"tuple" + ex[1] + ] + end + @ast ctx arglist [K"call" + arrowname::K"Placeholder" + children(arglist)... + ] + end +end + +function expand_arrow(ctx, ex) + @chk numchildren(ex) == 2 + expand_forms_2(ctx, + @ast ctx ex [K"function" + expand_arrow_arglist(ctx, ex[1], string(kind(ex))) + ex[2] + ] + ) +end + +function expand_opaque_closure(ctx, ex) + arg_types_spec = ex[1] + return_lower_bound = ex[2] + return_upper_bound = ex[3] + allow_partial = ex[4] + func_expr = ex[5] + @chk kind(func_expr) == K"->" + @chk numchildren(func_expr) == 2 + args = func_expr[1] + @chk kind(args) == K"tuple" + check_no_parameters(ex, args) + + arg_names = SyntaxList(ctx) + arg_types = SyntaxList(ctx) + push!(arg_names, new_local_binding(ctx, args, "#self#"; kind=:argument)) + body_stmts = SyntaxList(ctx) + is_va = false + for (i, arg) in enumerate(children(args)) + (aname, atype, default, is_slurp) = expand_function_arg(ctx, body_stmts, arg, + i == numchildren(args), false) + is_va |= is_slurp + push!(arg_names, aname) + push!(arg_types, atype) + if !isnothing(default) + throw(LoweringError(default, "Default positional arguments cannot be used in an opaque closure")) + end + end + + nargs = length(arg_names) - 1 # ignoring #self# + + @ast ctx ex [K"_opaque_closure" + ssavar(ctx, ex, "opaque_closure_id") # only a placeholder. Must be :local + if is_core_nothing(arg_types_spec) + [K"curly" + "Tuple"::K"core" + arg_types... + ] + else + arg_types_spec + end + is_core_nothing(return_lower_bound) ? [K"curly" "Union"::K"core"] : return_lower_bound + is_core_nothing(return_upper_bound) ? "Any"::K"core" : return_upper_bound + allow_partial + nargs::K"Integer" + is_va::K"Bool" + ::K"SourceLocation"(func_expr) + [K"lambda"(func_expr, is_toplevel_thunk=false) + [K"block" arg_names...] + [K"block"] + [K"block" + body_stmts... + func_expr[2] + ] + ] + ] +end + +#------------------------------------------------------------------------------- +# Expand macro definitions + +function _make_macro_name(ctx, ex) + k = kind(ex) + if k == K"Identifier" || k == K"Symbol" + name = mapleaf(ctx, ex, k) + name.name_val = "@$(ex.name_val)" + name + elseif is_valid_modref(ex) + @chk numchildren(ex) == 2 + @ast ctx ex [K"." ex[1] _make_macro_name(ctx, ex[2])] + else + throw(LoweringError(ex, "invalid macro name")) + end +end + +# flisp: expand-macro-def +function expand_macro_def(ctx, ex) + @chk numchildren(ex) >= 1 (ex,"invalid macro definition") + if numchildren(ex) == 1 + name = ex[1] + # macro with zero methods + # `macro m end` + return @ast ctx ex [K"function" _make_macro_name(ctx, name)] + end + # TODO: Making this manual pattern matching robust is such a pain!!! + sig = ex[1] + @chk (kind(sig) == K"call" && numchildren(sig) >= 1) (sig, "invalid macro signature") + name = sig[1] + args = remove_empty_parameters(children(sig)) + @chk kind(args[end]) != K"parameters" (args[end], "macros cannot accept keyword arguments") + ret = @ast ctx ex [K"function" + [K"call"(sig) + _make_macro_name(ctx, name) + [K"::" + adopt_scope(@ast(ctx, sig, "__context__"::K"Identifier"), + kind(name) == K"." ? name[1] : name) + MacroContext::K"Value" + ] + # flisp: We don't mark these @nospecialize because all arguments to + # new macros will be of type SyntaxTree + args[2:end]... + ] + ex[2] + ] +end + +#------------------------------------------------------------------------------- +# Expand type definitions + +# Match `x<:T<:y` etc, returning `(name, lower_bound, upper_bound)` +# A bound is `nothing` if not specified +function analyze_typevar(ctx, ex) + k = kind(ex) + if k == K"Identifier" + (ex, nothing, nothing) + elseif k == K"comparison" && numchildren(ex) == 5 + kind(ex[3]) == K"Identifier" || throw(LoweringError(ex[3], "expected type name")) + if !((kind(ex[2]) == K"Identifier" && ex[2].name_val == "<:") && + (kind(ex[4]) == K"Identifier" && ex[4].name_val == "<:")) + throw(LoweringError(ex, "invalid type bounds")) + end + # a <: b <: c + (ex[3], ex[1], ex[5]) + elseif k == K"<:" && numchildren(ex) == 2 + kind(ex[1]) == K"Identifier" || throw(LoweringError(ex[1], "expected type name")) + (ex[1], nothing, ex[2]) + elseif k == K">:" && numchildren(ex) == 2 + kind(ex[2]) == K"Identifier" || throw(LoweringError(ex[2], "expected type name")) + (ex[2], ex[1], nothing) + else + throw(LoweringError(ex, "expected type name or type bounds")) + end +end + +function bounds_to_TypeVar(ctx, srcref, bounds) + name, lb, ub = bounds + # Generate call to one of + # TypeVar(name) + # TypeVar(name, ub) + # TypeVar(name, lb, ub) + @ast ctx srcref [K"call" + "TypeVar"::K"core" + name=>K"Symbol" + lb + if isnothing(ub) && !isnothing(lb) + "Any"::K"core" + else + ub + end + ] +end + +# Analyze type signatures such as `A{C} <: B where C` +# +# Return (name, typevar_names, typevar_stmts, supertype) where +# - `name` is the name of the type +# - `supertype` is the super type of the type +function analyze_type_sig(ctx, ex) + k = kind(ex) + if k == K"Identifier" + name = ex + type_params = () + supertype = @ast ctx ex "Any"::K"core" + elseif k == K"curly" && numchildren(ex) >= 1 && kind(ex[1]) == K"Identifier" + # name{type_params} + name = ex[1] + type_params = ex[2:end] + supertype = @ast ctx ex "Any"::K"core" + elseif k == K"<:" && numchildren(ex) == 2 + if kind(ex[1]) == K"Identifier" + name = ex[1] + type_params = () + supertype = ex[2] + elseif kind(ex[1]) == K"curly" && numchildren(ex[1]) >= 1 && kind(ex[1][1]) == K"Identifier" + name = ex[1][1] + type_params = ex[1][2:end] + supertype = ex[2] + end + end + @isdefined(name) || throw(LoweringError(ex, "invalid type signature")) + + return (name, type_params, supertype) +end + +# Expand type_params into (typevar_names, typevar_stmts) where +# - `typevar_names` are the names of the type's type parameters +# - `typevar_stmts` are a list of statements to define a `TypeVar` for each parameter +# name in `typevar_names`, to be emitted prior to uses of `typevar_names`. +# There is exactly one statement from each typevar. +function expand_typevars!(ctx, typevar_names, typevar_stmts, type_params) + for param in type_params + bounds = analyze_typevar(ctx, param) + n = bounds[1] + push!(typevar_names, n) + push!(typevar_stmts, @ast ctx param [K"block" + [K"local" n] + [K"=" n bounds_to_TypeVar(ctx, param, bounds)] + ]) + end + return nothing +end + +function expand_typevars(ctx, type_params) + typevar_names = SyntaxList(ctx) + typevar_stmts = SyntaxList(ctx) + expand_typevars!(ctx, typevar_names, typevar_stmts, type_params) + return (typevar_names, typevar_stmts) +end + +function expand_abstract_or_primitive_type(ctx, ex) + is_abstract = kind(ex) == K"abstract" + if is_abstract + @chk numchildren(ex) == 1 + else + @assert kind(ex) == K"primitive" + @chk numchildren(ex) == 2 + nbits = ex[2] + end + name, type_params, supertype = analyze_type_sig(ctx, ex[1]) + typevar_names, typevar_stmts = expand_typevars(ctx, type_params) + newtype_var = ssavar(ctx, ex, "new_type") + @ast ctx ex [K"block" + [K"scope_block"(scope_type=:hard) + [K"block" + [K"local" name] + [K"always_defined" name] + typevar_stmts... + [K"=" + newtype_var + [K"call" + (is_abstract ? "_abstracttype" : "_primitivetype")::K"core" + ctx.mod::K"Value" + name=>K"Symbol" + [K"call" "svec"::K"core" typevar_names...] + if !is_abstract + nbits + end + ] + ] + [K"=" name newtype_var] + [K"call" "_setsuper!"::K"core" newtype_var supertype] + [K"call" "_typebody!"::K"core" false::K"Bool" name] + ] + ] + [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex] ] + [K"global" name] + [K"if" + [K"&&" + [K"call" + "isdefinedglobal"::K"core" + ctx.mod::K"Value" + name=>K"Symbol" + false::K"Bool"] + [K"call" "_equiv_typedef"::K"core" name newtype_var] + ] + nothing_(ctx, ex) + [K"constdecl" name newtype_var] + ] + [K"latestworld"] + nothing_(ctx, ex) + ] +end + +function _match_struct_field(x0) + type=nothing + docs=nothing + atomic=false + _const=false + x = x0 + while true + k = kind(x) + if k == K"Identifier" + return (name=x, type=type, atomic=atomic, _const=_const, docs=docs) + elseif k == K"::" && numchildren(x) == 2 + isnothing(type) || throw(LoweringError(x0, "multiple types in struct field")) + type = x[2] + x = x[1] + elseif k == K"atomic" + atomic = true + x = x[1] + elseif k == K"const" + _const = true + x = x[1] + elseif k == K"doc" + docs = x[1] + x = x[2] + else + return nothing + end + end +end + +function _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, inner_defs, exs) + for e in exs + if kind(e) == K"block" + _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, + inner_defs, children(e)) + elseif kind(e) == K"=" + throw(LoweringError(e, "assignment syntax in structure fields is reserved")) + else + m = _match_struct_field(e) + if !isnothing(m) + # Struct field + push!(field_names, m.name) + n = length(field_names) + push!(field_types, isnothing(m.type) ? @ast(ctx, e, "Any"::K"core") : m.type) + if m.atomic + push!(field_attrs, @ast ctx e n::K"Integer") + push!(field_attrs, @ast ctx e "atomic"::K"Symbol") + end + if m._const + push!(field_attrs, @ast ctx e n::K"Integer") + push!(field_attrs, @ast ctx e "const"::K"Symbol") + end + if !isnothing(m.docs) + push!(field_docs, @ast ctx e n::K"Integer") + push!(field_docs, @ast ctx e m.docs) + end + else + # Inner constructors and inner functions + # TODO: Disallow arbitrary expressions inside `struct`? + push!(inner_defs, e) + end + end + end +end + +# generate call to `convert()` for `(call new ...)` expressions +function _new_call_convert_arg(ctx, full_struct_type, field_type, field_index, val) + if is_core_Any(field_type) + return val + end + # kt = kind(field_type) + # TODO: Allow kt == K"Identifier" && kt in static_params to avoid fieldtype call? + @ast ctx field_type [K"block" + tmp_type := [K"call" + "fieldtype"::K"core" + full_struct_type + field_index::K"Integer" + ] + convert_for_type_decl(ctx, field_type, val, tmp_type, false) + ] +end + +function default_inner_constructors(ctx, srcref, global_struct_name, + typevar_names, typevar_stmts, field_names, field_types) + # TODO: Consider using srcref = @HERE ? + exact_ctor = if isempty(typevar_names) + # Definition with exact types for all arguments + field_decls = SyntaxList(ctx) + @ast ctx srcref [K"function" + [K"call" + [K"::" [K"curly" "Type"::K"core" global_struct_name]] + [[K"::" n t] for (n,t) in zip(field_names, field_types)]... + ] + [K"new" + global_struct_name + field_names... + ] + ] + end + maybe_non_Any_field_types = filter(!is_core_Any, field_types) + converting_ctor = if !isempty(typevar_names) || !isempty(maybe_non_Any_field_types) + # Definition which takes `Any` for all arguments and uses + # `Base.convert()` to convert those to the exact field type. Only + # defined if at least one field type is not Any. + ctor_self = new_local_binding(ctx, srcref, "#ctor-self#"; kind=:argument) + @ast ctx srcref [K"function" + [K"call" + [K"::" + ctor_self + if isempty(typevar_names) + [K"curly" "Type"::K"core" global_struct_name] + else + [K"where" + [K"curly" + "Type"::K"core" + [K"curly" + global_struct_name + typevar_names... + ] + ] + [K"_typevars" [K"block" typevar_names...] [K"block" typevar_stmts...]] + ] + end + ] + field_names... + ] + [K"block" + [K"new" + ctor_self + [_new_call_convert_arg(ctx, ctor_self, type, i, name) + for (i, (name,type)) in enumerate(zip(field_names, field_types))]... + ] + ] + ] + end + if isnothing(exact_ctor) + converting_ctor + else + if isnothing(converting_ctor) + exact_ctor + else + @ast ctx srcref [K"block" + [K"if" + # Only define converting_ctor if at least one field type is not Any. + mapfoldl(t -> [K"call" "==="::K"core" "Any"::K"core" t], + (t,u) -> [K"&&" u t], + maybe_non_Any_field_types) + [K"block"] + converting_ctor + ] + exact_ctor + ] + end + end +end + +# Generate outer constructor for structs with type parameters. Eg, for +# struct X{U,V} +# x::U +# y::V +# end +# +# We basically generate +# function (::Type{X})(x::U, y::V) where {U,V} +# new(X{U,V}, x, y) +# end +# +function default_outer_constructor(ctx, srcref, global_struct_name, + typevar_names, typevar_stmts, field_names, field_types) + @ast ctx srcref [K"function" + [K"where" + [K"call" + # We use `::Type{$global_struct_name}` here rather than just + # `struct_name` because global_struct_name is a binding to a + # type - we know we're not creating a new `Function` and + # there's no reason to emit the 1-arg `Expr(:method, name)` in + # the next phase of expansion. + [K"::" [K"curly" "Type"::K"core" global_struct_name]] + [[K"::" n t] for (n,t) in zip(field_names, field_types)]... + ] + [K"_typevars" [K"block" typevar_names...] [K"block" typevar_stmts...]] + ] + [K"new" [K"curly" global_struct_name typevar_names...] field_names...] + ] +end + +function _is_new_call(ex) + kind(ex) == K"call" && + ((kind(ex[1]) == K"Identifier" && ex[1].name_val == "new") || + (kind(ex[1]) == K"curly" && kind(ex[1][1]) == K"Identifier" && ex[1][1].name_val == "new")) +end + +# Rewrite inner constructor signatures for struct `X` from `X(...)` +# to `(ctor_self::Type{X})(...)` +function _rewrite_ctor_sig(ctx, callex, struct_name, global_struct_name, struct_typevars, ctor_self) + @assert kind(callex) == K"call" + name = callex[1] + if is_same_identifier_like(struct_name, name) + # X(x,y) ==> (#ctor-self#::Type{X})(x,y) + ctor_self[] = new_local_binding(ctx, callex, "#ctor-self#"; kind=:argument) + @ast ctx callex [K"call" + [K"::" + ctor_self[] + [K"curly" "Type"::K"core" global_struct_name] + ] + callex[2:end]... + ] + elseif kind(name) == K"curly" && is_same_identifier_like(struct_name, name[1]) + # X{T}(x,y) ==> (#ctor-self#::Type{X{T}})(x,y) + self = new_local_binding(ctx, callex, "#ctor-self#"; kind=:argument) + if numchildren(name) - 1 == length(struct_typevars) + # Self fully parameterized - can be used as the full type to + # rewrite new() calls in constructor body. + ctor_self[] = self + end + @ast ctx callex [K"call" + [K"::" + self + [K"curly" + "Type"::K"core" + [K"curly" + global_struct_name + name[2:end]... + ] + ] + ] + callex[2:end]... + ] + else + callex + end +end + +# Rewrite calls to `new` in bodies of inner constructors and inner functions +# into `new` or `splatnew` expressions. For example: +# +# struct X{T,S} +# X() = new() +# X() = new{A,B}() +# X{T,S}() where {T,S} = new() +# X{A,B}() = new() +# X{A}() = new() +# (t::Type{X})() = new{A,B}() +# f() = new() +# f() = new{A,B}() +# f() = new{Ts...}() +# end +# +# Map to the following +# +# X() = ERROR +# (#ctor-self#::Type{X})() = (new X{A,B}) +# (Type{X{T,S}}() where {T,S} = (new #ctor-self#) +# X{A,B}() = (new #ctor-self#) +# X{A}() = ERROR +# (t::Type{X})() = (new X{A,B}) +# f() = ERROR +# f() = (new X{A,B}) +# f() = (new X{Ts...}) +# +# TODO: Arguably the following "could also work", but any symbolic match of +# this case would be heuristic and rely on assuming Type == Core.Type. So +# runtime checks would really be required and flisp lowering doesn't catch +# this case either. +# +# (t::Type{X{A,B}})() = new() +function _rewrite_ctor_new_calls(ctx, ex, struct_name, global_struct_name, ctor_self, + struct_typevars, field_types) + if is_leaf(ex) + return ex + elseif !_is_new_call(ex) + return mapchildren( + e->_rewrite_ctor_new_calls(ctx, e, struct_name, global_struct_name, + ctor_self, struct_typevars, field_types), + ctx, ex + ) + end + # Rewrite a call to new() + kw_arg_i = findfirst(e->(k = kind(e); k == K"=" || k == K"parameters"), children(ex)) + if !isnothing(kw_arg_i) + throw(LoweringError(ex[kw_arg_i], "`new` does not accept keyword arguments")) + end + full_struct_type = if kind(ex[1]) == K"curly" + # new{A,B}(...) + new_type_params = ex[1][2:end] + n_type_splat = sum(kind(t) == K"..." for t in new_type_params) + n_type_nonsplat = length(new_type_params) - n_type_splat + if n_type_splat == 0 && n_type_nonsplat < length(struct_typevars) + throw(LoweringError(ex[1], "too few type parameters specified in `new{...}`")) + elseif n_type_nonsplat > length(struct_typevars) + throw(LoweringError(ex[1], "too many type parameters specified in `new{...}`")) + end + @ast ctx ex[1] [K"curly" global_struct_name new_type_params...] + elseif !isnothing(ctor_self) + # new(...) in constructors + ctor_self + else + # new(...) inside non-constructor inner functions + if isempty(struct_typevars) + global_struct_name + else + throw(LoweringError(ex[1], "too few type parameters specified in `new`")) + end + end + new_args = ex[2:end] + n_splat = sum(kind(t) == K"..." for t in new_args) + n_nonsplat = length(new_args) - n_splat + n_fields = length(field_types) + function throw_n_fields_error(desc) + @ast ctx ex [K"call" + "throw"::K"core" + [K"call" + "ArgumentError"::K"top" + "too $desc arguments in `new` (expected $n_fields)"::K"String" + ] + ] + end + if n_nonsplat > n_fields + return throw_n_fields_error("many") + else + # "Too few" args are allowed in partially initialized structs + end + if n_splat == 0 + @ast ctx ex [K"block" + struct_type := full_struct_type + [K"new" + struct_type + [_new_call_convert_arg(ctx, struct_type, type, i, name) + for (i, (name,type)) in enumerate(zip(ex[2:end], field_types))]... + ] + ] + else + fields_all_Any = all(is_core_Any, field_types) + if fields_all_Any + @ast ctx ex [K"block" + struct_type := full_struct_type + [K"splatnew" + struct_type + # Note: `jl_new_structt` ensures length of this tuple is + # exactly the number of fields. + [K"call" "tuple"::K"core" ex[2:end]...] + ] + ] + else + # `new` with splatted args which are symbolically not `Core.Any` + # (might be `Any` at runtime but we can't know that here.) + @ast ctx ex [K"block" + args := [K"call" "tuple"::K"core" ex[2:end]...] + n_args := [K"call" "nfields"::K"core" args] + [K"if" + [K"call" "ult_int"::K"top" n_args n_fields::K"Integer"] + throw_n_fields_error("few") + ] + [K"if" + [K"call" "ult_int"::K"top" n_fields::K"Integer" n_args] + throw_n_fields_error("many") + ] + struct_type := full_struct_type + [K"new" + struct_type + [_new_call_convert_arg(ctx, struct_type, type, i, + [K"call" "getfield"::K"core" args i::K"Integer"]) + for (i, type) in enumerate(field_types)]... + ] + ] + end + end +end + +# Rewrite calls to `new( ... )` to `new` expressions on the appropriate +# type, determined by the containing type and constructor definitions. +# +# This is mainly for constructors, but also needs to work for inner functions +# which may call new() but are not constructors. +function rewrite_new_calls(ctx, ex, struct_name, global_struct_name, + typevar_names, field_names, field_types) + if kind(ex) == K"doc" + docs = ex[1] + ex = ex[2] + else + docs = nothing + end + if kind(ex) != K"function" + return ex + end + if !(numchildren(ex) == 2 && is_eventually_call(ex[1])) + throw(LoweringError(ex, "Expected constructor or named inner function")) + end + + ctor_self = Ref{Union{Nothing,SyntaxTree}}(nothing) + expand_function_def(ctx, ex, docs, + callex->_rewrite_ctor_sig(ctx, callex, struct_name, + global_struct_name, typevar_names, ctor_self), + body->_rewrite_ctor_new_calls(ctx, body, struct_name, global_struct_name, + ctor_self[], typevar_names, field_types) + ) +end + +function _constructor_min_initalized(ex::SyntaxTree) + if _is_new_call(ex) + if any(kind(e) == K"..." for e in ex[2:end]) + # Lowering ensures new with splats always inits all fields + # or in the case of splatnew this is enforced by the runtime. + typemax(Int) + else + numchildren(ex) - 1 + end + elseif !is_leaf(ex) + minimum((_constructor_min_initalized(e) for e in children(ex)), init=typemax(Int)) + else + typemax(Int) + end +end + +# Let S be a struct we're defining in module M. Below is a hack to allow its +# field types to refer to S as M.S. See #56497. +function insert_struct_shim(ctx, fieldtypes, name) + function replace_type(ex) + if kind(ex) == K"." && + numchildren(ex) == 2 && + kind(ex[2]) == K"Symbol" && + ex[2].name_val == name.name_val + @ast ctx ex [K"call" "struct_name_shim"::K"core" ex[1] ex[2] ctx.mod::K"Value" name] + elseif numchildren(ex) > 0 + @ast ctx ex [ex.kind map(replace_type, children(ex))...] + else + ex + end + end + map(replace_type, fieldtypes) +end + +function expand_struct_def(ctx, ex, docs) + @chk numchildren(ex) == 2 + type_sig = ex[1] + type_body = ex[2] + if kind(type_body) != K"block" + throw(LoweringError(type_body, "expected block for `struct` fields")) + end + struct_name, type_params, supertype = analyze_type_sig(ctx, type_sig) + typevar_names, typevar_stmts = expand_typevars(ctx, type_params) + field_names = SyntaxList(ctx) + field_types = SyntaxList(ctx) + field_attrs = SyntaxList(ctx) + field_docs = SyntaxList(ctx) + inner_defs = SyntaxList(ctx) + _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, + inner_defs, children(type_body)) + is_mutable = has_flags(ex, JuliaSyntax.MUTABLE_FLAG) + min_initialized = minimum((_constructor_min_initalized(e) for e in inner_defs), + init=length(field_names)) + newtype_var = ssavar(ctx, ex, "struct_type") + hasprev = ssavar(ctx, ex, "hasprev") + prev = ssavar(ctx, ex, "prev") + newdef = ssavar(ctx, ex, "newdef") + layer = new_scope_layer(ctx, struct_name) + global_struct_name = adopt_scope(struct_name, layer) + if !isempty(typevar_names) + # Generate expression like `prev_struct.body.body.parameters` + prev_typevars = global_struct_name + for _ in 1:length(typevar_names) + prev_typevars = @ast ctx type_sig [K"." prev_typevars "body"::K"Symbol"] + end + prev_typevars = @ast ctx type_sig [K"." prev_typevars "parameters"::K"Symbol"] + end + + # New local variable names for constructor args to avoid clashing with any + # type names + if isempty(inner_defs) + field_names_2 = adopt_scope(field_names, layer) + end + + need_outer_constructor = false + if isempty(inner_defs) && !isempty(typevar_names) + # To generate an outer constructor each struct type parameter must be + # able to be inferred from the list of fields passed as constuctor + # arguments. + # + # More precisely, it must occur in a field type, or in the bounds of a + # subsequent type parameter. For example the following won't work + # struct X{T} + # a::Int + # end + # X(a::Int) where T = #... construct X{T} ?? + # + # But the following does + # struct X{T} + # a::T + # end + # X(a::T) where {T} = # construct X{typeof(a)}(a) + need_outer_constructor = true + for i in 1:length(typevar_names) + typevar_name = typevar_names[i] + typevar_in_fields = any(contains_identifier(ft, typevar_name) for ft in field_types) + if !typevar_in_fields + typevar_in_bounds = any(type_params[i+1:end]) do param + # Check the bounds of subsequent type params + (_,lb,ub) = analyze_typevar(ctx, param) + # todo: flisp lowering tests `lb` here so we also do. But + # in practice this doesn't seem to constrain `typevar_name` + # and the generated constructor doesn't work? + (!isnothing(ub) && contains_identifier(ub, typevar_name)) || + (!isnothing(lb) && contains_identifier(lb, typevar_name)) + end + if !typevar_in_bounds + need_outer_constructor = false + break + end + end + end + end + + # The following lowering covers several subtle issues in the ordering of + # typevars when "redefining" structs. + # See https://github.com/JuliaLang/julia/pull/36121 + @ast ctx ex [K"block" + [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex] ] + [K"scope_block"(scope_type=:hard) + # Needed for later constdecl to work, though plain global form may be removed soon. + [K"global" global_struct_name] + [K"block" + [K"local" struct_name] + [K"always_defined" struct_name] + typevar_stmts... + [K"=" + newtype_var + [K"call" + "_structtype"::K"core" + ctx.mod::K"Value" + struct_name=>K"Symbol" + [K"call"(type_sig) "svec"::K"core" typevar_names...] + [K"call"(type_body) "svec"::K"core" [n=>K"Symbol" for n in field_names]...] + [K"call"(type_body) "svec"::K"core" field_attrs...] + is_mutable::K"Bool" + min_initialized::K"Integer" + ] + ] + [K"=" struct_name newtype_var] + [K"call"(supertype) "_setsuper!"::K"core" newtype_var supertype] + [K"=" hasprev + [K"&&" [K"call" "isdefinedglobal"::K"core" + ctx.mod::K"Value" + struct_name=>K"Symbol" + false::K"Bool"] + [K"call" "_equiv_typedef"::K"core" global_struct_name newtype_var] + ]] + [K"=" prev [K"if" hasprev global_struct_name false::K"Bool"]] + [K"if" hasprev + [K"block" + # if this is compatible with an old definition, use the old parameters, but the + # new object. This will fail to capture recursive cases, but the call to typebody! + # below is permitted to choose either type definition to put into the binding table + if !isempty(typevar_names) + # And resassign the typevar_names - these may be + # referenced in the definition of the field + # types below + [K"=" [K"tuple" typevar_names...] prev_typevars] + end + ] + ] + [K"=" newdef + [K"call"(type_body) + "_typebody!"::K"core" + prev + newtype_var + [K"call" "svec"::K"core" insert_struct_shim(ctx, field_types, struct_name)...] + ]] + [K"constdecl" + global_struct_name + newdef + ] + [K"latestworld"] + # Default constructors + if isempty(inner_defs) + default_inner_constructors(ctx, ex, global_struct_name, + typevar_names, typevar_stmts, field_names_2, field_types) + else + map!(inner_defs, inner_defs) do def + rewrite_new_calls(ctx, def, struct_name, global_struct_name, + typevar_names, field_names, field_types) + end + [K"block" inner_defs...] + end + if need_outer_constructor + default_outer_constructor(ctx, ex, global_struct_name, + typevar_names, typevar_stmts, field_names_2, field_types) + end + ] + ] + + # Documentation + if !isnothing(docs) || !isempty(field_docs) + [K"call"(isnothing(docs) ? ex : docs) + bind_docs!::K"Value" + struct_name + isnothing(docs) ? nothing_(ctx, ex) : docs[1] + ::K"SourceLocation"(ex) + [K"=" + "field_docs"::K"Identifier" + [K"call" "svec"::K"core" field_docs...] + ] + ] + end + nothing_(ctx, ex) + ] +end + +#------------------------------------------------------------------------------- +# Expand `where` syntax + +function expand_where(ctx, srcref, lhs, rhs) + bounds = analyze_typevar(ctx, rhs) + v = bounds[1] + @ast ctx srcref [K"let" + [K"block" [K"=" v bounds_to_TypeVar(ctx, srcref, bounds)]] + [K"call" "UnionAll"::K"core" v lhs] + ] +end + +function expand_wheres(ctx, ex) + body = ex[1] + rhs = ex[2] + if kind(rhs) == K"braces" + # S{X,Y} where {X,Y} + for r in reverse(children(rhs)) + body = expand_where(ctx, ex, body, r) + end + elseif kind(rhs) == K"_typevars" + # Eg, `S{X,Y} where {X, Y}` but with X and Y + # already allocated `TypeVar`s + for r in reverse(children(rhs[1])) + body = @ast ctx ex [K"call" "UnionAll"::K"core" r body] + end + else + # S{X} where X + body = expand_where(ctx, ex, body, rhs) + end + body +end + +# Match implicit where parameters for `Foo{<:Bar}` ==> `Foo{T} where T<:Bar` +function expand_curly(ctx, ex) + @assert kind(ex) == K"curly" + check_no_parameters(ex, "unexpected semicolon in type parameter list") + check_no_assignment(children(ex), "misplace assignment in type parameter list") + + typevar_stmts = SyntaxList(ctx) + type_args = SyntaxList(ctx) + implicit_typevars = SyntaxList(ctx) + + i = 1 + for e in children(ex) + k = kind(e) + if (k == K"<:" || k == K">:") && numchildren(e) == 1 + # `X{<:A}` and `X{>:A}` + name = @ast ctx e "#T$i"::K"Placeholder" + i += 1 + typevar = k == K"<:" ? + bounds_to_TypeVar(ctx, e, (name, nothing, e[1])) : + bounds_to_TypeVar(ctx, e, (name, e[1], nothing)) + arg = emit_assign_tmp(typevar_stmts, ctx, typevar) + push!(implicit_typevars, arg) + else + arg = e + end + push!(type_args, arg) + end + + type = @ast ctx ex [K"call" "apply_type"::K"core" type_args...] + if !isempty(implicit_typevars) + type = @ast ctx ex [K"block" + typevar_stmts... + [K"where" type [K"_typevars" [K"block" implicit_typevars...] [K"block" typevar_stmts...]]] + ] + end + + return type +end + +#------------------------------------------------------------------------------- +# Expand import / using / export + +function _append_importpath(ctx, path_spec, path) + prev_was_dot = true + for component in children(path) + k = kind(component) + if k == K"quote" + # Permit quoted path components as in + # import A.(:b).:c + component = component[1] + end + @chk kind(component) in (K"Identifier", K".") + name = component.name_val + is_dot = kind(component) == K"." + if is_dot && !prev_was_dot + throw(LoweringError(component, "invalid import path: `.` in identifier path")) + end + prev_was_dot = is_dot + push!(path_spec, @ast(ctx, component, name::K"String")) + end + path_spec +end + +function expand_import(ctx, ex) + is_using = kind(ex) == K"using" + if kind(ex[1]) == K":" + # import M: x.y as z, w + # (import (: (importpath M) (as (importpath x y) z) (importpath w))) + # => + # (call module_import + # false + # (call core.svec "M") + # (call core.svec 2 "x" "y" "z" 1 "w" "w")) + @chk numchildren(ex[1]) >= 2 + from = ex[1][1] + @chk kind(from) == K"importpath" + from_path = @ast ctx from [K"call" + "svec"::K"core" + _append_importpath(ctx, SyntaxList(ctx), from)... + ] + paths = ex[1][2:end] + else + # import A.B + # (using (importpath A B)) + # (call module_import true nothing (call core.svec 1 "w")) + @chk numchildren(ex) >= 1 + from_path = nothing_(ctx, ex) + paths = children(ex) + end + path_spec = SyntaxList(ctx) + for path in paths + as_name = nothing + if kind(path) == K"as" + @chk numchildren(path) == 2 + as_name = path[2] + @chk kind(as_name) == K"Identifier" + path = path[1] + end + @chk kind(path) == K"importpath" + push!(path_spec, @ast(ctx, path, numchildren(path)::K"Integer")) + _append_importpath(ctx, path_spec, path) + push!(path_spec, isnothing(as_name) ? nothing_(ctx, ex) : + @ast(ctx, as_name, as_name.name_val::K"String")) + end + @ast ctx ex [K"block" + [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]] + [K"call" + module_import ::K"Value" + ctx.mod ::K"Value" + is_using ::K"Value" + from_path + [K"call" + "svec"::K"core" + path_spec... + ] + ] + ] +end + +# Expand `public` or `export` +function expand_public(ctx, ex) + @ast ctx ex [K"call" + module_public::K"Value" + ctx.mod::K"Value" + (kind(ex) == K"export")::K"Bool" + (e.name_val::K"String" for e in children(ex))... + ] +end + +#------------------------------------------------------------------------------- +# Expand module definitions + +function expand_module(ctx, ex::SyntaxTree) + modname_ex = ex[1] + @chk kind(modname_ex) == K"Identifier" + modname = modname_ex.name_val + + std_defs = if !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG) + @ast ctx (@HERE) [ + K"block" + [K"using"(@HERE) + [K"importpath" + "Base" ::K"Identifier" + ] + ] + [K"function"(@HERE) + [K"call" + "eval" ::K"Identifier" + "x" ::K"Identifier" + ] + [K"call" + "eval" ::K"core" + modname ::K"Identifier" + "x" ::K"Identifier" + ] + ] + [K"function"(@HERE) + [K"call" + "include" ::K"Identifier" + "x" ::K"Identifier" + ] + [K"call" + "_call_latest" ::K"core" + "include" ::K"top" + modname ::K"Identifier" + "x" ::K"Identifier" + ] + ] + [K"function"(@HERE) + [K"call" + "include" ::K"Identifier" + [K"::" + "mapexpr" ::K"Identifier" + "Function" ::K"top" + ] + "x" ::K"Identifier" + ] + [K"call" + "_call_latest" ::K"core" + "include" ::K"top" + "mapexpr" ::K"Identifier" + modname ::K"Identifier" + "x" ::K"Identifier" + ] + ] + ] + end + + body = ex[2] + @chk kind(body) == K"block" + + @ast ctx ex [K"block" + [K"assert" + "global_toplevel_only"::K"Symbol" + [K"inert" ex] + ] + [K"call" + eval_module ::K"Value" + ctx.mod ::K"Value" + modname ::K"String" + [K"inert"(body) + [K"toplevel" + std_defs + children(body)... + ] + ] + ] + ] +end + +#------------------------------------------------------------------------------- +# Desugaring's "big switch": expansion of some simple forms; dispatch to other +# expansion functions for the rest. + +""" +Lowering pass 2 - desugaring + +This pass simplifies expressions by expanding complicated syntax sugar into a +small set of core syntactic forms. For example, field access syntax `a.b` is +expanded to a function call `getproperty(a, :b)`. +""" +function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) + k = kind(ex) + if k == K"atomic" + throw(LoweringError(ex, "unimplemented or unsupported atomic declaration")) + elseif k == K"call" + expand_call(ctx, ex) + elseif k == K"dotcall" || ((k == K"&&" || k == K"||") && is_dotted(ex)) + expand_forms_2(ctx, expand_fuse_broadcast(ctx, ex)) + elseif k == K"." + expand_forms_2(ctx, expand_dot(ctx, ex)) + elseif k == K"?" + @chk numchildren(ex) == 3 + expand_forms_2(ctx, @ast ctx ex [K"if" children(ex)...]) + elseif k == K"&&" || k == K"||" + @chk numchildren(ex) > 1 + cs = expand_cond_children(ctx, ex) + # Attributing correct provenance for `cs[1:end-1]` is tricky in cases + # like `a && (b && c)` because the expression constructed here arises + # from the source fragment `a && (b` which doesn't follow the tree + # structure. For now we attribute to the parent node. + cond = length(cs) == 2 ? + cs[1] : + makenode(ctx, ex, k, cs[1:end-1]) + # This transformation assumes the type assertion `cond::Bool` will be + # added by a later compiler pass (currently done in codegen) + if k == K"&&" + @ast ctx ex [K"if" cond cs[end] false::K"Bool"] + else + @ast ctx ex [K"if" cond true::K"Bool" cs[end]] + end + elseif k == K"::" + @chk numchildren(ex) == 2 "`::` must be written `value::type` outside function argument lists" + @ast ctx ex [K"call" + "typeassert"::K"core" + expand_forms_2(ctx, ex[1]) + expand_forms_2(ctx, ex[2]) + ] + elseif k == K"<:" || k == K">:" || k == K"-->" + expand_forms_2(ctx, @ast ctx ex [K"call" + adopt_scope(string(k)::K"Identifier", ex) + children(ex)... + ]) + elseif k == K"op=" + expand_forms_2(ctx, expand_update_operator(ctx, ex)) + elseif k == K"=" + if is_dotted(ex) + expand_forms_2(ctx, expand_fuse_broadcast(ctx, ex)) + else + expand_assignment(ctx, ex) + end + elseif k == K"break" + numchildren(ex) > 0 ? ex : + @ast ctx ex [K"break" "loop_exit"::K"symbolic_label"] + elseif k == K"continue" + @ast ctx ex [K"break" "loop_cont"::K"symbolic_label"] + elseif k == K"comparison" + expand_forms_2(ctx, expand_compare_chain(ctx, ex)) + elseif k == K"doc" + @chk numchildren(ex) == 2 + sig = expand_forms_2(ctx, ex[2], ex) + elseif k == K"for" + expand_forms_2(ctx, expand_for(ctx, ex)) + elseif k == K"comprehension" + @chk numchildren(ex) == 1 + @chk kind(ex[1]) == K"generator" + @ast ctx ex [K"call" + "collect"::K"top" + expand_forms_2(ctx, ex[1]) + ] + elseif k == K"typed_comprehension" + @chk numchildren(ex) == 2 + @chk kind(ex[2]) == K"generator" + if numchildren(ex[2]) == 2 && kind(ex[2][2]) == K"iteration" + # Hack to lower simple typed comprehensions to loops very early, + # greatly reducing the number of functions and load on the compiler + expand_forms_2(ctx, expand_comprehension_to_loops(ctx, ex)) + else + @ast ctx ex [K"call" + "collect"::K"top" + expand_forms_2(ctx, ex[1]) + expand_forms_2(ctx, ex[2]) + ] + end + elseif k == K"generator" + expand_forms_2(ctx, expand_generator(ctx, ex)) + elseif k == K"->" || k == K"do" + expand_forms_2(ctx, expand_arrow(ctx, ex)) + elseif k == K"function" + expand_forms_2(ctx, expand_function_def(ctx, ex, docs)) + elseif k == K"macro" + @ast ctx ex [K"block" + [K"assert" + "global_toplevel_only"::K"Symbol" + [K"inert" ex] + ] + expand_forms_2(ctx, expand_macro_def(ctx, ex)) + ] + elseif k == K"if" || k == K"elseif" + @chk numchildren(ex) >= 2 + @ast ctx ex [k + expand_condition(ctx, ex[1]) + expand_forms_2(ctx, ex[2:end])... + ] + elseif k == K"let" + expand_forms_2(ctx, expand_let(ctx, ex)) + elseif k == K"const" + expand_const_decl(ctx, ex) + elseif k == K"local" || k == K"global" + if numchildren(ex) == 1 && kind(ex[1]) == K"Identifier" + # Don't recurse when already simplified - `local x`, etc + ex + else + expand_forms_2(ctx, expand_decls(ctx, ex)) + end + elseif k == K"where" + expand_forms_2(ctx, expand_wheres(ctx, ex)) + elseif k == K"braces" || k == K"bracescat" + throw(LoweringError(ex, "{ } syntax is reserved for future use")) + elseif k == K"string" + if numchildren(ex) == 1 && kind(ex[1]) == K"String" + ex[1] + else + @ast ctx ex [K"call" + "string"::K"top" + expand_forms_2(ctx, children(ex))... + ] + end + elseif k == K"try" + expand_forms_2(ctx, expand_try(ctx, ex)) + elseif k == K"tuple" + if has_parameters(ex) + if numchildren(ex) > 1 + throw(LoweringError(ex[end], "unexpected semicolon in tuple - use `,` to separate tuple elements")) + end + expand_forms_2(ctx, expand_named_tuple(ctx, ex, children(ex[1]))) + elseif any_assignment(children(ex)) + expand_forms_2(ctx, expand_named_tuple(ctx, ex, children(ex))) + else + expand_forms_2(ctx, @ast ctx ex [K"call" + "tuple"::K"core" + children(ex)... + ]) + end + elseif k == K"$" + throw(LoweringError(ex, "`\$` expression outside string or quote block")) + elseif k == K"module" + expand_module(ctx, ex) + elseif k == K"import" || k == K"using" + expand_import(ctx, ex) + elseif k == K"export" || k == K"public" + expand_public(ctx, ex) + elseif k == K"abstract" || k == K"primitive" + expand_forms_2(ctx, expand_abstract_or_primitive_type(ctx, ex)) + elseif k == K"struct" + expand_forms_2(ctx, expand_struct_def(ctx, ex, docs)) + elseif k == K"ref" + sctx = with_stmts(ctx) + (arr, idxs) = expand_ref_components(sctx, ex) + expand_forms_2(ctx, + @ast ctx ex [K"block" + sctx.stmts... + [K"call" + "getindex"::K"top" + arr + idxs... + ] + ] + ) + elseif k == K"curly" + expand_forms_2(ctx, expand_curly(ctx, ex)) + elseif k == K"toplevel" + # The toplevel form can't be lowered here - it needs to just be quoted + # and passed through to a call to eval. + @ast ctx ex [K"block" + [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]] + [K"call" + eval ::K"Value" + ctx.mod ::K"Value" + [K"inert" ex] + ] + ] + elseif k == K"vect" + check_no_parameters(ex, "unexpected semicolon in array expression") + check_no_assignment(children(ex)) + @ast ctx ex [K"call" + "vect"::K"top" + expand_forms_2(ctx, children(ex))... + ] + elseif k == K"hcat" + check_no_assignment(children(ex)) + @ast ctx ex [K"call" + "hcat"::K"top" + expand_forms_2(ctx, children(ex))... + ] + elseif k == K"typed_hcat" + check_no_assignment(children(ex)) + @ast ctx ex [K"call" + "typed_hcat"::K"top" + expand_forms_2(ctx, children(ex))... + ] + elseif k == K"opaque_closure" + expand_forms_2(ctx, expand_opaque_closure(ctx, ex)) + elseif k == K"vcat" || k == K"typed_vcat" + expand_forms_2(ctx, expand_vcat(ctx, ex)) + elseif k == K"ncat" || k == K"typed_ncat" + expand_forms_2(ctx, expand_ncat(ctx, ex)) + elseif k == K"while" + @chk numchildren(ex) == 2 + @ast ctx ex [K"break_block" "loop_exit"::K"symbolic_label" + [K"_while" + expand_condition(ctx, ex[1]) + [K"break_block" "loop_cont"::K"symbolic_label" + [K"scope_block"(scope_type=:neutral) + expand_forms_2(ctx, ex[2]) + ] + ] + ] + ] + elseif k == K"inert" + ex + elseif k == K"&" + throw(LoweringError(ex, "invalid syntax")) + elseif k == K"$" + throw(LoweringError(ex, "`\$` expression outside string or quote")) + elseif k == K"..." + throw(LoweringError(ex, "`...` expression outside call")) + elseif is_leaf(ex) + ex + elseif k == K"return" + if numchildren(ex) == 0 + @ast ctx ex [K"return" "nothing"::K"core"] + elseif numchildren(ex) == 1 + mapchildren(e->expand_forms_2(ctx,e), ctx, ex) + else + throw(LoweringError(ex, "More than one argument to return")) + end + else + mapchildren(e->expand_forms_2(ctx,e), ctx, ex) + end +end + +function expand_forms_2(ctx::DesugaringContext, exs::Union{Tuple,AbstractVector}) + res = SyntaxList(ctx) + for e in exs + push!(res, expand_forms_2(ctx, e)) + end + res +end + +function expand_forms_2(ctx::StatementListCtx, args...) + expand_forms_2(ctx.ctx, args...) +end + +function expand_forms_2(ctx::MacroExpansionContext, ex::SyntaxTree) + ctx1 = DesugaringContext(ctx) + ex1 = expand_forms_2(ctx1, reparent(ctx1, ex)) + ctx1, ex1 +end + diff --git a/src/vendored/JuliaLowering/src/eval.jl b/src/vendored/JuliaLowering/src/eval.jl new file mode 100644 index 00000000..ccf2ac6a --- /dev/null +++ b/src/vendored/JuliaLowering/src/eval.jl @@ -0,0 +1,372 @@ +function lower(mod::Module, ex0) + ctx1, ex1 = expand_forms_1( mod, ex0) + ctx2, ex2 = expand_forms_2( ctx1, ex1) + ctx3, ex3 = resolve_scopes( ctx2, ex2) + ctx4, ex4 = convert_closures(ctx3, ex3) + ctx5, ex5 = linearize_ir( ctx4, ex4) + ex5 +end + +function macroexpand(mod::Module, ex) + ctx1, ex1 = expand_forms_1(mod, ex) + ex1 +end + +_CodeInfo_need_ver = v"1.12.0-DEV.512" +if VERSION < _CodeInfo_need_ver + function _CodeInfo(args...) + error("Constructing a CodeInfo currently requires Julia version $_CodeInfo_need_ver or greater") + end +else + # debuginfo changed completely as of https://github.com/JuliaLang/julia/pull/52415 + # nargs / isva was added as of https://github.com/JuliaLang/julia/pull/54341 + # field rettype added in https://github.com/JuliaLang/julia/pull/54655 + # field has_image_globalref added in https://github.com/JuliaLang/julia/pull/57433 + # CodeInfo constructor. TODO: Should be in Core + let + fns = fieldnames(Core.CodeInfo) + fts = fieldtypes(Core.CodeInfo) + conversions = [:(convert($t, $n)) for (t,n) in zip(fts, fns)] + + expected_fns = (:code, :debuginfo, :ssavaluetypes, :ssaflags, :slotnames, :slotflags, :slottypes, :rettype, :parent, :edges, :min_world, :max_world, :method_for_inference_limit_heuristics, :nargs, :propagate_inbounds, :has_fcall, :has_image_globalref, :nospecializeinfer, :isva, :inlining, :constprop, :purity, :inlining_cost) + expected_fts = (Vector{Any}, Core.DebugInfo, Any, Vector{UInt32}, Vector{Symbol}, Vector{UInt8}, Any, Any, Any, Any, UInt64, UInt64, Any, UInt64, Bool, Bool, Bool, Bool, Bool, UInt8, UInt8, UInt16, UInt16) + + code = if fns != expected_fns + unexpected_fns = collect(setdiff(Set(fns), Set(expected_fns))) + missing_fns = collect(setdiff(Set(expected_fns), Set(fns))) + :(function _CodeInfo(args...) + error("Unrecognized CodeInfo fields: Maybe version $VERSION is too new for this version of JuliaLowering?" + * isempty(unexpected_fns) ? "" : "\nUnexpected fields found: $($unexpected_fns)" + * isempty(missing_fns) ? "" : "\nMissing fields: $($missing_fns)") + end) + elseif fts != expected_fts + :(function _CodeInfo(args...) + error("Unrecognized CodeInfo field types: Maybe version $VERSION is too new for this version of JuliaLowering?") + end) + else + :(function _CodeInfo($(fns...)) + $(Expr(:new, :(Core.CodeInfo), conversions...)) + end) + end + + eval(@__MODULE__, code) + end +end + +function _compress_debuginfo(info) + filename, edges, codelocs = info + edges = Core.svec(map(_compress_debuginfo, edges)...) + codelocs = @ccall jl_compress_codelocs((-1)::Int32, codelocs::Any, + div(length(codelocs),3)::Csize_t)::String + Core.DebugInfo(Symbol(filename), nothing, edges, codelocs) +end + +function ir_debug_info_state(ex) + e1 = first(flattened_provenance(ex)) + topfile = filename(e1) + [(topfile, [], Vector{Int32}())] +end + +function add_ir_debug_info!(current_codelocs_stack, stmt) + locstk = [(filename(e), source_location(e)[1]) for e in flattened_provenance(stmt)] + for j in 1:max(length(locstk), length(current_codelocs_stack)) + if j > length(locstk) || (length(current_codelocs_stack) >= j && + current_codelocs_stack[j][1] != locstk[j][1]) + while length(current_codelocs_stack) >= j + info = pop!(current_codelocs_stack) + push!(last(current_codelocs_stack)[2], info) + end + end + if j > length(locstk) + break + elseif j > length(current_codelocs_stack) + push!(current_codelocs_stack, (locstk[j][1], [], Vector{Int32}())) + end + end + for (j, (file,line)) in enumerate(locstk) + fn, edges, codelocs = current_codelocs_stack[j] + @assert fn == file + if j < length(locstk) + edge_index = length(edges) + 1 + edge_codeloc_index = fld1(length(current_codelocs_stack[j+1][3]) + 1, 3) + else + edge_index = 0 + edge_codeloc_index = 0 + end + push!(codelocs, line) + push!(codelocs, edge_index) + push!(codelocs, edge_codeloc_index) + end +end + +function finish_ir_debug_info!(current_codelocs_stack) + while length(current_codelocs_stack) > 1 + info = pop!(current_codelocs_stack) + push!(last(current_codelocs_stack)[2], info) + end + + _compress_debuginfo(only(current_codelocs_stack)) +end + +# Convert SyntaxTree to the CodeInfo+Expr data stuctures understood by the +# Julia runtime +function to_code_info(ex, mod, funcname, slots) + input_code = children(ex) + stmts = Any[] + + current_codelocs_stack = ir_debug_info_state(ex) + + nargs = sum((s.kind==:argument for s in slots), init=0) + slotnames = Vector{Symbol}(undef, length(slots)) + slot_rename_inds = Dict{String,Int}() + slotflags = Vector{UInt8}(undef, length(slots)) + for (i, slot) in enumerate(slots) + name = slot.name + # TODO: Do we actually want unique names here? The C code in + # `jl_new_code_info_from_ir` has logic to simplify gensym'd names and + # use the empty string for compiler-generated bindings. + ni = get(slot_rename_inds, name, 0) + slot_rename_inds[name] = ni + 1 + if ni > 0 + name = "$name@$ni" + end + sname = Symbol(name) + slotnames[i] = sname + slotflags[i] = # Inference | Codegen + slot.is_read << 3 | # SLOT_USED | jl_vinfo_sa + slot.is_single_assign << 4 | # SLOT_ASSIGNEDONCE | - + slot.is_maybe_undef << 5 | # SLOT_USEDUNDEF | jl_vinfo_usedundef + slot.is_called << 6 # SLOT_CALLED | - + if slot.is_nospecialize + # Ideally this should be a slot flag instead + add_ir_debug_info!(current_codelocs_stack, ex) + push!(stmts, Expr(:meta, :nospecialize, Core.SlotNumber(i))) + end + end + + ssa_offset = length(stmts) + for stmt in children(ex) + push!(stmts, to_lowered_expr(mod, stmt, ssa_offset)) + add_ir_debug_info!(current_codelocs_stack, stmt) + end + + debuginfo = finish_ir_debug_info!(current_codelocs_stack) + + # TODO: Set ssaflags based on call site annotations: + # - @inbounds annotations + # - call site @inline / @noinline + # - call site @assume_effects + ssaflags = zeros(UInt32, length(stmts)) + + # TODO: Set true for @propagate_inbounds + propagate_inbounds = false + # TODO: Set true if there's a foreigncall + has_fcall = false + # TODO: Set for @nospecializeinfer + nospecializeinfer = false + # TODO: Set based on @inline -> 0x01 or @noinline -> 0x02 + inlining = 0x00 + # TODO: Set based on @constprop :aggressive -> 0x01 or @constprop :none -> 0x02 + constprop = 0x00 + # TODO: Set based on Base.@assume_effects + purity = 0x0000 + + # TODO: Should we set these? + rettype = Any + has_image_globalref = false + + # The following CodeInfo fields always get their default values for + # uninferred code. + ssavaluetypes = length(stmts) # Why does the runtime code do this? + slottypes = nothing + parent = nothing + method_for_inference_limit_heuristics = nothing + edges = nothing + min_world = Csize_t(1) + max_world = typemax(Csize_t) + isva = false + inlining_cost = 0xffff + + _CodeInfo( + stmts, + debuginfo, + ssavaluetypes, + ssaflags, + slotnames, + slotflags, + slottypes, + rettype, + parent, + edges, + min_world, + max_world, + method_for_inference_limit_heuristics, + nargs, + propagate_inbounds, + has_fcall, + has_image_globalref, + nospecializeinfer, + isva, + inlining, + constprop, + purity, + inlining_cost + ) +end + +function to_lowered_expr(mod, ex, ssa_offset=0) + k = kind(ex) + if is_literal(k) + ex.value + elseif k == K"core" + name = ex.name_val + if name == "cglobal" + # cglobal isn't a true name within core - instead it's a builtin + :cglobal + else + GlobalRef(Core, Symbol(name)) + end + elseif k == K"top" + GlobalRef(Base, Symbol(ex.name_val)) + elseif k == K"globalref" + GlobalRef(ex.mod, Symbol(ex.name_val)) + elseif k == K"Identifier" + # Implicitly refers to name in parent module + # TODO: Should we even have plain identifiers at this point or should + # they all effectively be resolved into GlobalRef earlier? + Symbol(ex.name_val) + elseif k == K"SourceLocation" + QuoteNode(source_location(LineNumberNode, ex)) + elseif k == K"Symbol" + QuoteNode(Symbol(ex.name_val)) + elseif k == K"slot" + Core.SlotNumber(ex.var_id) + elseif k == K"static_parameter" + Expr(:static_parameter, ex.var_id) + elseif k == K"SSAValue" + Core.SSAValue(ex.var_id + ssa_offset) + elseif k == K"return" + Core.ReturnNode(to_lowered_expr(mod, ex[1], ssa_offset)) + elseif k == K"inert" + ex[1] + elseif k == K"code_info" + funcname = ex.is_toplevel_thunk ? + "top-level scope" : + "none" # FIXME + ir = to_code_info(ex[1], mod, funcname, ex.slots) + if ex.is_toplevel_thunk + Expr(:thunk, ir) + else + ir + end + elseif k == K"Value" + ex.value + elseif k == K"goto" + Core.GotoNode(ex[1].id) + elseif k == K"gotoifnot" + Core.GotoIfNot(to_lowered_expr(mod, ex[1], ssa_offset), ex[2].id) + elseif k == K"enter" + catch_idx = ex[1].id + numchildren(ex) == 1 ? + Core.EnterNode(catch_idx) : + Core.EnterNode(catch_idx, to_lowered_expr(mod, ex[2], ssa_offset)) + elseif k == K"method" + cs = map(e->to_lowered_expr(mod, e, ssa_offset), children(ex)) + # Ad-hoc unwrapping to satisfy `Expr(:method)` expectations + c1 = cs[1] isa QuoteNode ? cs[1].value : cs[1] + Expr(:method, c1, cs[2:end]...) + elseif k == K"newvar" + Core.NewvarNode(to_lowered_expr(mod, ex[1], ssa_offset)) + elseif k == K"new_opaque_closure" + args = map(e->to_lowered_expr(mod, e, ssa_offset), children(ex)) + Expr(:new_opaque_closure, args...) + elseif k == K"meta" + args = Any[to_lowered_expr(mod, e, ssa_offset) for e in children(ex)] + # Unpack K"Symbol" QuoteNode as `Expr(:meta)` requires an identifier here. + args[1] = args[1].value + Expr(:meta, args...) + else + # Allowed forms according to https://docs.julialang.org/en/v1/devdocs/ast/ + # + # call invoke static_parameter `=` method struct_type abstract_type + # primitive_type global const new splatnew isdefined + # enter leave pop_exception inbounds boundscheck loopinfo copyast meta + # lambda + head = k == K"call" ? :call : + k == K"new" ? :new : + k == K"splatnew" ? :splatnew : + k == K"=" ? :(=) : + k == K"global" ? :global : + k == K"constdecl" ? :const : + k == K"leave" ? :leave : + k == K"isdefined" ? :isdefined : + k == K"latestworld" ? :latestworld : + k == K"globaldecl" ? :globaldecl : + k == K"pop_exception" ? :pop_exception : + k == K"captured_local" ? :captured_local : + k == K"gc_preserve_begin" ? :gc_preserve_begin : + k == K"gc_preserve_end" ? :gc_preserve_end : + k == K"foreigncall" ? :foreigncall : + k == K"cfunction" ? :cfunction : + k == K"opaque_closure_method" ? :opaque_closure_method : + nothing + if isnothing(head) + throw(LoweringError(ex, "Unhandled form for kind $k")) + end + Expr(head, map(e->to_lowered_expr(mod, e, ssa_offset), children(ex))...) + end +end + +#------------------------------------------------------------------------------- +# Our version of eval takes our own data structures +function Core.eval(mod::Module, ex::SyntaxTree) + k = kind(ex) + if k == K"toplevel" + x = nothing + for e in children(ex) + x = eval(mod, e) + end + return x + end + linear_ir = lower(mod, ex) + expr_form = to_lowered_expr(mod, linear_ir) + eval(mod, expr_form) +end + +""" + include(mod::Module, path::AbstractString) + +Evaluate the contents of the input source file in the global scope of module +`mod`. Every module (except those defined with baremodule) has its own +definition of `include()` omitting the `mod` argument, which evaluates the file +in that module. Returns the result of the last evaluated expression of the +input file. During including, a task-local include path is set to the directory +containing the file. Nested calls to include will search relative to that path. +This function is typically used to load source interactively, or to combine +files in packages that are broken into multiple source files. +""" +function include(mod::Module, path::AbstractString) + path, prev = Base._include_dependency(mod, path) + code = read(path, String) + tls = task_local_storage() + tls[:SOURCE_PATH] = path + try + return include_string(mod, code, path) + finally + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + end +end + +""" + include_string(mod::Module, code::AbstractString, filename::AbstractString="string") + +Like `include`, except reads code from the given string rather than from a file. +""" +function include_string(mod::Module, code::AbstractString, filename::AbstractString="string") + eval(mod, parseall(SyntaxTree, code; filename=filename)) +end + diff --git a/src/vendored/JuliaLowering/src/hooks.jl b/src/vendored/JuliaLowering/src/hooks.jl new file mode 100644 index 00000000..4b3748e7 --- /dev/null +++ b/src/vendored/JuliaLowering/src/hooks.jl @@ -0,0 +1,153 @@ +using ..JuliaSyntax + + +# Becomes `Core._lower()` upon activating JuliaLowering. Returns an svec with +# the lowered code (usually expr) as its first element, and whatever we want +# after it, until the API stabilizes +function core_lowerer_hook(code, mod::Module, file="none", line=0, world=typemax(Csize_t), warn=false) + if Base.isexpr(code, :syntaxtree) + # Getting toplevel.c to check for types it doesn't know about is hard. + # We wrap SyntaxTrees with this random expr head so that the call to + # `jl_needs_lowering` in `jl_toplevel_eval_flex` returns true; this way + # the SyntaxTree is handed back to us, unwraped here, and lowered. + code = code.args[1] + end + if code isa Expr + @warn("""JuliaLowering received an Expr instead of a SyntaxTree. + This is currently expected when evaluating modules. + Falling back to flisp...""", + code=code, file=file, line=line, mod=mod) + return Base.fl_lower(code, mod, file, line, world, warn) + elseif !(code isa SyntaxTree) + # LineNumberNode, Symbol, probably others... + return Core.svec(code) + end + try + ctx1, st1 = expand_forms_1( mod, code) + ctx2, st2 = expand_forms_2( ctx1, st1) + ctx3, st3 = resolve_scopes( ctx2, st2) + ctx4, st4 = convert_closures(ctx3, st3) + ctx5, st5 = linearize_ir( ctx4, st4) + ex = to_lowered_expr(mod, st5) + return Core.svec(ex, st5, ctx5) + catch exc + @error("JuliaLowering failed — falling back to flisp!", + exception=(exc,catch_backtrace()), + code=code, file=file, line=line, mod=mod) + return Base.fl_lower(st0, mod, file, line, world, warn) + end +end + +# TODO: This is code copied from JuliaSyntax, adapted to produce +# `Expr(:syntaxtree, st::SyntaxTree)`. +function core_parse_for_lowering_hook(code, filename::String, lineno::Int, offset::Int, options::Symbol) + if Core._lower != core_lowerer_hook + # If lowering can't handle SyntaxTree, return Expr. + # (assumes no Core._lower function other than our core_lowerer_hook can handle SyntaxTree) + return JuliaSyntax.core_parser_hook(code, filename, lineno, offset, options) + end + try + # TODO: Check that we do all this input wrangling without copying the + # code buffer + if code isa Core.SimpleVector + # The C entry points will pass us this form. + (ptr,len) = code + code = String(unsafe_wrap(Array, ptr, len)) + elseif !(code isa String || code isa SubString || code isa Vector{UInt8}) + # For non-Base string types, convert to UTF-8 encoding, using an + # invokelatest to avoid world age issues. + code = Base.invokelatest(String, code) + end + stream = JuliaSyntax.ParseStream(code, offset+1) + if options === :statement || options === :atom + # To copy the flisp parser driver: + # * Parsing atoms consumes leading trivia + # * Parsing statements consumes leading+trailing trivia + JuliaSyntax.bump_trivia(stream) + if peek(stream) == K"EndMarker" + # If we're at the end of stream after skipping whitespace, just + # return `nothing` to indicate this rather than attempting to + # parse a statement or atom and failing. + return Core.svec(nothing, last_byte(stream)) + end + end + JuliaSyntax.parse!(stream; rule=options) + if options === :statement + JuliaSyntax.bump_trivia(stream; skip_newlines=false) + if peek(stream) == K"NewlineWs" + JuliaSyntax.bump(stream) + end + end + + if JuliaSyntax.any_error(stream) + pos_before_comments = JuliaSyntax.last_non_whitespace_byte(stream) + tree = JuliaSyntax.build_tree(SyntaxNode, stream, first_line=lineno, filename=filename) + tag = JuliaSyntax._incomplete_tag(tree, pos_before_comments) + exc = JuliaSyntax.ParseError(stream, filename=filename, first_line=lineno, + incomplete_tag=tag) + msg = sprint(showerror, exc) + error_ex = Expr(tag === :none ? :error : :incomplete, + Meta.ParseError(msg, exc)) + ex = if options === :all + # When encountering a toplevel error, the reference parser + # * truncates the top level expression arg list before that error + # * includes the last line number + # * appends the error message + topex = Expr(tree) + @assert topex.head == :toplevel + i = findfirst(JuliaSyntax._has_nested_error, topex.args) + if i > 1 && topex.args[i-1] isa LineNumberNode + i -= 1 + end + resize!(topex.args, i-1) + _,errort = JuliaSyntax._first_error(tree) + push!(topex.args, LineNumberNode(JuliaSyntax.source_line(errort), filename)) + push!(topex.args, error_ex) + topex + else + error_ex + end + else + # See unwrapping of `:syntaxtree` above. + ex = Expr(:syntaxtree, JuliaSyntax.build_tree(SyntaxTree, stream; filename=filename, first_line=lineno)) + end + + # Note the next byte in 1-based indexing is `last_byte(stream) + 1` but + # the Core hook must return an offset (ie, it's 0-based) so the factors + # of one cancel here. + last_offset = last_byte(stream) + + # Rewrap result in an svec for use by the C code + return Core.svec(ex, last_offset) + catch exc + @error("""JuliaSyntax parser failed — falling back to flisp! + This is not your fault. Please submit a bug report to https://github.com/JuliaLang/JuliaSyntax.jl/issues""", + exception=(exc,catch_backtrace()), + offset=offset, + code=code) + + Base.fl_parse(code, filename, lineno, offset, options) + end +end + +const _has_v1_13_hooks = isdefined(Core, :_lower) + +function activate!(enable=true) + if !_has_v1_13_hooks + error("Cannot use JuliaLowering without `Core._lower` binding or in $VERSION < 1.13") + end + + if enable + if !isnothing(Base.active_repl_backend) + # TODO: These act on parsed exprs, which we don't have. + # Reimplementation needed (e.g. for scoping rules). + empty!(Base.active_repl_backend.ast_transforms) + end + + Core._setlowerer!(core_lowerer_hook) + Core._setparser!(core_parse_for_lowering_hook) + else + Core._setlowerer!(Base.fl_lower) + Core._setparser!(JuliaSyntax.core_parse_hook) + end +end diff --git a/src/vendored/JuliaLowering/src/kinds.jl b/src/vendored/JuliaLowering/src/kinds.jl new file mode 100644 index 00000000..09f33487 --- /dev/null +++ b/src/vendored/JuliaLowering/src/kinds.jl @@ -0,0 +1,154 @@ + +# The following kinds are used in intermediate forms by lowering but are not +# part of the surface syntax +function _register_kinds() + JuliaSyntax.register_kinds!(JuliaLowering, 1, [ + # "Syntax extensions" - expression kinds emitted by macros or macro + # expansion, and known to lowering. These are part of the AST API but + # without having surface syntax. + "BEGIN_EXTENSION_KINDS" + # atomic fields or accesses (see `@atomic`) + "atomic" + # Flag for @generated parts of a functon + "generated" + # Temporary rooting of identifiers (GC.@preserve) + "gc_preserve_begin" + "gc_preserve_end" + # A literal Julia value of any kind, as might be inserted into the + # AST during macro expansion + "Value" + # A (quoted) `Symbol` + "Symbol" + # Compiler metadata hints + "meta" + # TODO: Use `meta` for inbounds and loopinfo etc? + "inbounds" + "boundscheck" + "inline" + "noinline" + "loopinfo" + # Call into foreign code. Emitted by `@ccall` + "foreigncall" + # Special form for constructing a function callable from C + "cfunction" + # Special form emitted by `Base.Experimental.@opaque` + "opaque_closure" + # Test whether a variable is defined + "isdefined" + # [K"throw_undef_if_not" var cond] + # This form is used internally in Core.Compiler but might be + # emitted by packages such as Diffractor. In principle it needs to + # be passed through lowering in a similar way to `isdefined` + "throw_undef_if_not" + # named labels for `@label` and `@goto` + "symbolic_label" + # Goto named label + "symbolic_goto" + # Internal initializer for struct types, for inner constructors/functions + "new" + "splatnew" + # Catch-all for additional syntax extensions without the need to + # extend `Kind`. Known extensions include: + # locals, islocal + # The content of an assertion is not considered to be quoted, so + # use K"Symbol" or K"inert" inside where necessary. + "extension" + "END_EXTENSION_KINDS" + + # The following kinds are internal to lowering + "BEGIN_LOWERING_KINDS" + # Semantic assertions used by lowering. The content of an assertion + # is not considered to be quoted, so use K"Symbol" etc inside where necessary. + "assert" + # Unique identifying integer for bindings (of variables, constants, etc) + "BindingId" + # Various heads harvested from flisp lowering. + # (TODO: May or may not need all these - assess later) + "break_block" + # Like block, but introduces a lexical scope; used during scope resolution. + "scope_block" + # [K"always_defined" x] is an assertion that variable `x` is assigned before use + # ('local-def in flisp implementation is K"local" plus K"always_defined" + "always_defined" + "_while" + "_do_while" + "_typevars" # used for supplying already-allocated `TypeVar`s to `where` + "with_static_parameters" + "top" + "core" + "lambda" + # "A source location literal" - a node which exists only to record + # a sourceref + "SourceLocation" + # [K"function_decl" name] + # Declare a zero-method generic function with global `name` or + # creates a closure object and assigns it to the local `name`. + "function_decl" + # [K"function_type name] + # Evaluates to the type of the function or closure with given `name` + "function_type" + # [K"method_defs" name block] + # The code in `block` defines methods for generic function `name` + "method_defs" + # The code in `block` defines methods for generic function `name` + "_opaque_closure" + # The enclosed statements must be executed at top level + "toplevel_butfirst" + "assign_const_if_global" + "moved_local" + "label" + "trycatchelse" + "tryfinally" + # The contained block of code causes no side effects and can be + # removed by a later lowering pass if its value isn't used. + # (That is, it's removable in the same sense as + # `@assume_effects :removable`.) + "removable" + "decl" + # [K"captured_local" index] + # A local variable captured into a global method. Contains the + # `index` of the associated `Box` in the rewrite list. + "captured_local" + # Causes the linearization pass to conditionally emit a world age increment + "latestworld_if_toplevel" + "END_LOWERING_KINDS" + + # The following kinds are emitted by lowering and used in Julia's untyped IR + "BEGIN_IR_KINDS" + # Identifier for a value which is only assigned once + "SSAValue" + # Local variable in a `CodeInfo` code object (including lambda arguments) + "slot" + # Static parameter to a `CodeInfo` code object ("type parameters" to methods) + "static_parameter" + # References/declares a global variable within a module + "globalref" + "globaldecl" + # Two-argument constant declaration and assignment. + # Translated to :const in the IR for now (we use K"const" already in parsing). + "constdecl" + # Unconditional goto + "goto" + # Conditional goto + "gotoifnot" + # Exception handling + "enter" + "leave" + "pop_exception" + # Lowering targets for method definitions arising from `function` etc + "method" + # (re-)initialize a slot to undef + # See Core.NewvarNode + "newvar" + # Result of lowering a `K"lambda"` after bindings have been + # converted to slot/globalref/SSAValue. + "code_info" + # Internal initializer for opaque closures + "new_opaque_closure" + # Wrapper for the lambda of around opaque closure methods + "opaque_closure_method" + # World age increment + "latestworld" + "END_IR_KINDS" + ]) +end diff --git a/src/vendored/JuliaLowering/src/linear_ir.jl b/src/vendored/JuliaLowering/src/linear_ir.jl new file mode 100644 index 00000000..ad06172f --- /dev/null +++ b/src/vendored/JuliaLowering/src/linear_ir.jl @@ -0,0 +1,1161 @@ +#------------------------------------------------------------------------------- +# Lowering pass 5: Flatten to linear IR + +function is_valid_ir_argument(ctx, ex) + k = kind(ex) + if is_simple_atom(ctx, ex) || k in KSet"inert top core quote" + true + elseif k == K"BindingId" + binfo = lookup_binding(ctx, ex) + bk = binfo.kind + # TODO: Can we allow bk == :local || bk == :argument || bk == :static_parameter ??? + # Why does flisp seem to allow (slot) and (static_parameter), but these + # aren't yet converted to by existing lowering?? + (bk == :slot || bk == :static_parameter) + else + false + end +end + +function is_ssa(ctx, ex) + kind(ex) == K"BindingId" && lookup_binding(ctx, ex).is_ssa +end + +# Target to jump to, including info on try handler nesting and catch block +# nesting +struct JumpTarget{GraphType} + label::SyntaxTree{GraphType} + handler_token_stack::SyntaxList{GraphType, Vector{NodeId}} + catch_token_stack::SyntaxList{GraphType, Vector{NodeId}} +end + +function JumpTarget(label::SyntaxTree{GraphType}, ctx) where {GraphType} + JumpTarget{GraphType}(label, copy(ctx.handler_token_stack), copy(ctx.catch_token_stack)) +end + +struct JumpOrigin{GraphType} + goto::SyntaxTree{GraphType} + index::Int + handler_token_stack::SyntaxList{GraphType, Vector{NodeId}} + catch_token_stack::SyntaxList{GraphType, Vector{NodeId}} +end + +function JumpOrigin(goto::SyntaxTree{GraphType}, index, ctx) where {GraphType} + JumpOrigin{GraphType}(goto, index, copy(ctx.handler_token_stack), copy(ctx.catch_token_stack)) +end + +struct FinallyHandler{GraphType} + tagvar::SyntaxTree{GraphType} + target::JumpTarget{GraphType} + exit_actions::Vector{Tuple{Symbol,Union{Nothing,SyntaxTree{GraphType}}}} +end + +function FinallyHandler(tagvar::SyntaxTree{GraphType}, target::JumpTarget) where {GraphType} + FinallyHandler{GraphType}(tagvar, target, + Vector{Tuple{Symbol, Union{Nothing,SyntaxTree{GraphType}}}}()) +end + + +""" +Context for creating linear IR. + +One of these is created per lambda expression to flatten the body down to +a sequence of statements (linear IR). +""" +struct LinearIRContext{GraphType} <: AbstractLoweringContext + graph::GraphType + code::SyntaxList{GraphType, Vector{NodeId}} + bindings::Bindings + next_label_id::Ref{Int} + is_toplevel_thunk::Bool + lambda_bindings::LambdaBindings + return_type::Union{Nothing, SyntaxTree{GraphType}} + break_targets::Dict{String, JumpTarget{GraphType}} + handler_token_stack::SyntaxList{GraphType, Vector{NodeId}} + catch_token_stack::SyntaxList{GraphType, Vector{NodeId}} + finally_handlers::Vector{FinallyHandler{GraphType}} + symbolic_jump_targets::Dict{String,JumpTarget{GraphType}} + symbolic_jump_origins::Vector{JumpOrigin{GraphType}} + mod::Module +end + +function LinearIRContext(ctx, is_toplevel_thunk, lambda_bindings, return_type) + graph = syntax_graph(ctx) + rett = isnothing(return_type) ? nothing : reparent(graph, return_type) + GraphType = typeof(graph) + LinearIRContext(graph, SyntaxList(ctx), ctx.bindings, Ref(0), + is_toplevel_thunk, lambda_bindings, rett, + Dict{String,JumpTarget{GraphType}}(), SyntaxList(ctx), SyntaxList(ctx), + Vector{FinallyHandler{GraphType}}(), Dict{String,JumpTarget{GraphType}}(), + Vector{JumpOrigin{GraphType}}(), ctx.mod) +end + +function current_lambda_bindings(ctx::LinearIRContext) + ctx.lambda_bindings +end + +function is_valid_body_ir_argument(ctx, ex) + if is_valid_ir_argument(ctx, ex) + true + elseif kind(ex) == K"BindingId" + binfo = lookup_binding(ctx, ex) + # Arguments are always defined + # TODO: use equiv of vinfo:never-undef when we have it + binfo.kind == :argument + else + false + end +end + +function is_simple_arg(ctx, ex) + k = kind(ex) + return is_simple_atom(ctx, ex) || k == K"BindingId" || k == K"quote" || k == K"inert" || + k == K"top" || k == K"core" || k == K"globalref" +end + +function is_single_assign_var(ctx::LinearIRContext, ex) + kind(ex) == K"BindingId" || return false + binfo = lookup_binding(ctx, ex) + # Arguments are always single-assign + # TODO: Use equiv of vinfo:sa when we have it + return binfo.kind == :argument +end + +function is_const_read_arg(ctx, ex) + k = kind(ex) + # Even if we have side effects, we know that singly-assigned + # locals cannot be affected by them so we can inline them anyway. + # TODO from flisp: "We could also allow const globals here" + return k == K"inert" || k == K"top" || k == K"core" || + is_simple_atom(ctx, ex) || is_single_assign_var(ctx, ex) +end + +function is_valid_ir_rvalue(ctx, lhs, rhs) + return is_ssa(ctx, lhs) || + is_valid_ir_argument(ctx, rhs) || + (kind(lhs) == K"BindingId" && + # FIXME: add: invoke ? + kind(rhs) in KSet"new splatnew cfunction isdefined call foreigncall gc_preserve_begin foreigncall new_opaque_closure") +end + +function check_no_local_bindings(ctx, ex, msg) + contains_nonglobal_binding = contains_unquoted(ex) do e + kind(e) == K"BindingId" && lookup_binding(ctx, e).kind !== :global + end + if contains_nonglobal_binding + throw(LoweringError(ex, msg)) + end +end + +# evaluate the arguments of a call, creating temporary locations as needed +function compile_args(ctx, args) + # First check if all the arguments are simple (and therefore side-effect free). + # Otherwise, we need to use ssa values for all arguments to ensure proper + # left-to-right evaluation semantics. + all_simple = all(a->is_simple_arg(ctx, a), args) + args_out = SyntaxList(ctx) + for arg in args + arg_val = compile(ctx, arg, true, false) + if (all_simple || is_const_read_arg(ctx, arg_val)) && is_valid_body_ir_argument(ctx, arg_val) + push!(args_out, arg_val) + else + push!(args_out, emit_assign_tmp(ctx, arg_val)) + end + end + return args_out +end + +# Compile the (sym,lib) argument to ccall/cglobal +function compile_C_library_symbol(ctx, ex) + if kind(ex) == K"call" && kind(ex[1]) == K"core" && ex[1].name_val == "tuple" + # Tuples like core.tuple(:funcname, mylib_name) are allowed and are + # kept inline, but may only reference globals. + check_no_local_bindings(ctx, ex, + "function name and library expression cannot reference local variables") + ex + else + only(compile_args(ctx, (ex,))) + end +end + +function emit(ctx::LinearIRContext, ex) + push!(ctx.code, ex) + return ex +end + +function emit(ctx::LinearIRContext, srcref, k, args...) + emit(ctx, makenode(ctx, srcref, k, args...)) +end + +# Emit computation of ex, assigning the result to an ssavar and returning that +function emit_assign_tmp(ctx::LinearIRContext, ex, name="tmp") + tmp = ssavar(ctx, ex, name) + emit(ctx, @ast ctx ex [K"=" tmp ex]) + return tmp +end + +function compile_pop_exception(ctx, srcref, src_tokens, dest_tokens) + # It's valid to leave the context of src_tokens for the context of + # dest_tokens when src_tokens is the same or nested within dest_tokens. + # It's enough to check the token on the top of the dest stack. + n = length(dest_tokens) + jump_ok = n == 0 || (n <= length(src_tokens) && dest_tokens[n].var_id == src_tokens[n].var_id) + jump_ok || throw(LoweringError(srcref, "Attempt to jump into catch block")) + if n < length(src_tokens) + @ast ctx srcref [K"pop_exception" src_tokens[n+1]] + else + nothing + end +end + +function compile_leave_handler(ctx, srcref, src_tokens, dest_tokens) + n = length(dest_tokens) + jump_ok = n == 0 || (n <= length(src_tokens) && dest_tokens[n].var_id == src_tokens[n].var_id) + jump_ok || throw(LoweringError(srcref, "Attempt to jump into try block")) + if n < length(src_tokens) + @ast ctx srcref [K"leave" src_tokens[n+1:end]...] + else + nothing + end +end + +function emit_pop_exception(ctx::LinearIRContext, srcref, dest_tokens) + pexc = compile_pop_exception(ctx, srcref, ctx.catch_token_stack, dest_tokens) + if !isnothing(pexc) + emit(ctx, pexc) + end +end + +function emit_leave_handler(ctx::LinearIRContext, srcref, dest_tokens) + ex = compile_leave_handler(ctx, srcref, ctx.handler_token_stack, dest_tokens) + if !isnothing(ex) + emit(ctx, ex) + end +end + +function emit_jump(ctx, srcref, target::JumpTarget) + emit_pop_exception(ctx, srcref, target.catch_token_stack) + emit_leave_handler(ctx, srcref, target.handler_token_stack) + emit(ctx, @ast ctx srcref [K"goto" target.label]) +end + +# Enter the current finally block, either through the landing pad (on_exit == +# :rethrow) or via a jump (on_exit ∈ (:return, :break)). +# +# An integer tag is created to identify the current code path and select the +# on_exit action to be taken at finally handler exit. +function enter_finally_block(ctx, srcref, on_exit, value) + @assert on_exit ∈ (:rethrow, :break, :return) + handler = last(ctx.finally_handlers) + push!(handler.exit_actions, (on_exit, value)) + tag = length(handler.exit_actions) + emit(ctx, @ast ctx srcref [K"=" handler.tagvar tag::K"Integer"]) + if on_exit != :rethrow + emit_jump(ctx, srcref, handler.target) + end +end + +# Helper function for emit_return +function _actually_return(ctx, ex) + # TODO: Handle the implicit return coverage hack for #53354 ? + rett = ctx.return_type + if !isnothing(rett) + ex = compile(ctx, convert_for_type_decl(ctx, rett, ex, rett, true), true, false) + end + simple_ret_val = isempty(ctx.catch_token_stack) ? + # returning lambda directly is needed for @generated + (is_valid_ir_argument(ctx, ex) || kind(ex) == K"lambda") : + is_simple_atom(ctx, ex) + if !simple_ret_val + ex = emit_assign_tmp(ctx, ex, "return_tmp") + end + emit_pop_exception(ctx, ex, ()) + emit(ctx, @ast ctx ex [K"return" ex]) + return nothing +end + +function emit_return(ctx, srcref, ex) + # todo: Mark implicit returns + if isnothing(ex) + return + elseif isempty(ctx.handler_token_stack) + _actually_return(ctx, ex) + return + end + # TODO: What's this !is_ssa(ctx, ex) here about? + x = if is_simple_atom(ctx, ex) && !(is_ssa(ctx, ex) && !isempty(ctx.finally_handlers)) + ex + elseif !isempty(ctx.finally_handlers) + # todo: Why does flisp lowering create a mutable variable here even + # though we don't mutate it? + # tmp = ssavar(ctx, srcref, "returnval_via_finally") # <- can we use this? + tmp = new_local_binding(ctx, srcref, "returnval_via_finally") + emit(ctx, @ast ctx srcref [K"=" tmp ex]) + tmp + else + emit_assign_tmp(ctx, ex, "returnval_via_finally") + end + if !isempty(ctx.finally_handlers) + enter_finally_block(ctx, srcref, :return, x) + else + emit(ctx, @ast ctx srcref [K"leave" ctx.handler_token_stack...]) + _actually_return(ctx, x) + end + return nothing +end + +function emit_return(ctx, ex) + emit_return(ctx, ex, ex) +end + +function emit_break(ctx, ex) + name = ex[1].name_val + target = get(ctx.break_targets, name, nothing) + if isnothing(target) + ty = name == "loop_exit" ? "break" : "continue" + throw(LoweringError(ex, "$ty must be used inside a `while` or `for` loop")) + end + if !isempty(ctx.finally_handlers) + handler = last(ctx.finally_handlers) + if length(target.handler_token_stack) < length(handler.target.handler_token_stack) + enter_finally_block(ctx, ex, :break, ex) + return + end + end + emit_jump(ctx, ex, target) +end + +# `op` may also be K"constdecl" +function emit_assignment_or_setglobal(ctx, srcref, lhs, rhs, op=K"=") + # (const (globalref _ _) _) does not use setglobal! + binfo = lookup_binding(ctx, lhs.var_id) + if binfo.kind == :global && op == K"=" + emit(ctx, @ast ctx srcref [ + K"call" + "setglobal!"::K"top" + binfo.mod::K"Value" + binfo.name::K"Symbol" + rhs + ]) + else + emit(ctx, srcref, op, lhs, rhs) + end +end + +function emit_assignment(ctx, srcref, lhs, rhs, op=K"=") + if !isnothing(rhs) + if is_valid_ir_rvalue(ctx, lhs, rhs) + emit_assignment_or_setglobal(ctx, srcref, lhs, rhs, op) + else + r = emit_assign_tmp(ctx, rhs) + emit_assignment_or_setglobal(ctx, srcref, lhs, r, op) + end + else + # in unreachable code (such as after return); still emit the assignment + # so that the structure of those uses is preserved + emit_assignment_or_setglobal(ctx, srcref, lhs, @ast ctx srcref "nothing"::K"core", op) + nothing + end +end + +function make_label(ctx, srcref) + id = ctx.next_label_id[] + ctx.next_label_id[] += 1 + makeleaf(ctx, srcref, K"label", id=id) +end + +# flisp: make&mark-label +function emit_label(ctx, srcref) + if !isempty(ctx.code) + # Use current label if available + e = ctx.code[end] + if kind(e) == K"label" + return e + end + end + l = make_label(ctx, srcref) + emit(ctx, l) + l +end + +function compile_condition_term(ctx, ex) + cond = compile(ctx, ex, true, false) + if !is_valid_body_ir_argument(ctx, cond) + cond = emit_assign_tmp(ctx, cond) + end + return cond +end + +# flisp: emit-cond +function compile_conditional(ctx, ex, false_label) + if kind(ex) == K"block" + for i in 1:numchildren(ex)-1 + compile(ctx, ex[i], false, false) + end + test = ex[end] + else + test = ex + end + k = kind(test) + if k == K"||" + true_label = make_label(ctx, test) + for (i,e) in enumerate(children(test)) + c = compile_condition_term(ctx, e) + if i < numchildren(test) + next_term_label = make_label(ctx, test) + # Jump over short circuit + emit(ctx, @ast ctx e [K"gotoifnot" c next_term_label]) + # Short circuit to true + emit(ctx, @ast ctx e [K"goto" true_label]) + emit(ctx, next_term_label) + else + emit(ctx, @ast ctx e [K"gotoifnot" c false_label]) + end + end + emit(ctx, true_label) + elseif k == K"&&" + for e in children(test) + c = compile_condition_term(ctx, e) + emit(ctx, @ast ctx e [K"gotoifnot" c false_label]) + end + else + c = compile_condition_term(ctx, test) + emit(ctx, @ast ctx test [K"gotoifnot" c false_label]) + end +end + +# Lowering of exception handling must ensure that +# +# * Each `enter` is matched with a `leave` on every possible non-exceptional +# program path (including implicit returns generated in tail position). +# * Each catch block which is entered and handles the exception - by exiting +# via a non-exceptional program path - leaves the block with `pop_exception`. +# * Each `finally` block runs, regardless of any early `return` or jumps +# via `break`/`continue`/`goto` etc. +# +# These invariants are upheld by tracking the nesting using +# `handler_token_stack` and `catch_token_stack` and using these when emitting +# any control flow (return / goto) which leaves the associated block. +# +# The following special forms are emitted into the IR: +# +# (= tok (enter catch_label dynscope)) +# push exception handler with catch block at `catch_label` and dynamic +# scope `dynscope`, yielding a token which is used by `leave` and +# `pop_exception`. `dynscope` is only used in the special `tryfinally` form +# without associated source level syntax (see the `@with` macro) +# +# (leave tok) +# pop exception handler back to the state of the `tok` from the associated +# `enter`. Multiple tokens can be supplied to pop multiple handlers using +# `(leave tok1 tok2 ...)`. +# +# (pop_exception tok) - pop exception stack back to state of associated enter +# +# See the devdocs for further discussion. +function compile_try(ctx::LinearIRContext, ex, needs_value, in_tail_pos) + @chk numchildren(ex) <= 3 + try_block = ex[1] + if kind(ex) == K"trycatchelse" + catch_block = ex[2] + else_block = numchildren(ex) == 2 ? nothing : ex[3] + finally_block = nothing + catch_label = make_label(ctx, catch_block) + else + catch_block = nothing + else_block = nothing + finally_block = ex[2] + catch_label = make_label(ctx, finally_block) + end + + end_label = !in_tail_pos || !isnothing(finally_block) ? make_label(ctx, ex) : nothing + try_result = needs_value && !in_tail_pos ? new_local_binding(ctx, ex, "try_result") : nothing + + # Exception handler block prefix + handler_token = ssavar(ctx, ex, "handler_token") + emit(ctx, @ast ctx ex [K"=" + handler_token + [K"enter" catch_label] # TODO: dynscope + ]) + if !isnothing(finally_block) + # TODO: Trivial finally block optimization from JuliaLang/julia#52593 (or + # support a special form for @with)? + finally_handler = FinallyHandler(new_local_binding(ctx, finally_block, "finally_tag"), + JumpTarget(end_label, ctx)) + push!(ctx.finally_handlers, finally_handler) + emit(ctx, @ast ctx finally_block [K"=" finally_handler.tagvar (-1)::K"Integer"]) + end + push!(ctx.handler_token_stack, handler_token) + + # Try block code. + try_val = compile(ctx, try_block, needs_value, false) + # Exception handler block postfix + if isnothing(else_block) + if in_tail_pos + if !isnothing(try_val) + emit_return(ctx, try_val) + end + else + if needs_value && !isnothing(try_val) + emit_assignment(ctx, ex, try_result, try_val) + end + emit(ctx, @ast ctx ex [K"leave" handler_token]) + end + pop!(ctx.handler_token_stack) + else + if !isnothing(try_val) && (in_tail_pos || needs_value) + emit(ctx, try_val) # TODO: Only for any side effects ? + end + emit(ctx, @ast ctx ex [K"leave" handler_token]) + pop!(ctx.handler_token_stack) + # Else block code + else_val = compile(ctx, else_block, needs_value, in_tail_pos) + if !in_tail_pos + if needs_value && !isnothing(else_val) + emit_assignment(ctx, ex, try_result, else_val) + end + end + end + if !in_tail_pos + emit(ctx, @ast ctx ex [K"goto" end_label]) + end + + # Catch pad + # Emit either catch or finally block. A combined try/catch/finally block + # was split into separate trycatchelse and tryfinally blocks earlier. + emit(ctx, catch_label) # <- Exceptional control flow enters here + if !isnothing(finally_block) + # Attribute the postfix and prefix to the finally block as a whole. + srcref = finally_block + enter_finally_block(ctx, srcref, :rethrow, nothing) + emit(ctx, end_label) # <- Non-exceptional control flow enters here + pop!(ctx.finally_handlers) + compile(ctx, finally_block, false, false) + # Finally block postfix: Emit a branch for every code path which enters + # the block to dynamically decide which return/break/rethrow exit action to take + for (tag, (on_exit, value)) in Iterators.reverse(enumerate(finally_handler.exit_actions)) + next_action_label = !in_tail_pos || tag != 1 || on_exit != :return ? + make_label(ctx, srcref) : nothing + if !isnothing(next_action_label) + next_action_label = make_label(ctx, srcref) + tmp = ssavar(ctx, srcref, "do_finally_action") + emit(ctx, @ast ctx srcref [K"=" tmp + [K"call" + "==="::K"core" + finally_handler.tagvar + tag::K"Integer" + ] + ]) + emit(ctx, @ast ctx srcref [K"gotoifnot" tmp next_action_label]) + end + if on_exit === :return + emit_return(ctx, value) + elseif on_exit === :break + emit_break(ctx, value) + elseif on_exit === :rethrow + emit(ctx, @ast ctx srcref [K"call" "rethrow"::K"top"]) + else + @assert false + end + if !isnothing(next_action_label) + emit(ctx, next_action_label) + end + end + else + push!(ctx.catch_token_stack, handler_token) + catch_val = compile(ctx, catch_block, needs_value, in_tail_pos) + if !isnothing(try_result) && !isnothing(catch_val) + emit_assignment(ctx, ex, try_result, catch_val) + end + if !in_tail_pos + emit(ctx, @ast ctx ex [K"pop_exception" handler_token]) + emit(ctx, end_label) + else + # (pop_exception done in emit_return) + end + pop!(ctx.catch_token_stack) + end + try_result +end + +# This pass behaves like an interpreter on the given code. +# To perform stateful operations, it calls `emit` to record that something +# needs to be done. In value position, it returns an expression computing +# the needed value. +function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos) + k = kind(ex) + if k == K"BindingId" || is_literal(k) || k == K"quote" || k == K"inert" || + k == K"top" || k == K"core" || k == K"Value" || k == K"Symbol" || + k == K"SourceLocation" + if in_tail_pos + emit_return(ctx, ex) + elseif needs_value + ex + else + if k == K"BindingId" && !is_ssa(ctx, ex) + emit(ctx, ex) # keep identifiers for undefined-var checking + end + nothing + end + elseif k == K"Placeholder" + if needs_value + throw(LoweringError(ex, "all-underscore identifiers are write-only and their values cannot be used in expressions")) + end + nothing + elseif k == K"TOMBSTONE" + @chk !needs_value (ex,"TOMBSTONE encountered in value position") + nothing + elseif k == K"call" || k == K"new" || k == K"splatnew" || k == K"foreigncall" || + k == K"new_opaque_closure" || k == K"cfunction" + if k == K"foreigncall" + args = SyntaxList(ctx) + push!(args, compile_C_library_symbol(ctx, ex[1])) + # 2nd to 5th arguments of foreigncall are special. They must be + # left in place but cannot reference locals. + check_no_local_bindings(ctx, ex[2], "ccall return type cannot reference local variables") + for argt in children(ex[3]) + check_no_local_bindings(ctx, argt, + "ccall argument types cannot reference local variables") + end + append!(args, ex[2:5]) + append!(args, compile_args(ctx, ex[6:end])) + args + elseif k == K"cfunction" + # Arguments of cfunction must be left in place except for argument + # 2 (fptr) + args = copy(children(ex)) + args[2] = only(compile_args(ctx, args[2:2])) + check_no_local_bindings(ctx, ex[3], + "cfunction return type cannot reference local variables") + for arg in children(ex[4]) + check_no_local_bindings(ctx, arg, + "cfunction argument cannot reference local variables") + end + elseif k == K"call" && is_core_ref(ex[1], "cglobal") + args = SyntaxList(ctx) + push!(args, ex[1]) + push!(args, compile_C_library_symbol(ctx, ex[2])) + append!(args, compile_args(ctx, ex[3:end])) + else + args = compile_args(ctx, children(ex)) + end + callex = makenode(ctx, ex, k, args) + if in_tail_pos + emit_return(ctx, ex, callex) + elseif needs_value + callex + else + emit(ctx, callex) + nothing + end + elseif k == K"=" || k == K"constdecl" + lhs = ex[1] + if kind(lhs) == K"Placeholder" + compile(ctx, ex[2], needs_value, in_tail_pos) + else + rhs = compile(ctx, ex[2], true, false) + # TODO look up arg-map for renaming if lhs was reassigned + if needs_value && !isnothing(rhs) + r = emit_assign_tmp(ctx, rhs) + emit_assignment_or_setglobal(ctx, ex, lhs, r, k) + if in_tail_pos + emit_return(ctx, ex, r) + else + r + end + else + emit_assignment(ctx, ex, lhs, rhs, k) + end + end + elseif k == K"block" || k == K"scope_block" + nc = numchildren(ex) + if nc == 0 + if in_tail_pos + emit_return(ctx, nothing_(ctx, ex)) + elseif needs_value + nothing_(ctx, ex) + else + nothing + end + else + res = nothing + for i in 1:nc + islast = i == nc + res = compile(ctx, ex[i], islast && needs_value, islast && in_tail_pos) + end + res + end + elseif k == K"break_block" + end_label = make_label(ctx, ex) + name = ex[1].name_val + outer_target = get(ctx.break_targets, name, nothing) + ctx.break_targets[name] = JumpTarget(end_label, ctx) + compile(ctx, ex[2], false, false) + if isnothing(outer_target) + delete!(ctx.break_targets, name) + else + ctx.break_targets = outer_target + end + emit(ctx, end_label) + if needs_value + compile(ctx, nothing_(ctx, ex), needs_value, in_tail_pos) + end + elseif k == K"break" + emit_break(ctx, ex) + elseif k == K"symbolic_label" + label = emit_label(ctx, ex) + name = ex.name_val + if haskey(ctx.symbolic_jump_targets, name) + throw(LoweringError(ex, "Label `$name` defined multiple times")) + end + push!(ctx.symbolic_jump_targets, name=>JumpTarget(label, ctx)) + if in_tail_pos + emit_return(ctx, ex, nothing_(ctx, ex)) + elseif needs_value + throw(LoweringError(ex, "misplaced label in value position")) + end + elseif k == K"symbolic_goto" + push!(ctx.symbolic_jump_origins, JumpOrigin(ex, length(ctx.code)+1, ctx)) + emit(ctx, makeleaf(ctx, ex, K"TOMBSTONE")) # ? pop_exception + emit(ctx, makeleaf(ctx, ex, K"TOMBSTONE")) # ? leave + emit(ctx, makeleaf(ctx, ex, K"TOMBSTONE")) # ? goto + nothing + elseif k == K"return" + compile(ctx, ex[1], true, true) + nothing + elseif k == K"removable" + if needs_value + compile(ctx, ex[1], needs_value, in_tail_pos) + else + nothing + end + elseif k == K"if" || k == K"elseif" + @chk numchildren(ex) <= 3 + has_else = numchildren(ex) > 2 + else_label = make_label(ctx, ex) + compile_conditional(ctx, ex[1], else_label) + if in_tail_pos + compile(ctx, ex[2], needs_value, in_tail_pos) + emit(ctx, else_label) + if has_else + compile(ctx, ex[3], needs_value, in_tail_pos) + else + emit_return(ctx, ex, nothing_(ctx, ex)) + end + nothing + else + val = needs_value && new_local_binding(ctx, ex, "if_val") + v1 = compile(ctx, ex[2], needs_value, in_tail_pos) + if needs_value + emit_assignment(ctx, ex, val, v1) + end + if has_else || needs_value + end_label = make_label(ctx, ex) + emit(ctx, @ast ctx ex [K"goto" end_label]) + else + end_label = nothing + end + emit(ctx, else_label) + v2 = if has_else + compile(ctx, ex[3], needs_value, in_tail_pos) + elseif needs_value + nothing_(ctx, ex) + end + if needs_value + emit_assignment(ctx, ex, val, v2) + end + if !isnothing(end_label) + emit(ctx, end_label) + end + val + end + elseif k == K"trycatchelse" || k == K"tryfinally" + compile_try(ctx, ex, needs_value, in_tail_pos) + elseif k == K"method" + # TODO + # throw(LoweringError(ex, + # "Global method definition needs to be placed at the top level, or use `eval`")) + if numchildren(ex) == 1 + if in_tail_pos + emit_return(ctx, ex) + elseif needs_value + ex + else + emit(ctx, ex) + end + else + @chk numchildren(ex) == 3 + fname = ex[1] + sig = compile(ctx, ex[2], true, false) + if !is_valid_ir_argument(ctx, sig) + sig = emit_assign_tmp(ctx, sig) + end + lam = ex[3] + if kind(lam) == K"lambda" + lam = compile_lambda(ctx, lam) + else + lam = emit_assign_tmp(ctx, compile(ctx, lam, true, false)) + end + emit(ctx, ex, K"method", fname, sig, lam) + @assert !needs_value && !in_tail_pos + nothing + end + elseif k == K"opaque_closure_method" + @ast ctx ex [K"opaque_closure_method" + ex[1] + ex[2] + ex[3] + ex[4] + compile_lambda(ctx, ex[5]) + ] + elseif k == K"lambda" + lam = compile_lambda(ctx, ex) + if in_tail_pos + emit_return(ctx, lam) + elseif needs_value + lam + else + emit(ctx, lam) + end + elseif k == K"gc_preserve_begin" + makenode(ctx, ex, k, compile_args(ctx, children(ex))) + elseif k == K"gc_preserve_end" + if needs_value + throw(LoweringError(ex, "misplaced kind $k in value position")) + end + emit(ctx, ex) + nothing + elseif k == K"global" + if needs_value + throw(LoweringError(ex, "misplaced kind $k in value position")) + end + emit(ctx, ex) + ctx.is_toplevel_thunk && emit(ctx, makenode(ctx, ex, K"latestworld")) + nothing + elseif k == K"meta" + emit(ctx, ex) + if needs_value + val = @ast ctx ex "nothing"::K"core" + if in_tail_pos + emit_return(ctx, val) + else + val + end + end + elseif k == K"_while" + end_label = make_label(ctx, ex) + top_label = emit_label(ctx, ex) + compile_conditional(ctx, ex[1], end_label) + compile(ctx, ex[2], false, false) + emit(ctx, @ast ctx ex [K"goto" top_label]) + emit(ctx, end_label) + if needs_value + compile(ctx, nothing_(ctx, ex), needs_value, in_tail_pos) + end + elseif k == K"_do_while" + end_label = make_label(ctx, ex) + top_label = emit_label(ctx, ex) + compile(ctx, ex[1], false, false) + compile_conditional(ctx, ex[2], end_label) + emit(ctx, @ast ctx ex [K"goto" top_label]) + emit(ctx, end_label) + if needs_value + compile(ctx, nothing_(ctx, ex), needs_value, in_tail_pos) + end + elseif k == K"isdefined" || k == K"captured_local" || k == K"throw_undef_if_not" || + k == K"boundscheck" + if in_tail_pos + emit_return(ctx, ex) + elseif needs_value + ex + end + elseif k == K"newvar" + @assert !needs_value + is_duplicate = !isempty(ctx.code) && + (e = last(ctx.code); kind(e) == K"newvar" && e[1].var_id == ex[1].var_id) + if !is_duplicate + # TODO: also exclude deleted vars + emit(ctx, ex) + end + elseif k == K"globaldecl" + if needs_value + throw(LoweringError(ex, "misplaced global declaration")) + end + if numchildren(ex) == 1 || is_identifier_like(ex[2]) + emit(ctx, ex) + else + rr = ssavar(ctx, ex[2]) + emit(ctx, @ast ctx ex [K"=" rr ex[2]]) + emit(ctx, @ast ctx ex [K"globaldecl" ex[1] rr]) + end + ctx.is_toplevel_thunk && emit(ctx, makenode(ctx, ex, K"latestworld")) + elseif k == K"latestworld" + emit(ctx, makeleaf(ctx, ex, K"latestworld")) + elseif k == K"latestworld_if_toplevel" + ctx.is_toplevel_thunk && emit(ctx, makeleaf(ctx, ex, K"latestworld")) + else + throw(LoweringError(ex, "Invalid syntax; $(repr(k))")) + end +end + +function _remove_vars_with_isdefined_check!(vars, ex) + if is_leaf(ex) || is_quoted(ex) + return + elseif kind(ex) == K"isdefined" + delete!(vars, ex[1].var_id) + else + for e in children(ex) + _remove_vars_with_isdefined_check!(vars, e) + end + end +end + +# Find newvar nodes that are unnecessary because +# 1. The variable is not captured and +# 2. The variable is assigned before any branches. +# +# This is used to remove newvar nodes that are not needed for re-initializing +# variables to undefined (see Julia issue #11065). It doesn't look for variable +# *uses*, because any variables used-before-def that also pass this test are +# *always* used undefined, and therefore don't need to be reinitialized. The +# one exception to that is `@isdefined`, which can observe an undefined +# variable without throwing an error. +function unnecessary_newvar_ids(ctx, stmts) + vars = Set{IdTag}() + ids_assigned_before_branch = Set{IdTag}() + for ex in stmts + _remove_vars_with_isdefined_check!(vars, ex) + k = kind(ex) + if k == K"newvar" + id = ex[1].var_id + if !lookup_binding(ctx, id).is_captured + push!(vars, id) + end + elseif k == K"goto" || k == K"gotoifnot" || (k == K"=" && kind(ex[2]) == K"enter") + empty!(vars) + elseif k == K"=" + id = ex[1].var_id + if id in vars + delete!(vars, id) + push!(ids_assigned_before_branch, id) + end + end + end + ids_assigned_before_branch +end + +# flisp: compile-body +function compile_body(ctx, ex) + compile(ctx, ex, true, true) + + # Fix up any symbolic gotos. (We can't do this earlier because the goto + # might precede the label definition in unstructured control flow.) + for origin in ctx.symbolic_jump_origins + name = origin.goto.name_val + target = get(ctx.symbolic_jump_targets, name, nothing) + if isnothing(target) + throw(LoweringError(origin.goto, "label `$name` referenced but not defined")) + end + i = origin.index + pop_ex = compile_pop_exception(ctx, origin.goto, origin.catch_token_stack, + target.catch_token_stack) + if !isnothing(pop_ex) + @assert kind(ctx.code[i]) == K"TOMBSTONE" + ctx.code[i] = pop_ex + i += 1 + end + leave_ex = compile_leave_handler(ctx, origin.goto, origin.handler_token_stack, + target.handler_token_stack) + if !isnothing(leave_ex) + @assert kind(ctx.code[i]) == K"TOMBSTONE" + ctx.code[i] = leave_ex + i += 1 + end + @assert kind(ctx.code[i]) == K"TOMBSTONE" + ctx.code[i] = @ast ctx origin.goto [K"goto" target.label] + end + + # Filter out unnecessary newvar nodes + ids_assigned_before_branch = unnecessary_newvar_ids(ctx, ctx.code) + filter!(ctx.code) do ex + !(kind(ex) == K"newvar" && ex[1].var_id in ids_assigned_before_branch) + end +end + +#------------------------------------------------------------------------------- + +# Recursively renumber an expression within linear IR +# flisp: renumber-stuff +function _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, ex) + k = kind(ex) + if k == K"BindingId" + id = ex.var_id + if haskey(ssa_rewrites, id) + makeleaf(ctx, ex, K"SSAValue"; var_id=ssa_rewrites[id]) + else + new_id = get(slot_rewrites, id, nothing) + binfo = lookup_binding(ctx, id) + if !isnothing(new_id) + sk = binfo.kind == :local || binfo.kind == :argument ? K"slot" : + binfo.kind == :static_parameter ? K"static_parameter" : + throw(LoweringError(ex, "Found unexpected binding of kind $(binfo.kind)")) + makeleaf(ctx, ex, sk; var_id=new_id) + else + if binfo.kind !== :global + throw(LoweringError(ex, "Found unexpected binding of kind $(binfo.kind)")) + end + makeleaf(ctx, ex, K"globalref", binfo.name, mod=binfo.mod) + end + end + elseif k == K"meta" + # Somewhat-hack for Expr(:meta, :generated, gen) which has + # weird top-level semantics for `gen`, but we still need to translate + # the binding it contains to a globalref. + mapchildren(ctx, ex) do e + _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, e) + end + elseif is_literal(k) || is_quoted(k) + ex + elseif k == K"label" + @ast ctx ex label_table[ex.id]::K"label" + elseif k == K"code_info" + ex + else + mapchildren(ctx, ex) do e + _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, e) + end + end +end + +# flisp: renumber-lambda, compact-ir +function renumber_body(ctx, input_code, slot_rewrites) + # Step 1: Remove any assignments to SSA variables, record the indices of labels + ssa_rewrites = Dict{IdTag,IdTag}() + label_table = Dict{Int,Int}() + code = SyntaxList(ctx) + for ex in input_code + k = kind(ex) + ex_out = nothing + if k == K"=" && is_ssa(ctx, ex[1]) + lhs_id = ex[1].var_id + if is_ssa(ctx, ex[2]) + # For SSA₁ = SSA₂, record that all uses of SSA₁ should be replaced by SSA₂ + ssa_rewrites[lhs_id] = ssa_rewrites[ex[2].var_id] + else + # Otherwise, record which `code` index this SSA value refers to + ssa_rewrites[lhs_id] = length(code) + 1 + ex_out = ex[2] + end + elseif k == K"label" + label_table[ex.id] = length(code) + 1 + elseif k == K"TOMBSTONE" + # remove statement + else + ex_out = ex + end + if !isnothing(ex_out) + push!(code, ex_out) + end + end + + # Step 2: + # * Translate any SSA uses and labels into indices in the code table + # * Translate locals into slot indices + for i in 1:length(code) + code[i] = _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, code[i]) + end + code +end + +struct Slot + name::String + kind::Symbol + is_nospecialize::Bool + is_read::Bool + is_single_assign::Bool + is_maybe_undef::Bool + is_called::Bool +end + +function compile_lambda(outer_ctx, ex) + lambda_args = ex[1] + static_parameters = ex[2] + ret_var = numchildren(ex) == 4 ? ex[4] : nothing + # TODO: Add assignments for reassigned arguments to body + lambda_bindings = ex.lambda_bindings + ctx = LinearIRContext(outer_ctx, ex.is_toplevel_thunk, lambda_bindings, ret_var) + compile_body(ctx, ex[3]) + slots = Vector{Slot}() + slot_rewrites = Dict{IdTag,Int}() + for arg in children(lambda_args) + if kind(arg) == K"Placeholder" + # Unused functions arguments like: `_` or `::T` + push!(slots, Slot(arg.name_val, :argument, false, false, false, false, false)) + else + @assert kind(arg) == K"BindingId" + id = arg.var_id + binfo = lookup_binding(ctx, id) + lbinfo = lookup_lambda_binding(ctx, id) + @assert binfo.kind == :local || binfo.kind == :argument + # FIXME: is_single_assign, is_maybe_undef + push!(slots, Slot(binfo.name, :argument, binfo.is_nospecialize, + lbinfo.is_read, false, false, lbinfo.is_called)) + slot_rewrites[id] = length(slots) + end + end + # Sorting the lambda locals is required to remove dependence on Dict iteration order. + for (id, lbinfo) in sort(collect(pairs(lambda_bindings.bindings)), by=first) + if !lbinfo.is_captured + binfo = lookup_binding(ctx.bindings, id) + if binfo.kind == :local + # FIXME: is_single_assign, is_maybe_undef + push!(slots, Slot(binfo.name, :local, false, + lbinfo.is_read, false, false, lbinfo.is_called)) + slot_rewrites[id] = length(slots) + end + end + end + for (i,arg) in enumerate(children(static_parameters)) + @assert kind(arg) == K"BindingId" + id = arg.var_id + info = lookup_binding(ctx.bindings, id) + @assert info.kind == :static_parameter + slot_rewrites[id] = i + end + # @info "" @ast ctx ex [K"block" ctx.code...] + code = renumber_body(ctx, ctx.code, slot_rewrites) + @ast ctx ex [K"code_info"(is_toplevel_thunk=ex.is_toplevel_thunk, + slots=slots) + [K"block"(ex[3]) + code... + ] + ] +end + +""" +This pass converts nested ASTs in the body of a lambda into a list of +statements (ie, Julia's linear/untyped IR). + +Most of the compliexty of this pass is in lowering structured control flow (if, +loops, etc) to gotos and exception handling to enter/leave. We also convert +`K"BindingId"` into K"slot", `K"globalref"` or `K"SSAValue` as appropriate. +""" +function linearize_ir(ctx, ex) + graph = ensure_attributes(ctx.graph, + slots=Vector{Slot}, + mod=Module, + id=Int) + # TODO: Cleanup needed - `_ctx` is just a dummy context here. But currently + # required to call reparent() ... + GraphType = typeof(graph) + _ctx = LinearIRContext(graph, SyntaxList(graph), ctx.bindings, + Ref(0), false, LambdaBindings(), nothing, + Dict{String,JumpTarget{typeof(graph)}}(), + SyntaxList(graph), SyntaxList(graph), + Vector{FinallyHandler{GraphType}}(), + Dict{String, JumpTarget{GraphType}}(), + Vector{JumpOrigin{GraphType}}(), ctx.mod) + res = compile_lambda(_ctx, reparent(_ctx, ex)) + _ctx, res +end + diff --git a/src/vendored/JuliaLowering/src/macro_expansion.jl b/src/vendored/JuliaLowering/src/macro_expansion.jl new file mode 100644 index 00000000..1e4ac756 --- /dev/null +++ b/src/vendored/JuliaLowering/src/macro_expansion.jl @@ -0,0 +1,340 @@ +# Lowering pass 1: Macro expansion, simple normalizations and quote expansion + +""" +A `ScopeLayer` is a mechanism for automatic hygienic macros; every identifier +is assigned to a particular layer and can only match against bindings which are +themselves part of that layer. + +Normal code contains a single scope layer, whereas each macro expansion +generates a new layer. +""" +struct ScopeLayer + id::LayerId + mod::Module + is_macro_expansion::Bool # FIXME +end + +struct MacroExpansionContext{GraphType} <: AbstractLoweringContext + graph::GraphType + bindings::Bindings + scope_layers::Vector{ScopeLayer} + current_layer::ScopeLayer +end + +#-------------------------------------------------- +# Expansion of quoted expressions +function collect_unquoted!(ctx, unquoted, ex, depth) + if kind(ex) == K"$" && depth == 0 + # children(ex) is usually length 1, but for double interpolation it may + # be longer and the children may contain K"..." expressions. Wrapping + # in a tuple groups the arguments together correctly in those cases. + push!(unquoted, @ast ctx ex [K"tuple" children(ex)...]) + else + inner_depth = kind(ex) == K"quote" ? depth + 1 : + kind(ex) == K"$" ? depth - 1 : + depth + for e in children(ex) + collect_unquoted!(ctx, unquoted, e, inner_depth) + end + end + return unquoted +end + +function expand_quote(ctx, ex) + unquoted = SyntaxList(ctx) + collect_unquoted!(ctx, unquoted, ex, 0) + # Unlike user-defined macro expansion, we don't call append_sourceref for + # the entire expression produced by `quote` expansion. We could, but it + # seems unnecessary for `quote` because the surface syntax is a transparent + # representation of the expansion process. However, it's useful to add the + # extra srcref in a more targetted way for $ interpolations inside + # interpolate_ast, so we do that there. + # + # In principle, particular user-defined macros could opt into a similar + # mechanism. + # + # TODO: Should we try adding a srcref to the `quote` node only for the + # extra syntax generated by expand_quote so srcref essentially becomes + # (ex, @HERE) ? + @ast ctx ex [K"call" + interpolate_ast::K"Value" + [K"inert" ex] + unquoted... + ] +end + +#-------------------------------------------------- +struct MacroContext <: AbstractLoweringContext + graph::SyntaxGraph + macrocall::Union{SyntaxTree,LineNumberNode,SourceRef} + scope_layer::ScopeLayer +end + +function adopt_scope(ex, ctx::MacroContext) + adopt_scope(ex, ctx.scope_layer.id) +end + +struct MacroExpansionError + context::Union{Nothing,MacroContext} + ex::SyntaxTree + msg::String + position::Symbol +end + +""" +`position` - the source position relative to the node - may be `:begin` or `:end` or `:all` +""" +function MacroExpansionError(ex::SyntaxTree, msg::AbstractString; position=:all) + MacroExpansionError(nothing, ex, msg, position) +end + +function Base.showerror(io::IO, exc::MacroExpansionError) + print(io, "MacroExpansionError") + ctx = exc.context + if !isnothing(ctx) + print(io, " while expanding ", ctx.macrocall[1], + " in module ", ctx.scope_layer.mod) + end + print(io, ":\n") + # TODO: Display niceties: + # * Show the full provenance tree somehow, in addition to the primary + # source location we're showing here? + # * What if the expression doesn't arise from a source file? + # * How to deal with highlighting trivia? Could provide a token kind or + # child position within the raw tree? How to abstract this?? + src = sourceref(exc.ex) + fb = first_byte(src) + lb = last_byte(src) + pos = exc.position + byterange = pos == :all ? (fb:lb) : + pos == :begin ? (fb:fb-1) : + pos == :end ? (lb+1:lb) : + error("Unknown position $pos") + highlight(io, src.file, byterange, note=exc.msg) +end + +function eval_macro_name(ctx, ex) + # `ex1` might contain a nontrivial mix of scope layers so we can't just + # `eval()` it, as it's already been partially lowered by this point. + # Instead, we repeat the latter parts of `lower()` here. + ex1 = expand_forms_1(ctx, ex) + ctx2, ex2 = expand_forms_2(ctx, ex1) + ctx3, ex3 = resolve_scopes(ctx2, ex2) + ctx4, ex4 = convert_closures(ctx3, ex3) + ctx5, ex5 = linearize_ir(ctx4, ex4) + mod = ctx.current_layer.mod + expr_form = to_lowered_expr(mod, ex5) + eval(mod, expr_form) +end + +function expand_macro(ctx, ex) + @assert kind(ex) == K"macrocall" + + macname = ex[1] + macfunc = eval_macro_name(ctx, macname) + # Macro call arguments may be either + # * Unprocessed by the macro expansion pass + # * Previously processed, but spliced into a further macro call emitted by + # a macro expansion. + # In either case, we need to set any unset scope layers before passing the + # arguments to the macro call. + mctx = MacroContext(ctx.graph, ex, ctx.current_layer) + macro_args = Any[mctx] + for i in 2:numchildren(ex) + push!(macro_args, set_scope_layer(ctx, ex[i], ctx.current_layer.id, false)) + end + macro_invocation_world = Base.get_world_counter() + expanded = try + # TODO: Allow invoking old-style macros for compat + invokelatest(macfunc, macro_args...) + catch exc + if exc isa MacroExpansionError + # Add context to the error. + # TODO: Using rethrow() is kinda ugh. Is there a way to avoid it? + rethrow(MacroExpansionError(mctx, exc.ex, exc.msg, exc.position)) + else + throw(MacroExpansionError(mctx, ex, "Error expanding macro", :all)) + end + end + + if expanded isa SyntaxTree + if !is_compatible_graph(ctx, expanded) + # If the macro has produced syntax outside the macro context, copy it over. + # TODO: Do we expect this always to happen? What is the API for access + # to the macro expansion context? + expanded = copy_ast(ctx, expanded) + end + expanded = append_sourceref(ctx, expanded, ex) + # Module scope for the returned AST is the module where this particular + # method was defined (may be different from `parentmodule(macfunc)`) + mod_for_ast = lookup_method_instance(macfunc, macro_args, macro_invocation_world).def.module + new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod_for_ast, true) + push!(ctx.scope_layers, new_layer) + inner_ctx = MacroExpansionContext(ctx.graph, ctx.bindings, ctx.scope_layers, new_layer) + expanded = expand_forms_1(inner_ctx, expanded) + else + expanded = @ast ctx ex expanded::K"Value" + end + return expanded +end + +# Add a secondary source of provenance to each expression in the tree `ex`. +function append_sourceref(ctx, ex, secondary_prov) + srcref = (ex, secondary_prov) + if !is_leaf(ex) + if kind(ex) == K"macrocall" + makenode(ctx, srcref, ex, children(ex)...) + else + makenode(ctx, srcref, ex, + map(e->append_sourceref(ctx, e, secondary_prov), children(ex))...) + end + else + makeleaf(ctx, srcref, ex) + end +end + +""" +Lowering pass 1 + +This pass contains some simple expansion to make the rest of desugaring easier +to write and expands user defined macros. Macros see the surface syntax, so +need to be dealt with before other lowering. + +* Does identifier normalization +* Strips semantically irrelevant "container" nodes like parentheses +* Expands macros +* Processes quoted syntax turning `K"quote"` into `K"inert"` (eg, expanding + interpolations) +""" +function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) + k = kind(ex) + if k == K"Identifier" + name_str = ex.name_val + if all(==('_'), name_str) + @ast ctx ex ex=>K"Placeholder" + elseif is_ccall_or_cglobal(name_str) + @ast ctx ex name_str::K"core" + else + layerid = get(ex, :scope_layer, ctx.current_layer.id) + makeleaf(ctx, ex, ex, kind=K"Identifier", scope_layer=layerid) + end + elseif k == K"Identifier" || k == K"MacroName" || k == K"StringMacroName" + layerid = get(ex, :scope_layer, ctx.current_layer.id) + makeleaf(ctx, ex, ex, kind=K"Identifier", scope_layer=layerid) + elseif k == K"var" || k == K"char" || k == K"parens" + # Strip "container" nodes + @chk numchildren(ex) == 1 + expand_forms_1(ctx, ex[1]) + elseif k == K"juxtapose" + layerid = get(ex, :scope_layer, ctx.current_layer.id) + @chk numchildren(ex) == 2 + @ast ctx ex [K"call" + "*"::K"Identifier"(scope_layer=layerid) + expand_forms_1(ctx, ex[1]) + expand_forms_1(ctx, ex[2]) + ] + elseif k == K"quote" + @chk numchildren(ex) == 1 + # TODO: Upstream should set a general flag for detecting parenthesized + # expressions so we don't need to dig into `green_tree` here. Ugh! + plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) && + kind(ex[1]) == K"Identifier" && + (sr = sourceref(ex); sr isa SourceRef && kind(sr.green_tree[2]) != K"parens") + if plain_symbol + # As a compromise for compatibility, we treat non-parenthesized + # colon quoted identifiers like `:x` as plain Symbol literals + # because these are ubiquitiously used in Julia programs as ad hoc + # enum-like entities rather than pieces of AST. + @ast ctx ex[1] ex[1]=>K"Symbol" + else + expand_forms_1(ctx, expand_quote(ctx, ex[1])) + end + elseif k == K"macrocall" + expand_macro(ctx, ex) + elseif k == K"module" || k == K"toplevel" || k == K"inert" + ex + elseif k == K"." && numchildren(ex) == 2 + e2 = expand_forms_1(ctx, ex[2]) + if kind(e2) == K"Identifier" || kind(e2) == K"Placeholder" + # FIXME: Do the K"Symbol" transformation in the parser?? + e2 = @ast ctx e2 e2=>K"Symbol" + end + @ast ctx ex [K"." expand_forms_1(ctx, ex[1]) e2] + elseif (k == K"call" || k == K"dotcall") + # Do some initial desugaring of call and dotcall here to simplify + # the later desugaring pass + args = SyntaxList(ctx) + if is_infix_op_call(ex) || is_postfix_op_call(ex) + @chk numchildren(ex) >= 2 "Postfix/infix operators must have at least two positional arguments" + farg = ex[2] + push!(args, ex[1]) + append!(args, ex[3:end]) + else + @chk numchildren(ex) > 0 "Call expressions must have a function name" + farg = ex[1] + append!(args, ex[2:end]) + end + if !isempty(args) + if kind(args[end]) == K"do" + # move do block into first argument location + pushfirst!(args, pop!(args)) + end + end + if length(args) == 2 && is_same_identifier_like(farg, "^") && kind(args[2]) == K"Integer" + # Do literal-pow expansion here as it's later used in both call and + # dotcall expansion. + @ast ctx ex [k + "literal_pow"::K"top" + expand_forms_1(ctx, farg) + expand_forms_1(ctx, args[1]) + [K"call" + [K"call" + "apply_type"::K"core" + "Val"::K"top" + args[2] + ] + ] + ] + else + if kind(farg) == K"." && numchildren(farg) == 1 + # (.+)(x,y) is treated as a dotcall + k = K"dotcall" + farg = farg[1] + end + # Preserve call type flags (mostly ignored in the next pass as + # we've already reordered arguments.) + callflags = JuliaSyntax.call_type_flags(ex) + @ast ctx ex [k(syntax_flags=(callflags == 0 ? nothing : callflags)) + expand_forms_1(ctx, farg) + (expand_forms_1(ctx, a) for a in args)... + ] + end + elseif is_leaf(ex) + ex + elseif k == K"<:" || k == K">:" || k == K"-->" + # TODO: Should every form get layerid systematically? Or only the ones + # which expand_forms_2 needs? + layerid = get(ex, :scope_layer, ctx.current_layer.id) + mapchildren(e->expand_forms_1(ctx,e), ctx, ex; scope_layer=layerid) + else + mapchildren(e->expand_forms_1(ctx,e), ctx, ex) + end +end + +function expand_forms_1(mod::Module, ex::SyntaxTree) + graph = ensure_attributes(syntax_graph(ex), + var_id=IdTag, + scope_layer=LayerId, + __macro_ctx__=Nothing, + meta=CompileHints) + layers = ScopeLayer[ScopeLayer(1, mod, false)] + ctx = MacroExpansionContext(graph, Bindings(), layers, layers[1]) + ex2 = expand_forms_1(ctx, reparent(ctx, ex)) + graph2 = delete_attributes(graph, :__macro_ctx__) + # TODO: Returning the context with pass-specific mutable data is a bad way + # to carry state into the next pass. + ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, + ctx.current_layer) + return ctx2, reparent(ctx2, ex2) +end + diff --git a/src/vendored/JuliaLowering/src/runtime.jl b/src/vendored/JuliaLowering/src/runtime.jl new file mode 100644 index 00000000..23252400 --- /dev/null +++ b/src/vendored/JuliaLowering/src/runtime.jl @@ -0,0 +1,416 @@ +# Runtime support for +# 1. Functions called by the code emitted from lowering +# 2. Introspecting Julia's state during lowering +# +# These should probably all move to `Core` at some point. + +#------------------------------------------------------------------------------- +# Functions/types used by code emitted from lowering, but not called by it directly + +# Return the current exception. In JuliaLowering we use this rather than the +# special form `K"the_exception"` to reduces the number of special forms. +Base.@assume_effects :removable :nothrow function current_exception() + @ccall jl_current_exception(current_task()::Any)::Any +end + +#-------------------------------------------------- +# Supporting functions for AST interpolation (`quote`) +struct InterpolationContext{Graph} <: AbstractLoweringContext + graph::Graph + values::Tuple + current_index::Ref{Int} +end + +function _contains_active_interp(ex, depth) + k = kind(ex) + if k == K"$" && depth == 0 + return true + end + inner_depth = k == K"quote" ? depth + 1 : + k == K"$" ? depth - 1 : + depth + return any(_contains_active_interp(c, inner_depth) for c in children(ex)) +end + +# Produce interpolated node for `$x` syntax +function _interpolated_value(ctx, srcref, ex) + if ex isa SyntaxTree + if !is_compatible_graph(ctx, ex) + ex = copy_ast(ctx, ex) + end + append_sourceref(ctx, ex, srcref) + else + makeleaf(ctx, srcref, K"Value", ex) + end +end + +function _interpolate_ast(ctx::InterpolationContext, ex, depth) + if ctx.current_index[] > length(ctx.values) || !_contains_active_interp(ex, depth) + return ex + end + + # We have an interpolation deeper in the tree somewhere - expand to an + # expression + inner_depth = kind(ex) == K"quote" ? depth + 1 : + kind(ex) == K"$" ? depth - 1 : + depth + expanded_children = SyntaxList(ctx) + for e in children(ex) + if kind(e) == K"$" && inner_depth == 0 + vals = ctx.values[ctx.current_index[]]::Tuple + ctx.current_index[] += 1 + for (i,v) in enumerate(vals) + srcref = numchildren(e) == 1 ? e : e[i] + push!(expanded_children, _interpolated_value(ctx, srcref, v)) + end + else + push!(expanded_children, _interpolate_ast(ctx, e, inner_depth)) + end + end + + makenode(ctx, ex, head(ex), expanded_children) +end + +function interpolate_ast(ex, values...) + # Construct graph for interpolation context. We inherit this from the macro + # context where possible by detecting it using __macro_ctx__. This feels + # hacky though. + # + # Perhaps we should use a ScopedValue for this instead or get it from + # the macro __context__? None of the options feel great here. + graph = nothing + for vals in values + for v in vals + if v isa SyntaxTree && hasattr(syntax_graph(v), :__macro_ctx__) + graph = syntax_graph(v) + break + end + end + end + if isnothing(graph) + graph = ensure_attributes(SyntaxGraph(), kind=Kind, syntax_flags=UInt16, source=SourceAttrType, + value=Any, name_val=String, scope_layer=LayerId) + end + ctx = InterpolationContext(graph, values, Ref(1)) + # We must copy the AST into our context to use it as the source reference + # of generated expressions. + ex1 = copy_ast(ctx, ex) + if kind(ex1) == K"$" + @assert length(values) == 1 + vs = values[1] + if length(vs) > 1 + # :($($(xs...))) where xs is more than length 1 + throw(LoweringError(ex1, "More than one value in bare `\$` expression")) + end + _interpolated_value(ctx, ex1, only(vs)) + else + _interpolate_ast(ctx, ex1, 0) + end +end + +#-------------------------------------------------- +# Functions called by closure conversion +function eval_closure_type(mod, closure_type_name, field_names, field_is_box) + type_params = Core.TypeVar[] + field_types = [] + for (name, isbox) in zip(field_names, field_is_box) + if !isbox + T = Core.TypeVar(Symbol(name, "_type")) + push!(type_params, T) + push!(field_types, T) + else + push!(field_types, Core.Box) + end + end + type = Core._structtype(mod, closure_type_name, + Core.svec(type_params...), + Core.svec(field_names...), + Core.svec(), + false, + length(field_names)) + Core._setsuper!(type, Core.Function) + Base.eval(mod, :(const $closure_type_name = $type)) + Core._typebody!(false, type, Core.svec(field_types...)) + type +end + +# Interpolate captured local variables into the CodeInfo for a global method +function replace_captured_locals!(codeinfo, locals) + for (i, ex) in enumerate(codeinfo.code) + if Meta.isexpr(ex, :captured_local) + codeinfo.code[i] = locals[ex.args[1]] + end + end + codeinfo +end + +#-------------------------------------------------- +# Functions which create modules or mutate their bindings + +# Construct new bare module including only the "default names" +# +# using Core +# const modname = modval +# public modname +# +# And run statments in the toplevel expression `body` +function eval_module(parentmod, modname, body) + # Here we just use `eval()` with an Expr. + # If we wanted to avoid this we'd need to reproduce a lot of machinery from + # jl_eval_module_expr() + # + # 1. Register / deparent toplevel modules + # 2. Set binding in parent module + # 3. Deal with replacing modules + # * Warn if replacing + # * Root old module being replaced + # 4. Run __init__ + # * Also run __init__ for any children after parent is defined + # mod = @ccall jl_new_module(Symbol(modname)::Symbol, parentmod::Module)::Any + # ... + name = Symbol(modname) + eval(parentmod, :( + baremodule $name + $eval($name, $body) + end + )) +end + +# Evaluate content of `import` or `using` statement +function module_import(into_mod::Module, is_using::Bool, + from_mod::Union{Nothing,Core.SimpleVector}, paths::Core.SimpleVector) + # For now, this function converts our lowered representation back to Expr + # and calls eval() to avoid replicating all of the fiddly logic in + # jl_toplevel_eval_flex. + # TODO: ccall Julia runtime functions directly? + # * jl_module_using jl_module_use_as + # * import_module jl_module_import_as + path_args = [] + i = 1 + while i < length(paths) + nsyms = paths[i]::Int + n = i + nsyms + path = Expr(:., [Symbol(paths[i+j]::String) for j = 1:nsyms]...) + as_name = paths[i+nsyms+1] + push!(path_args, isnothing(as_name) ? path : + Expr(:as, path, Symbol(as_name))) + i += nsyms + 2 + end + ex = if isnothing(from_mod) + Expr(is_using ? :using : :import, + path_args...) + else + from_path = Expr(:., [Symbol(s::String) for s in from_mod]...) + Expr(is_using ? :using : :import, + Expr(:(:), from_path, path_args...)) + end + eval(into_mod, ex) + nothing +end + +function module_public(mod::Module, is_exported::Bool, identifiers...) + # symbol jl_module_public is no longer exported as of #57765 + eval(mod, Expr((is_exported ? :export : :public), map(Symbol, identifiers)...)) +end + +#-------------------------------------------------- +# Docsystem integration +function _bind_func_docs!(f, docstr, method_metadata::Core.SimpleVector) + mod = parentmodule(f) + bind = Base.Docs.Binding(mod, nameof(f)) + full_sig = method_metadata[1] + arg_sig = Tuple{full_sig[2:end]...} + lineno = method_metadata[3] + metadata = Dict{Symbol, Any}( + :linenumber => lineno.line, + :module => mod, + ) + if !isnothing(lineno.file) + push!(metadata, :path => string(lineno.file)) + end + Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), arg_sig) +end + +function bind_docs!(f::Function, docstr, method_metadata::Core.SimpleVector) + _bind_func_docs!(f, docstr, method_metadata) +end + +# Document constructors +function bind_docs!(::Type{Type{T}}, docstr, method_metadata::Core.SimpleVector) where T + _bind_func_docs!(T, docstr, method_metadata) +end + +function bind_docs!(type::Type, docstr, method_metadata::Core.SimpleVector) + _bind_func_docs!(type, docstr, method_metadata) +end + +function bind_docs!(type::Type, docstr, lineno::LineNumberNode; field_docs=Core.svec()) + mod = parentmodule(type) + bind = Base.Docs.Binding(mod, nameof(type)) + metadata = Dict{Symbol, Any}( + :linenumber => lineno, + :module => mod, + ) + if !isnothing(lineno.file) + push!(metadata, :path => string(lineno.file)) + end + if !isempty(field_docs) + fd = Dict{Symbol, Any}() + fns = fieldnames(type) + for i = 1:2:length(field_docs) + fd[fns[field_docs[i]]] = field_docs[i+1] + end + metadata[:fields] = fd + end + Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), Union{}) +end + +#-------------------------------------------------- +# Runtime support infrastructure for `@generated` + +# An alternative to Core.GeneratedFunctionStub which works on SyntaxTree rather +# than Expr. +struct GeneratedFunctionStub + gen + srcref + argnames::Core.SimpleVector + spnames::Core.SimpleVector +end + +# Call the `@generated` code generator function and wrap the results of the +# expression into a CodeInfo. +# +# `args` passed into stub by the Julia runtime are (parent_func, static_params..., arg_types...) +function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize args...) + # Some of the lowering pipeline from lower() and the pass-specific setup is + # re-implemented here because generated functions are very much (but not + # entirely) like macro expansion. + # + # TODO: Reduce duplication where possible. + + mod = parentmodule(g.gen) + + # Attributes from parsing + graph = ensure_attributes(SyntaxGraph(), kind=Kind, syntax_flags=UInt16, source=SourceAttrType, + value=Any, name_val=String) + + # Attributes for macro expansion + graph = ensure_attributes(graph, + var_id=IdTag, + scope_layer=LayerId, + __macro_ctx__=Nothing, + meta=CompileHints, + # Additional attribute for resolve_scopes, for + # adding our custom lambda below + is_toplevel_thunk=Bool + ) + + # Macro expansion + layers = ScopeLayer[ScopeLayer(1, mod, false)] + ctx1 = MacroExpansionContext(graph, Bindings(), layers, layers[1]) + + # Run code generator - this acts like a macro expander and like a macro + # expander it gets a MacroContext. + mctx = MacroContext(syntax_graph(ctx1), g.srcref, layers[1]) + ex0 = g.gen(mctx, args...) + if ex0 isa SyntaxTree + if !is_compatible_graph(ctx1, ex0) + # If the macro has produced syntax outside the macro context, copy it over. + # TODO: Do we expect this always to happen? What is the API for access + # to the macro expansion context? + ex0 = copy_ast(ctx1, ex0) + end + else + ex0 = @ast ctx ex expanded::K"Value" + end + # Expand any macros emitted by the generator + ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0)) + ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__), + ctx1.bindings, ctx1.scope_layers, ctx1.current_layer) + ex1 = reparent(ctx1, ex1) + + # Desugaring + ctx2, ex2 = expand_forms_2( ctx1, ex1) + + # Wrap expansion in a non-toplevel lambda and run scope resolution + ex2 = @ast ctx2 ex0 [K"lambda"(is_toplevel_thunk=false) + [K"block" + (string(n)::K"Identifier" for n in g.argnames)... + ] + [K"block" + (string(n)::K"Identifier" for n in g.spnames)... + ] + ex2 + ] + ctx3, ex3 = resolve_scopes( ctx2, ex2) + + + # Rest of lowering + ctx4, ex4 = convert_closures(ctx3, ex3) + ctx5, ex5 = linearize_ir( ctx4, ex4) + ci = to_lowered_expr(mod, ex5) + @assert ci isa Core.CodeInfo + return ci +end + + +#------------------------------------------------------------------------------- +# The following functions are called directly by lowering to inspect Julia's state. + +# Get the binding for `name` if one is already resolved in module `mod`. Note +# that we cannot use `isdefined(::Module, ::Symbol)` here, because that causes +# binding resolution which is a massive side effect we must avoid in lowering. +function _get_module_binding(mod, name; create=false) + b = @ccall jl_get_module_binding(mod::Module, name::Symbol, create::Cint)::Ptr{Core.Binding} + b == C_NULL ? nothing : unsafe_pointer_to_objref(b) +end + +# Return true if a `name` is defined in and *by* the module `mod`. +# Has no side effects, unlike isdefined() +# +# (This should do what fl_defined_julia_global does for flisp lowering) +function is_defined_and_owned_global(mod, name) + Base.binding_kind(mod, name) === Base.PARTITION_KIND_GLOBAL +end + +# "Reserve" a binding: create the binding if it doesn't exist but do not assign +# to it. +function reserve_module_binding(mod, name) + # TODO: Fix the race condition here: We should really hold the Module's + # binding lock during this test-and-set type operation. But the binding + # lock is only accessible from C. See also the C code in + # `fl_module_unique_name`. + if _get_module_binding(mod, name; create=false) === nothing + _get_module_binding(mod, name; create=true) !== nothing + else + return false + end +end + +# Reserve a global binding named "$basename#$i" in module `mod` for the +# smallest `i` starting at `0`. +# +# TODO: Remove the use of this where possible. Currently this is used within +# lowering to create unique global names for keyword function bodies and +# closure types as a more local alternative to current-julia-module-counter. +# However, we should ideally defer it to eval-time to make lowering itself +# completely non-mutating. +function reserve_module_binding_i(mod, basename) + i = 0 + while true + name = "$basename$i" + if reserve_module_binding(mod, Symbol(name)) + return name + end + i += 1 + end +end + +function lookup_method_instance(func, args, world::Integer) + allargs = Vector{Any}(undef, length(args) + 1) + allargs[1] = func + allargs[2:end] = args + mi = @ccall jl_method_lookup(allargs::Ptr{Any}, length(allargs)::Csize_t, + world::Csize_t)::Ptr{Cvoid} + return mi == C_NULL ? nothing : unsafe_pointer_to_objref(mi) +end diff --git a/src/vendored/JuliaLowering/src/scope_analysis.jl b/src/vendored/JuliaLowering/src/scope_analysis.jl new file mode 100644 index 00000000..59314149 --- /dev/null +++ b/src/vendored/JuliaLowering/src/scope_analysis.jl @@ -0,0 +1,780 @@ +# Lowering pass 3: scope and variable analysis + +""" +Key to use when transforming names into bindings +""" +struct NameKey + name::String + layer::LayerId +end + +function Base.isless(a::NameKey, b::NameKey) + (a.name, a.layer) < (b.name, b.layer) +end + +# Identifiers produced by lowering will have the following layer by default. +# +# To make new mutable variables without colliding names, lowering can +# - generate new var_id's directly (like the gensyms used by the old system) +# - create additional layers, though this may be unnecessary +const _lowering_internal_layer = -1 + +function NameKey(ex::SyntaxTree) + @chk kind(ex) == K"Identifier" + NameKey(ex.name_val, get(ex, :scope_layer, _lowering_internal_layer)) +end + +#------------------------------------------------------------------------------- +_insert_if_not_present!(dict, key, val) = get!(dict, key, val) + +function _find_scope_vars!(ctx, assignments, locals, destructured_args, globals, used_names, used_bindings, ex) + k = kind(ex) + if k == K"Identifier" + _insert_if_not_present!(used_names, NameKey(ex), ex) + elseif k == K"BindingId" + push!(used_bindings, ex.var_id) + elseif is_leaf(ex) || is_quoted(k) || + k in KSet"scope_block lambda module toplevel" + return + elseif k == K"local" + if getmeta(ex, :is_destructured_arg, false) + push!(destructured_args, ex[1]) + else + _insert_if_not_present!(locals, NameKey(ex[1]), ex) + end + elseif k == K"global" + _insert_if_not_present!(globals, NameKey(ex[1]), ex) + elseif k == K"assign_const_if_global" + # like v = val, except that if `v` turns out global(either implicitly or + # by explicit `global`), it gains an implicit `const` + _insert_if_not_present!(assignments, NameKey(ex[1]), ex) + elseif k == K"=" || k == K"constdecl" + v = decl_var(ex[1]) + if !(kind(v) in KSet"BindingId globalref Placeholder") + _insert_if_not_present!(assignments, NameKey(v), v) + end + _find_scope_vars!(ctx, assignments, locals, destructured_args, globals, used_names, used_bindings, ex[2]) + elseif k == K"function_decl" + v = ex[1] + kv = kind(v) + if kv == K"Identifier" + _insert_if_not_present!(assignments, NameKey(v), v) + elseif kv == K"BindingId" + binfo = lookup_binding(ctx, v) + if !binfo.is_ssa && binfo.kind != :global + @assert false "allow local BindingId as function name?" + end + else + @assert false + end + else + for e in children(ex) + _find_scope_vars!(ctx, assignments, locals, destructured_args, globals, used_names, used_bindings, e) + end + end +end + +# Find names of all identifiers used in the given expression, grouping them +# into sets by type of usage. +# +# NB: This only works propery after desugaring +function find_scope_vars(ctx, ex) + ExT = typeof(ex) + assignments = Dict{NameKey,ExT}() + locals = Dict{NameKey,ExT}() + destructured_args = Vector{ExT}() + globals = Dict{NameKey,ExT}() + used_names = Dict{NameKey,ExT}() + used_bindings = Set{IdTag}() + for e in children(ex) + _find_scope_vars!(ctx, assignments, locals, destructured_args, globals, used_names, used_bindings, e) + end + + # Sort by key so that id generation is deterministic + assignments = sort!(collect(pairs(assignments)), by=first) + locals = sort!(collect(pairs(locals)), by=first) + globals = sort!(collect(pairs(globals)), by=first) + used_names = sort!(collect(pairs(used_names)), by=first) + used_bindings = sort!(collect(used_bindings)) + + return assignments, locals, destructured_args, globals, used_names, used_bindings +end + +struct ScopeInfo + # True if scope is the global top level scope + is_toplevel_global_scope::Bool + # True if scope is part of top level code, or a non-lambda scope nested + # inside top level code. Thus requiring special scope resolution rules. + in_toplevel_thunk::Bool + # Soft/hard scope. For top level thunks only + is_soft::Bool + is_hard::Bool + # Map from variable names to IDs which appear in this scope but not in the + # parent scope + # TODO: Rename to `locals` or local_bindings? + var_ids::Dict{NameKey,IdTag} + # Bindings used by the enclosing lambda + lambda_bindings::LambdaBindings +end + +struct ScopeResolutionContext{GraphType} <: AbstractLoweringContext + graph::GraphType + bindings::Bindings + mod::Module + scope_layers::Vector{ScopeLayer} + # name=>id mappings for all discovered global vars + global_vars::Dict{NameKey,IdTag} + # Stack of name=>id mappings for each scope, innermost scope last. + scope_stack::Vector{ScopeInfo} + # Variables which were implicitly global due to being assigned to in top + # level code + implicit_toplevel_globals::Set{NameKey} +end + +function ScopeResolutionContext(ctx) + graph = ensure_attributes(ctx.graph, lambda_bindings=LambdaBindings) + ScopeResolutionContext(graph, + ctx.bindings, + ctx.mod, + ctx.scope_layers, + Dict{NameKey,IdTag}(), + Vector{ScopeInfo}(), + Set{NameKey}()) +end + +function current_lambda_bindings(ctx::ScopeResolutionContext) + last(ctx.scope_stack).lambda_bindings +end + +function lookup_var(ctx, varkey::NameKey, exclude_toplevel_globals=false) + for i in lastindex(ctx.scope_stack):-1:1 + ids = ctx.scope_stack[i].var_ids + id = get(ids, varkey, nothing) + if !isnothing(id) && (!exclude_toplevel_globals || + i > 1 || lookup_binding(ctx, id).kind != :global) + return id + end + end + return exclude_toplevel_globals ? nothing : get(ctx.global_vars, varkey, nothing) +end + +function var_kind(ctx, id::IdTag) + lookup_binding(ctx, id).kind +end + +function var_kind(ctx, varkey::NameKey, exclude_toplevel_globals=false) + id = lookup_var(ctx, varkey, exclude_toplevel_globals) + isnothing(id) ? nothing : lookup_binding(ctx, id).kind +end + +function init_binding(ctx, srcref, varkey::NameKey, kind::Symbol; kws...) + id = kind === :global ? get(ctx.global_vars, varkey, nothing) : nothing + if isnothing(id) + mod = kind === :global ? ctx.scope_layers[varkey.layer].mod : nothing + ex = new_binding(ctx, srcref, varkey.name, kind; mod=mod, kws...) + id = ex.var_id + end + if kind === :global + ctx.global_vars[varkey] = id + end + id +end + +# Add lambda arguments and static parameters +function add_lambda_args(ctx, var_ids, args, args_kind) + for arg in args + ka = kind(arg) + if ka == K"Identifier" + varkey = NameKey(arg) + if haskey(var_ids, varkey) + vk = lookup_binding(ctx, var_ids[varkey]).kind + _is_arg(k) = k == :argument || k == :local + msg = _is_arg(vk) && _is_arg(args_kind) ? "function argument name not unique" : + vk == :static_parameter && args_kind == :static_parameter ? "function static parameter name not unique" : + "static parameter name not distinct from function argument" + throw(LoweringError(arg, msg)) + end + is_always_defined = args_kind == :argument || args_kind == :static_parameter + id = init_binding(ctx, arg, varkey, args_kind; + is_nospecialize=getmeta(arg, :nospecialize, false), + is_always_defined=is_always_defined) + var_ids[varkey] = id + elseif ka != K"BindingId" && ka != K"Placeholder" + throw(LoweringError(arg, "Unexpected lambda arg kind")) + end + end +end + +# Analyze identifier usage within a scope +# * Allocate a new binding for each identifier which the scope introduces. +# * Record the identifier=>binding mapping in a lookup table +# * Return a `ScopeInfo` with the mapping plus additional scope metadata +function analyze_scope(ctx, ex, scope_type, is_toplevel_global_scope=false, + lambda_args=nothing, lambda_static_parameters=nothing) + parentscope = isempty(ctx.scope_stack) ? nothing : ctx.scope_stack[end] + is_outer_lambda_scope = kind(ex) == K"lambda" + in_toplevel_thunk = is_toplevel_global_scope || + (!is_outer_lambda_scope && parentscope.in_toplevel_thunk) + + assignments, locals, destructured_args, globals, + used_names, used_bindings = find_scope_vars(ctx, ex) + + # Construct a mapping from identifiers to bindings + # + # This will contain a binding ID for each variable which is introduced by + # the scope, including + # * Explicit locals + # * Explicit globals + # * Implicit locals created by assignment + var_ids = Dict{NameKey,IdTag}() + + if !isnothing(lambda_args) + add_lambda_args(ctx, var_ids, lambda_args, :argument) + add_lambda_args(ctx, var_ids, lambda_static_parameters, :static_parameter) + add_lambda_args(ctx, var_ids, destructured_args, :local) + end + + # Add explicit locals + for (varkey,e) in locals + if haskey(var_ids, varkey) + vk = lookup_binding(ctx, var_ids[varkey]).kind + if vk === :argument && is_outer_lambda_scope + throw(LoweringError(e, "local variable name `$(varkey.name)` conflicts with an argument")) + elseif vk === :static_parameter + throw(LoweringError(e, "local variable name `$(varkey.name)` conflicts with a static parameter")) + end + elseif var_kind(ctx, varkey) === :static_parameter + throw(LoweringError(e, "local variable name `$(varkey.name)` conflicts with a static parameter")) + else + var_ids[varkey] = init_binding(ctx, e[1], varkey, :local) + end + end + + # Add explicit globals + for (varkey,e) in globals + if haskey(var_ids, varkey) + vk = lookup_binding(ctx, var_ids[varkey]).kind + if vk === :local + throw(LoweringError(e, "Variable `$(varkey.name)` declared both local and global")) + elseif vk === :argument && is_outer_lambda_scope + throw(LoweringError(e, "global variable name `$(varkey.name)` conflicts with an argument")) + elseif vk === :static_parameter + throw(LoweringError(e, "global variable name `$(varkey.name)` conflicts with a static parameter")) + end + elseif var_kind(ctx, varkey) === :static_parameter + throw(LoweringError(e, "global variable name `$(varkey.name)` conflicts with a static parameter")) + end + var_ids[varkey] = init_binding(ctx, e[1], varkey, :global) + end + + # Compute implicit locals and globals + if is_toplevel_global_scope + is_hard_scope = false + is_soft_scope = false + + # Assignments are implicitly global at top level, unless they come from + # a macro expansion + for (varkey,e) in assignments + vk = haskey(var_ids, varkey) ? + lookup_binding(ctx, var_ids[varkey]).kind : + var_kind(ctx, varkey, true) + if vk === nothing + if ctx.scope_layers[varkey.layer].is_macro_expansion + var_ids[varkey] = init_binding(ctx, e, varkey, :local) + else + init_binding(ctx, e, varkey, :global) + push!(ctx.implicit_toplevel_globals, varkey) + end + end + end + else + is_hard_scope = in_toplevel_thunk && (parentscope.is_hard || scope_type === :hard) + is_soft_scope = in_toplevel_thunk && !is_hard_scope && + (scope_type === :neutral ? parentscope.is_soft : scope_type === :soft) + + # Outside top level code, most assignments create local variables implicitly + for (varkey,e) in assignments + vk = haskey(var_ids, varkey) ? + lookup_binding(ctx, var_ids[varkey]).kind : + var_kind(ctx, varkey, true) + if vk === :static_parameter + throw(LoweringError(e, "local variable name `$(varkey.name)` conflicts with a static parameter")) + elseif vk !== nothing + continue + end + # Assignment is to a newly discovered variable name + is_ambiguous_local = false + if in_toplevel_thunk && !is_hard_scope + # In a top level thunk but *inside* a nontrivial scope + layer = ctx.scope_layers[varkey.layer] + if !layer.is_macro_expansion && (varkey in ctx.implicit_toplevel_globals || + is_defined_and_owned_global(layer.mod, Symbol(varkey.name))) + # Special scope rules to make assignments to globals work + # like assignments to locals do inside a function. + if is_soft_scope + # Soft scope (eg, for loop in REPL) => treat as a global + init_binding(ctx, e, varkey, :global) + continue + else + # Ambiguous case (eg, nontrivial scopes in package top level code) + # => Treat as local but generate warning when assigned to + is_ambiguous_local = true + end + end + end + var_ids[varkey] = init_binding(ctx, e, varkey, :local; + is_ambiguous_local=is_ambiguous_local) + end + end + + #-------------------------------------------------- + # At this point we've discovered all the bindings defined in this scope and + # added them to `var_ids`. + # + # Next we record information about how the new bindings relate to the + # enclosing lambda + # * All non-globals are recorded (kind :local and :argument will later be turned into slots) + # * Captured variables are detected and recorded + # + # TODO: Move most or-all of this to the VariableAnalysis sub-pass + lambda_bindings = if is_outer_lambda_scope + if isempty(lambda_args) + LambdaBindings() + else + selfarg = first(lambda_args) + selfid = kind(selfarg) == K"BindingId" ? + selfarg.var_id : var_ids[NameKey(selfarg)] + LambdaBindings(selfid) + end + else + parentscope.lambda_bindings + end + + for id in values(var_ids) + binfo = lookup_binding(ctx, id) + if !binfo.is_ssa && binfo.kind !== :global + init_lambda_binding(lambda_bindings, id) + end + end + + # FIXME: This assumes used bindings are internal to the lambda and cannot + # be from the environment, and also assumes they are assigned. That's + # correct for now but in general we should go by the same code path that + # identifiers do. + for id in used_bindings + binfo = lookup_binding(ctx, id) + if (binfo.kind === :local && !binfo.is_ssa) || binfo.kind === :argument || + binfo.kind === :static_parameter + if !has_lambda_binding(lambda_bindings, id) + init_lambda_binding(lambda_bindings, id) + end + end + end + + for (varkey, e) in used_names + id = haskey(var_ids, varkey) ? var_ids[varkey] : lookup_var(ctx, varkey) + if id === nothing + # Identifiers which are used but not defined in some scope are + # newly discovered global bindings + init_binding(ctx, e, varkey, :global) + elseif !in_toplevel_thunk + binfo = lookup_binding(ctx, id) + if binfo.kind !== :global + if !has_lambda_binding(lambda_bindings, id) + # Used vars from a scope *outside* the current lambda are captured + init_lambda_binding(lambda_bindings, id, is_captured=true) + update_binding!(ctx, id; is_captured=true) + end + end + end + end + + if !in_toplevel_thunk + for (varkey,_) in assignments + id = haskey(var_ids, varkey) ? var_ids[varkey] : lookup_var(ctx, varkey) + binfo = lookup_binding(ctx, id) + if binfo.kind !== :global + if !has_lambda_binding(lambda_bindings, id) + # Assigned vars from a scope *outside* the current lambda are captured + init_lambda_binding(lambda_bindings, id, is_captured=true) + update_binding!(ctx, id; is_captured=true) + end + end + end + end + + return ScopeInfo(is_toplevel_global_scope, in_toplevel_thunk, is_soft_scope, + is_hard_scope, var_ids, lambda_bindings) +end + +function add_local_decls!(ctx, stmts, srcref, scope) + # Add local decls to start of block so that closure conversion can + # initialize if necessary. + for id in sort!(collect(values(scope.var_ids))) + binfo = lookup_binding(ctx, id) + if binfo.kind == :local + push!(stmts, @ast ctx srcref [K"local" binding_ex(ctx, id)]) + end + end +end + +function _resolve_scopes(ctx, ex::SyntaxTree) + k = kind(ex) + if k == K"Identifier" + @ast ctx ex lookup_var(ctx, NameKey(ex))::K"BindingId" + elseif is_leaf(ex) || is_quoted(ex) || k == K"toplevel" + ex + # elseif k == K"global" + # ex + elseif k == K"local" + makeleaf(ctx, ex, K"TOMBSTONE") + elseif k == K"decl" + ex_out = mapchildren(e->_resolve_scopes(ctx, e), ctx, ex) + name = ex_out[1] + if kind(name) != K"Placeholder" + binfo = lookup_binding(ctx, name) + if binfo.kind == :global && !ctx.scope_stack[end].in_toplevel_thunk + throw(LoweringError(ex, "type declarations for global variables must be at top level, not inside a function")) + end + end + id = ex_out[1] + if kind(id) != K"Placeholder" + binfo = lookup_binding(ctx, id) + if !isnothing(binfo.type) + throw(LoweringError(ex, "multiple type declarations found for `$(binfo.name)`")) + end + update_binding!(ctx, id; type=ex_out[2]) + end + ex_out + elseif k == K"always_defined" + id = lookup_var(ctx, NameKey(ex[1])) + update_binding!(ctx, id; is_always_defined=true) + makeleaf(ctx, ex, K"TOMBSTONE") + elseif k == K"lambda" + is_toplevel_thunk = ex.is_toplevel_thunk + scope = analyze_scope(ctx, ex, nothing, is_toplevel_thunk, + children(ex[1]), children(ex[2])) + + push!(ctx.scope_stack, scope) + arg_bindings = _resolve_scopes(ctx, ex[1]) + sparm_bindings = _resolve_scopes(ctx, ex[2]) + body_stmts = SyntaxList(ctx) + add_local_decls!(ctx, body_stmts, ex, scope) + body = _resolve_scopes(ctx, ex[3]) + if kind(body) == K"block" + append!(body_stmts, children(body)) + else + push!(body_stmts, body) + end + ret_var = numchildren(ex) == 4 ? _resolve_scopes(ctx, ex[4]) : nothing + pop!(ctx.scope_stack) + + @ast ctx ex [K"lambda"(lambda_bindings=scope.lambda_bindings, + is_toplevel_thunk=is_toplevel_thunk) + arg_bindings + sparm_bindings + [K"block" + body_stmts... + ] + ret_var + ] + elseif k == K"scope_block" + scope = analyze_scope(ctx, ex, ex.scope_type) + push!(ctx.scope_stack, scope) + stmts = SyntaxList(ctx) + add_local_decls!(ctx, stmts, ex, scope) + for e in children(ex) + push!(stmts, _resolve_scopes(ctx, e)) + end + pop!(ctx.scope_stack) + @ast ctx ex [K"block" stmts...] + elseif k == K"extension" + etype = extension_type(ex) + if etype == "islocal" + id = lookup_var(ctx, NameKey(ex[2])) + islocal = !isnothing(id) && var_kind(ctx, id) != :global + @ast ctx ex islocal::K"Bool" + elseif etype == "locals" + stmts = SyntaxList(ctx) + locals_dict = ssavar(ctx, ex, "locals_dict") + push!(stmts, @ast ctx ex [K"=" + locals_dict + [K"call" + [K"call" + "apply_type"::K"core" + "Dict"::K"top" + "Symbol"::K"core" + "Any"::K"core" + ] + ] + ]) + for scope in ctx.scope_stack + for id in values(scope.var_ids) + binfo = lookup_binding(ctx, id) + if binfo.kind == :global || binfo.is_internal + continue + end + binding = binding_ex(ctx, id) + push!(stmts, @ast ctx ex [K"if" + [K"isdefined" binding] + [K"call" + "setindex!"::K"top" + locals_dict + binding + binfo.name::K"Symbol" + ] + ]) + end + end + push!(stmts, locals_dict) + makenode(ctx, ex, K"block", stmts) + end + elseif k == K"assert" + etype = extension_type(ex) + if etype == "require_existing_locals" + for v in ex[2:end] + vk = var_kind(ctx, NameKey(v)) + if vk !== :local + throw(LoweringError(v, "`outer` annotations must match with a local variable in an outer scope but no such variable was found")) + end + end + elseif etype == "global_toplevel_only" + if !ctx.scope_stack[end].is_toplevel_global_scope + e = ex[2][1] + throw(LoweringError(e, "$(kind(e)) is only allowed in global scope")) + end + elseif etype == "toplevel_only" + if !ctx.scope_stack[end].in_toplevel_thunk + e = ex[2][1] + throw(LoweringError(e, "this syntax is only allowed in top level code")) + end + else + throw(LoweringError(ex, "Unknown syntax assertion")) + end + makeleaf(ctx, ex, K"TOMBSTONE") + elseif k == K"function_decl" + resolved = mapchildren(e->_resolve_scopes(ctx, e), ctx, ex) + name = resolved[1] + if kind(name) == K"BindingId" + bk = lookup_binding(ctx, name).kind + if bk == :argument + throw(LoweringError(name, "Cannot add method to a function argument")) + elseif bk == :global && !ctx.scope_stack[end].in_toplevel_thunk + throw(LoweringError(name, + "Global method definition needs to be placed at the top level, or use `eval()`")) + end + end + resolved + elseif k == K"assign_const_if_global" + id = _resolve_scopes(ctx, ex[1]) + bk = lookup_binding(ctx, id).kind + if bk == :local && numchildren(ex) != 1 + @ast ctx ex _resolve_scopes(ctx, [K"=" children(ex)...]) + elseif bk != :local # TODO: should this be == :global? + @ast ctx ex _resolve_scopes(ctx, [K"constdecl" children(ex)...]) + else + makeleaf(ctx, ex, K"TOMBSTONE") + end + else + mapchildren(e->_resolve_scopes(ctx, e), ctx, ex) + end +end + +function _resolve_scopes(ctx, exs::AbstractVector) + out = SyntaxList(ctx) + for e in exs + push!(out, _resolve_scopes(ctx, e)) + end + out +end + +#------------------------------------------------------------------------------- +# Sub-pass to compute additional information about variable usage as required +# by closure conversion, etc +struct ClosureBindings + name_stack::Vector{String} # Names of functions the closure is nested within + lambdas::Vector{LambdaBindings} # Bindings for each method of the closure +end + +ClosureBindings(name_stack) = ClosureBindings(name_stack, Vector{LambdaBindings}()) + +struct VariableAnalysisContext{GraphType} <: AbstractLoweringContext + graph::GraphType + bindings::Bindings + mod::Module + lambda_bindings::LambdaBindings + # Stack of method definitions for closure naming + method_def_stack::SyntaxList{GraphType} + # Collection of information about each closure, principally which methods + # are part of the closure (and hence captures). + closure_bindings::Dict{IdTag,ClosureBindings} +end + +function VariableAnalysisContext(graph, bindings, mod, lambda_bindings) + VariableAnalysisContext(graph, bindings, mod, lambda_bindings, + SyntaxList(graph), Dict{IdTag,ClosureBindings}()) +end + +function current_lambda_bindings(ctx::VariableAnalysisContext) + ctx.lambda_bindings +end + +function init_closure_bindings!(ctx, fname) + func_name_id = fname.var_id + @assert lookup_binding(ctx, func_name_id).kind === :local + get!(ctx.closure_bindings, func_name_id) do + name_stack = Vector{String}() + for parentname in ctx.method_def_stack + if kind(parentname) == K"BindingId" + push!(name_stack, lookup_binding(ctx, parentname).name) + end + end + push!(name_stack, lookup_binding(ctx, func_name_id).name) + ClosureBindings(name_stack) + end +end + +# Update ctx.bindings and ctx.lambda_bindings metadata based on binding usage +function analyze_variables!(ctx, ex) + k = kind(ex) + if k == K"BindingId" + if has_lambda_binding(ctx, ex) + # TODO: Move this after closure conversion so that we don't need + # to model the closure conversion transformations here. + update_lambda_binding!(ctx, ex, is_read=true) + else + binfo = lookup_binding(ctx, ex.var_id) + if !binfo.is_ssa && binfo.kind != :global + # The type of typed locals is invisible in the previous pass, + # but is filled in here. + init_lambda_binding(ctx.lambda_bindings, ex.var_id, is_captured=true, is_read=true) + update_binding!(ctx, ex, is_captured=true) + end + end + elseif is_leaf(ex) || is_quoted(ex) + return + elseif k == K"local" || k == K"global" + # Presence of BindingId within local/global is ignored. + return + elseif k == K"=" + lhs = ex[1] + if kind(lhs) != K"Placeholder" + update_binding!(ctx, lhs, add_assigned=1) + if has_lambda_binding(ctx, lhs) + update_lambda_binding!(ctx, lhs, is_assigned=true) + end + lhs_binfo = lookup_binding(ctx, lhs) + if !isnothing(lhs_binfo.type) + # Assignments introduce a variable's type later during closure + # conversion, but we must model that explicitly here. + analyze_variables!(ctx, lhs_binfo.type) + end + end + analyze_variables!(ctx, ex[2]) + elseif k == K"function_decl" + name = ex[1] + if lookup_binding(ctx, name.var_id).kind === :local + init_closure_bindings!(ctx, name) + end + update_binding!(ctx, name, add_assigned=1) + if has_lambda_binding(ctx, name) + update_lambda_binding!(ctx, name, is_assigned=true) + end + elseif k == K"function_type" + if kind(ex[1]) != K"BindingId" || lookup_binding(ctx, ex[1]).kind !== :local + analyze_variables!(ctx, ex[1]) + end + elseif k == K"constdecl" + id = ex[1] + if lookup_binding(ctx, id).kind == :local + throw(LoweringError(ex, "unsupported `const` declaration on local variable")) + end + update_binding!(ctx, id; is_const=true) + elseif k == K"call" + name = ex[1] + if kind(name) == K"BindingId" + id = name.var_id + if has_lambda_binding(ctx, id) + # TODO: Move this after closure conversion so that we don't need + # to model the closure conversion transformations. + update_lambda_binding!(ctx, id, is_called=true) + end + end + foreach(e->analyze_variables!(ctx, e), children(ex)) + elseif k == K"method_defs" + push!(ctx.method_def_stack, ex[1]) + analyze_variables!(ctx, ex[2]) + pop!(ctx.method_def_stack) + elseif k == K"_opaque_closure" + name = ex[1] + init_closure_bindings!(ctx, name) + push!(ctx.method_def_stack, name) + analyze_variables!(ctx, ex[2]) + analyze_variables!(ctx, ex[3]) + analyze_variables!(ctx, ex[4]) + analyze_variables!(ctx, ex[9]) + pop!(ctx.method_def_stack) + elseif k == K"lambda" + lambda_bindings = ex.lambda_bindings + if !ex.is_toplevel_thunk && !isempty(ctx.method_def_stack) + # Record all lambdas for the same closure type in one place + func_name = last(ctx.method_def_stack) + if kind(func_name) == K"BindingId" + func_name_id = func_name.var_id + if lookup_binding(ctx, func_name_id).kind === :local + push!(ctx.closure_bindings[func_name_id].lambdas, lambda_bindings) + end + end + end + ctx2 = VariableAnalysisContext(ctx.graph, ctx.bindings, ctx.mod, lambda_bindings, + ctx.method_def_stack, ctx.closure_bindings) + foreach(e->analyze_variables!(ctx2, e), ex[3:end]) # body & return type + for (id,lbinfo) in pairs(lambda_bindings.bindings) + if lbinfo.is_captured + # Add any captured bindings to the enclosing lambda, if necessary. + outer_lbinfo = lookup_lambda_binding(ctx.lambda_bindings, id) + if isnothing(outer_lbinfo) + # Inner lambda captures a variable. If it's not yet present + # in the outer lambda, the outer lambda must capture it as + # well so that the closure associated to the inner lambda + # can be initialized when `function_decl` is hit. + init_lambda_binding(ctx.lambda_bindings, id, is_captured=true, is_read=true) + end + end + end + else + foreach(e->analyze_variables!(ctx, e), children(ex)) + end + nothing +end + +function resolve_scopes(ctx::ScopeResolutionContext, ex) + if kind(ex) != K"lambda" + # Wrap in a top level thunk if we're not already expanding a lambda. + # (Maybe this should be done elsewhere?) + ex = @ast ctx ex [K"lambda"(is_toplevel_thunk=true) + [K"block"] + [K"block"] + ex + ] + end + _resolve_scopes(ctx, ex) +end + +""" +This pass analyzes scopes and the names (locals/globals etc) used within them. + +Names of kind `K"Identifier"` are transformed into binding identifiers of +kind `K"BindingId"`. The associated `Bindings` table in the context records +metadata about each binding. + +This pass also records the set of binding IDs used locally within the +enclosing lambda form and information about variables captured by closures. +""" +function resolve_scopes(ctx::DesugaringContext, ex) + ctx2 = ScopeResolutionContext(ctx) + ex2 = resolve_scopes(ctx2, reparent(ctx2, ex)) + ctx3 = VariableAnalysisContext(ctx2.graph, ctx2.bindings, ctx2.mod, ex2.lambda_bindings) + analyze_variables!(ctx3, ex2) + ctx3, ex2 +end diff --git a/src/vendored/JuliaLowering/src/syntax_graph.jl b/src/vendored/JuliaLowering/src/syntax_graph.jl new file mode 100644 index 00000000..82c91302 --- /dev/null +++ b/src/vendored/JuliaLowering/src/syntax_graph.jl @@ -0,0 +1,747 @@ +const NodeId = Int + +""" +Directed graph with arbitrary attributes on nodes. Used here for representing +one or several syntax trees. + +TODO: Global attributes! +""" +struct SyntaxGraph{Attrs} + edge_ranges::Vector{UnitRange{Int}} + edges::Vector{NodeId} + attributes::Attrs +end + +SyntaxGraph() = SyntaxGraph{Dict{Symbol,Any}}(Vector{UnitRange{Int}}(), + Vector{NodeId}(), Dict{Symbol,Any}()) + +# "Freeze" attribute names and types, encoding them in the type of the returned +# SyntaxGraph. +function freeze_attrs(graph::SyntaxGraph) + frozen_attrs = (; pairs(graph.attributes)...) + SyntaxGraph(graph.edge_ranges, graph.edges, frozen_attrs) +end + +function _show_attrs(io, attributes::Dict) + show(io, MIME("text/plain"), attributes) +end +function _show_attrs(io, attributes::NamedTuple) + show(io, MIME("text/plain"), Dict(pairs(attributes)...)) +end + +function attrnames(graph::SyntaxGraph) + keys(graph.attributes) +end + +function Base.show(io::IO, ::MIME"text/plain", graph::SyntaxGraph) + print(io, typeof(graph), + " with $(length(graph.edge_ranges)) vertices, $(length(graph.edges)) edges, and attributes:\n") + _show_attrs(io, graph.attributes) +end + +function ensure_attributes!(graph::SyntaxGraph; kws...) + for (k,v) in pairs(kws) + @assert k isa Symbol + @assert v isa Type + if haskey(graph.attributes, k) + v0 = valtype(graph.attributes[k]) + v == v0 || throw(ErrorException("Attribute type mismatch $v != $v0")) + else + graph.attributes[k] = Dict{NodeId,v}() + end + end + graph +end + +function ensure_attributes(graph::SyntaxGraph; kws...) + g = SyntaxGraph(graph.edge_ranges, graph.edges, Dict(pairs(graph.attributes)...)) + ensure_attributes!(g; kws...) + freeze_attrs(g) +end + +function delete_attributes(graph::SyntaxGraph, attr_names...) + attributes = Dict(pairs(graph.attributes)...) + for name in attr_names + delete!(attributes, name) + end + SyntaxGraph(graph.edge_ranges, graph.edges, (; pairs(attributes)...)) +end + +function newnode!(graph::SyntaxGraph) + push!(graph.edge_ranges, 0:-1) # Invalid range start => leaf node + return length(graph.edge_ranges) +end + +function setchildren!(graph::SyntaxGraph, id, children::NodeId...) + setchildren!(graph, id, children) +end + +function setchildren!(graph::SyntaxGraph, id, children) + n = length(graph.edges) + graph.edge_ranges[id] = n+1:(n+length(children)) + # TODO: Reuse existing edges if possible + append!(graph.edges, children) +end + +function JuliaSyntax.is_leaf(graph::SyntaxGraph, id) + first(graph.edge_ranges[id]) == 0 +end + +function JuliaSyntax.numchildren(graph::SyntaxGraph, id) + length(graph.edge_ranges[id]) +end + +function JuliaSyntax.children(graph::SyntaxGraph, id) + @view graph.edges[graph.edge_ranges[id]] +end + +function JuliaSyntax.children(graph::SyntaxGraph, id, r::UnitRange) + @view graph.edges[graph.edge_ranges[id][r]] +end + +function child(graph::SyntaxGraph, id::NodeId, i::Integer) + graph.edges[graph.edge_ranges[id][i]] +end + +function getattr(graph::SyntaxGraph{<:Dict}, name::Symbol) + getfield(graph, :attributes)[name] +end + +function getattr(graph::SyntaxGraph{<:NamedTuple}, name::Symbol) + getfield(getfield(graph, :attributes), name) +end + +function getattr(graph::SyntaxGraph, name::Symbol, default) + get(getfield(graph, :attributes), name, default) +end + +function hasattr(graph::SyntaxGraph, name::Symbol) + getattr(graph, name, nothing) !== nothing +end + +# TODO: Probably terribly non-inferrable? +function setattr!(graph::SyntaxGraph, id; attrs...) + for (k,v) in pairs(attrs) + if !isnothing(v) + getattr(graph, k)[id] = v + end + end +end + +function Base.getproperty(graph::SyntaxGraph, name::Symbol) + # TODO: Remove access to internals? + name === :edge_ranges && return getfield(graph, :edge_ranges) + name === :edges && return getfield(graph, :edges) + name === :attributes && return getfield(graph, :attributes) + return getattr(graph, name) +end + +function sethead!(graph, id::NodeId, h::JuliaSyntax.SyntaxHead) + graph.kind[id] = kind(h) + f = flags(h) + if f != 0 + graph.syntax_flags[id] = f + end +end + +function sethead!(graph, id::NodeId, k::Kind) + graph.kind[id] = k +end + +function _convert_nodes(graph::SyntaxGraph, node::SyntaxNode) + id = newnode!(graph) + sethead!(graph, id, head(node)) + if !isnothing(node.val) + v = node.val + if v isa Symbol + # TODO: Fixes in JuliaSyntax to avoid ever converting to Symbol + setattr!(graph, id, name_val=string(v)) + else + setattr!(graph, id, value=v) + end + end + setattr!(graph, id, source=SourceRef(node.source, node.position, node.raw)) + if !is_leaf(node) + cs = map(children(node)) do n + _convert_nodes(graph, n) + end + setchildren!(graph, id, cs) + end + return id +end + +""" + syntax_graph(ctx) + +Return `SyntaxGraph` associated with `ctx` +""" +syntax_graph(graph::SyntaxGraph) = graph + +function check_same_graph(x, y) + if syntax_graph(x) !== syntax_graph(y) + error("Mismatching syntax graphs") + end +end + +function check_compatible_graph(x, y) + if !is_compatible_graph(x, y) + error("Incompatible syntax graphs") + end +end + +function is_compatible_graph(x, y) + syntax_graph(x).edges === syntax_graph(y).edges +end + +#------------------------------------------------------------------------------- +struct SyntaxTree{GraphType} + _graph::GraphType + _id::NodeId +end + +function Base.getproperty(ex::SyntaxTree, name::Symbol) + name === :_graph && return getfield(ex, :_graph) + name === :_id && return getfield(ex, :_id) + _id = getfield(ex, :_id) + return get(getproperty(getfield(ex, :_graph), name), _id) do + error("Property `$name[$_id]` not found") + end +end + +function Base.setproperty!(ex::SyntaxTree, name::Symbol, val) + return setattr!(ex._graph, ex._id; name=>val) +end + +function Base.propertynames(ex::SyntaxTree) + attrnames(ex) +end + +function Base.get(ex::SyntaxTree, name::Symbol, default) + attr = getattr(getfield(ex, :_graph), name, nothing) + return isnothing(attr) ? default : + get(attr, getfield(ex, :_id), default) +end + +function Base.getindex(ex::SyntaxTree, i::Integer) + SyntaxTree(ex._graph, child(ex._graph, ex._id, i)) +end + +function Base.getindex(ex::SyntaxTree, r::UnitRange) + SyntaxList(ex._graph, children(ex._graph, ex._id, r)) +end + +Base.firstindex(ex::SyntaxTree) = 1 +Base.lastindex(ex::SyntaxTree) = numchildren(ex) + +function hasattr(ex::SyntaxTree, name::Symbol) + attr = getattr(ex._graph, name, nothing) + return !isnothing(attr) && haskey(attr, ex._id) +end + +function attrnames(ex::SyntaxTree) + attrs = ex._graph.attributes + [name for (name, value) in pairs(attrs) if haskey(value, ex._id)] +end + +function setattr(ex::SyntaxTree; extra_attrs...) + graph = syntax_graph(ex) + id = newnode!(graph) + if !is_leaf(ex) + setchildren!(graph, id, _node_ids(graph, children(ex)...)) + end + ex2 = SyntaxTree(graph, id) + copy_attrs!(ex2, ex, true) + setattr!(ex2; extra_attrs...) + ex2 +end + +function setattr!(ex::SyntaxTree; attrs...) + setattr!(ex._graph, ex._id; attrs...) +end + +# JuliaSyntax tree API + +function JuliaSyntax.is_leaf(ex::SyntaxTree) + is_leaf(ex._graph, ex._id) +end + +function JuliaSyntax.numchildren(ex::SyntaxTree) + numchildren(ex._graph, ex._id) +end + +function JuliaSyntax.children(ex::SyntaxTree) + SyntaxList(ex._graph, children(ex._graph, ex._id)) +end + +function JuliaSyntax.head(ex::SyntaxTree) + JuliaSyntax.SyntaxHead(kind(ex), flags(ex)) +end + +function JuliaSyntax.kind(ex::SyntaxTree) + ex.kind +end + +function JuliaSyntax.flags(ex::SyntaxTree) + get(ex, :syntax_flags, 0x0000) +end + + +# Reference to bytes within a source file +struct SourceRef + file::SourceFile + first_byte::Int + # TODO: Do we need the green node, or would last_byte suffice? + green_tree::JuliaSyntax.GreenNode +end + +JuliaSyntax.sourcefile(src::SourceRef) = src.file +JuliaSyntax.byte_range(src::SourceRef) = src.first_byte:(src.first_byte + span(src.green_tree) - 1) + +# TODO: Adding these methods to support LineNumberNode is kind of hacky but we +# can remove these after JuliaLowering becomes self-bootstrapping for macros +# and we a proper SourceRef for @ast's @HERE form. +JuliaSyntax.byte_range(src::LineNumberNode) = 0:0 +JuliaSyntax.source_location(src::LineNumberNode) = (src.line, 0) +JuliaSyntax.source_location(::Type{LineNumberNode}, src::LineNumberNode) = src +JuliaSyntax.source_line(src::LineNumberNode) = src.line +# The follow somewhat strange cases are for where LineNumberNode is standing in +# for SourceFile because we've only got Expr-based provenance info +JuliaSyntax.sourcefile(src::LineNumberNode) = src +JuliaSyntax.source_location(src::LineNumberNode, byte_index::Integer) = (src.line, 0) +JuliaSyntax.source_location(::Type{LineNumberNode}, src::LineNumberNode, byte_index::Integer) = src +JuliaSyntax.filename(src::LineNumberNode) = string(src.file) + +function JuliaSyntax.highlight(io::IO, src::LineNumberNode; note="") + print(io, src, " - ", note) +end + +function JuliaSyntax.highlight(io::IO, src::SourceRef; kws...) + highlight(io, src.file, first_byte(src):last_byte(src); kws...) +end + +function Base.show(io::IO, ::MIME"text/plain", src::SourceRef) + highlight(io, src; note="these are the bytes you're looking for 😊", context_lines_inner=20) +end + + +function provenance(ex::SyntaxTree) + s = ex.source + if s isa NodeId + return (SyntaxTree(ex._graph, s),) + elseif s isa Tuple + return SyntaxTree.((ex._graph,), s) + else + return (s,) + end +end + + +function _sourceref(sources, id) + i = 1 + while true + i += 1 + s = sources[id] + if s isa NodeId + id = s + else + return s, id + end + end +end + +function sourceref(ex::SyntaxTree) + sources = ex._graph.source + id::NodeId = ex._id + while true + s, _ = _sourceref(sources, id) + if s isa Tuple + s = s[1] + end + if s isa NodeId + id = s + else + return s + end + end +end + +function _flattened_provenance(refs, graph, sources, id) + # TODO: Implement in terms of `provenance()`? + s, id2 = _sourceref(sources, id) + if s isa Tuple + for i in s + _flattened_provenance(refs, graph, sources, i) + end + else + push!(refs, SyntaxTree(graph, id2)) + end +end + +function flattened_provenance(ex::SyntaxTree) + refs = SyntaxList(ex) + _flattened_provenance(refs, ex._graph, ex._graph.source, ex._id) + return reverse(refs) +end + + +function is_ancestor(ex, ancestor) + if !is_compatible_graph(ex, ancestor) + return false + end + sources = ex._graph.source + id::NodeId = ex._id + while true + s = get(sources, id, nothing) + if s isa NodeId + id = s + if id == ancestor._id + return true + end + else + return false + end + end +end + +const SourceAttrType = Union{SourceRef,LineNumberNode,NodeId,Tuple} + +function SyntaxTree(graph::SyntaxGraph, node::SyntaxNode) + ensure_attributes!(graph, kind=Kind, syntax_flags=UInt16, source=SourceAttrType, + value=Any, name_val=String) + id = _convert_nodes(freeze_attrs(graph), node) + return SyntaxTree(graph, id) +end + +function SyntaxTree(node::SyntaxNode) + return SyntaxTree(SyntaxGraph(), node) +end + +attrsummary(name, value) = string(name) +attrsummary(name, value::Number) = "$name=$value" + +function _value_string(ex) + k = kind(ex) + str = k == K"Identifier" || k == K"MacroName" || is_operator(k) ? ex.name_val : + k == K"Placeholder" ? ex.name_val : + k == K"SSAValue" ? "%" : + k == K"BindingId" ? "#" : + k == K"label" ? "label" : + k == K"core" ? "core.$(ex.name_val)" : + k == K"top" ? "top.$(ex.name_val)" : + k == K"Symbol" ? ":$(ex.name_val)" : + k == K"globalref" ? "$(ex.mod).$(ex.name_val)" : + k == K"slot" ? "slot" : + k == K"latestworld" ? "(latestworld)" : + k == K"static_parameter" ? "static_parameter" : + k == K"symbolic_label" ? "label:$(ex.name_val)" : + k == K"symbolic_goto" ? "goto:$(ex.name_val)" : + k == K"SourceLocation" ? "SourceLocation:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : + repr(get(ex, :value, nothing)) + id = get(ex, :var_id, nothing) + if isnothing(id) + id = get(ex, :id, nothing) + end + if !isnothing(id) + idstr = subscript_str(id) + str = "$(str)$idstr" + end + if k == K"slot" || k == K"BindingId" + p = provenance(ex)[1] + while p isa SyntaxTree + if kind(p) == K"Identifier" + str = "$(str)/$(p.name_val)" + break + end + p = provenance(p)[1] + end + end + return str +end + +function _show_syntax_tree(io, ex, indent, show_kinds) + val = get(ex, :value, nothing) + nodestr = !is_leaf(ex) ? "[$(untokenize(head(ex)))]" : _value_string(ex) + + treestr = rpad(string(indent, nodestr), 40) + if show_kinds && is_leaf(ex) + treestr = treestr*" :: "*string(kind(ex)) + end + + std_attrs = Set([:name_val,:value,:kind,:syntax_flags,:source,:var_id]) + attrstr = join([attrsummary(n, getproperty(ex, n)) + for n in attrnames(ex) if n ∉ std_attrs], ",") + treestr = string(rpad(treestr, 60), " │ $attrstr") + + println(io, treestr) + if !is_leaf(ex) + new_indent = indent*" " + for n in children(ex) + _show_syntax_tree(io, n, new_indent, show_kinds) + end + end +end + +function Base.show(io::IO, ::MIME"text/plain", ex::SyntaxTree, show_kinds=true) + anames = join(string.(attrnames(syntax_graph(ex))), ",") + println(io, "SyntaxTree with attributes $anames") + _show_syntax_tree(io, ex, "", show_kinds) +end + +function _show_syntax_tree_sexpr(io, ex) + if is_leaf(ex) + if is_error(ex) + print(io, "(", untokenize(head(ex)), ")") + else + print(io, _value_string(ex)) + end + else + print(io, "(", untokenize(head(ex))) + first = true + for n in children(ex) + print(io, ' ') + _show_syntax_tree_sexpr(io, n) + first = false + end + print(io, ')') + end +end + +function Base.show(io::IO, ::MIME"text/x.sexpression", node::SyntaxTree) + _show_syntax_tree_sexpr(io, node) +end + +function Base.show(io::IO, node::SyntaxTree) + _show_syntax_tree_sexpr(io, node) +end + +function reparent(ctx, ex::SyntaxTree) + # Ensure `ex` has the same parent graph, in a somewhat loose sense. + # Could relax by copying if necessary? + # In that case, would we copy all the attributes? That would have slightly + # different semantics. + graph = syntax_graph(ctx) + @assert graph.edge_ranges === ex._graph.edge_ranges + SyntaxTree(graph, ex._id) +end + +function ensure_attributes(ex::SyntaxTree; kws...) + reparent(ensure_attributes(syntax_graph(ex); kws...), ex) +end + +syntax_graph(ex::SyntaxTree) = ex._graph + +function JuliaSyntax.build_tree(::Type{SyntaxTree}, stream::JuliaSyntax.ParseStream; kws...) + SyntaxTree(JuliaSyntax.build_tree(SyntaxNode, stream; kws...)) +end + +JuliaSyntax.sourcefile(ex::SyntaxTree) = sourcefile(sourceref(ex)) +JuliaSyntax.byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) + +function JuliaSyntax._expr_leaf_val(ex::SyntaxTree) + name = get(ex, :name_val, nothing) + if !isnothing(name) + Symbol(name) + else + ex.value + end +end + +Base.Expr(ex::SyntaxTree) = JuliaSyntax.to_expr(ex) + +#-------------------------------------------------- +function _find_SyntaxTree_macro(ex, line) + @assert !is_leaf(ex) + for c in children(ex) + rng = byte_range(c) + firstline = JuliaSyntax.source_line(sourcefile(c), first(rng)) + lastline = JuliaSyntax.source_line(sourcefile(c), last(rng)) + if line < firstline || lastline < line + continue + end + # We're in the line range. Either + if firstline == line && kind(c) == K"macrocall" && begin + name = c[1] + if kind(name) == K"." + name = name[2] + end + @assert kind(name) == K"MacroName" + name.name_val == "@SyntaxTree" + end + # We find the node we're looking for. NB: Currently assuming a max + # of one @SyntaxTree invocation per line. Though we could relax + # this with more heuristic matching of the Expr-AST... + @assert numchildren(c) == 2 + return c[2] + elseif !is_leaf(c) + # Recurse + ex1 = _find_SyntaxTree_macro(c, line) + if !isnothing(ex1) + return ex1 + end + end + end + return nothing # Will get here if multiple children are on the same line. +end + +""" +Macro to construct quoted SyntaxTree literals (instead of quoted Expr literals) +in normal Julia source code. + +Example: + +```julia +tree1 = @SyntaxTree :(some_unique_identifier) +tree2 = @SyntaxTree quote + x = 1 + \$tree1 = x +end +``` +""" +macro SyntaxTree(ex_old) + # The implementation here is hilarious and arguably very janky: we + # 1. Briefly check but throw away the Expr-AST + if !(Meta.isexpr(ex_old, :quote) || ex_old isa QuoteNode) + throw(ArgumentError("@SyntaxTree expects a `quote` block or `:`-quoted expression")) + end + # 2. Re-parse the current source file as SyntaxTree instead + fname = String(__source__.file) + if occursin(r"REPL\[\d+\]", fname) + # Assume we should look at last history entry in REPL + try + # Wow digging in like this is an awful hack but `@SyntaxTree` is + # already a hack so let's go for it I guess 😆 + text = Base.active_repl.mistate.interface.modes[1].hist.history[end] + if !occursin("@SyntaxTree", text) + error("Text not found in last REPL history line") + end + catch + error("Text not found in REPL history") + end + else + text = read(fname, String) + end + full_ex = parseall(SyntaxTree, text) + # 3. Using the current file and line number, dig into the re-parsed tree and + # discover the piece of AST which should be returned. + ex = _find_SyntaxTree_macro(full_ex, __source__.line) + # 4. Do the first step of JuliaLowering's syntax lowering to get + # synax interpolations to work + _, ex1 = expand_forms_1(__module__, ex) + @assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast + esc(Expr(:call, interpolate_ast, ex1[2][1], map(Expr, ex1[3:end])...)) +end + +#------------------------------------------------------------------------------- +# Lightweight vector of nodes ids with associated pointer to graph stored separately. +struct SyntaxList{GraphType, NodeIdVecType} <: AbstractVector{SyntaxTree} + graph::GraphType + ids::NodeIdVecType +end + +function SyntaxList(graph::SyntaxGraph, ids::AbstractVector{NodeId}) + SyntaxList{typeof(graph), typeof(ids)}(graph, ids) +end + +SyntaxList(graph::SyntaxGraph) = SyntaxList(graph, Vector{NodeId}()) +SyntaxList(ctx) = SyntaxList(syntax_graph(ctx)) + +syntax_graph(lst::SyntaxList) = lst.graph + +Base.size(v::SyntaxList) = size(v.ids) + +Base.IndexStyle(::Type{<:SyntaxList}) = IndexLinear() + +Base.getindex(v::SyntaxList, i::Int) = SyntaxTree(v.graph, v.ids[i]) + +function Base.getindex(v::SyntaxList, r::UnitRange) + SyntaxList(v.graph, view(v.ids, r)) +end + +function Base.setindex!(v::SyntaxList, ex::SyntaxTree, i::Int) + check_compatible_graph(v, ex) + v.ids[i] = ex._id +end + +function Base.setindex!(v::SyntaxList, id::NodeId, i::Int) + v.ids[i] = id +end + +function Base.push!(v::SyntaxList, ex::SyntaxTree) + check_compatible_graph(v, ex) + push!(v.ids, ex._id) +end + +function Base.pushfirst!(v::SyntaxList, ex::SyntaxTree) + check_compatible_graph(v, ex) + pushfirst!(v.ids, ex._id) +end + +function Base.similar(v::SyntaxList, size::Tuple=Base.size(v.ids)) + SyntaxList(v.graph, zeros(NodeId, size)) +end + +function Base.isassigned(v::SyntaxList, i::Integer) + v.ids[i] > 0 +end + +function Base.append!(v::SyntaxList, exs) + for e in exs + push!(v, e) + end + v +end + +function Base.append!(v::SyntaxList, exs::SyntaxList) + check_compatible_graph(v, exs) + append!(v.ids, exs.ids) + v +end + +function Base.push!(v::SyntaxList, id::NodeId) + push!(v.ids, id) +end + +function Base.pop!(v::SyntaxList) + SyntaxTree(v.graph, pop!(v.ids)) +end + +function Base.resize!(v::SyntaxList, n) + resize!(v.ids, n) + v +end + +function Base.empty!(v::SyntaxList) + empty!(v.ids) + v +end + +function Base.deleteat!(v::SyntaxList, inds) + deleteat!(v.ids, inds) +end + +function Base.copy(v::SyntaxList) + SyntaxList(v.graph, copy(v.ids)) +end + +function Base.filter(f, exs::SyntaxList) + out = SyntaxList(syntax_graph(exs)) + for ex in exs + if f(ex) + push!(out, ex) + end + end + out +end + +# Would like the following to be an overload of Base.map() ... but need +# somewhat arcane trickery to ensure that this only tries to collect into a +# SyntaxList when `f` yields a SyntaxTree. +# +# function mapsyntax(f, exs::SyntaxList) +# out = SyntaxList(syntax_graph(exs)) +# for ex in exs +# push!(out, f(ex)) +# end +# out +# end + diff --git a/src/vendored/JuliaLowering/src/syntax_macros.jl b/src/vendored/JuliaLowering/src/syntax_macros.jl new file mode 100644 index 00000000..5a18059d --- /dev/null +++ b/src/vendored/JuliaLowering/src/syntax_macros.jl @@ -0,0 +1,223 @@ +# The following are versions of macros from Base which act as "standard syntax +# extensions": +# +# * They emit syntactic forms with special `Kind`s and semantics known to +# lowering +# * There is no other Julia surface syntax for these `Kind`s. + +# In order to implement these here without getting into bootstrapping problems, +# we just write them as plain old macro-named functions and add the required +# __context__ argument ourselves. +# +# TODO: @inline, @noinline, @inbounds, @simd, @ccall, @isdefined, @assume_effects +# +# TODO: Eventually move these to proper `macro` definitions and use +# `JuliaLowering.include()` or something. Then we'll be in the fun little world +# of bootstrapping but it shouldn't be too painful :) + +function _apply_nospecialize(ctx, ex) + k = kind(ex) + if k == K"Identifier" || k == K"Placeholder" || k == K"tuple" + setmeta(ex; nospecialize=true) + elseif k == K"..." || k == K"::" || k == K"=" + if k == K"::" && numchildren(ex) == 1 + ex = @ast ctx ex [K"::" "_"::K"Placeholder" ex[1]] + end + mapchildren(c->_apply_nospecialize(ctx, c), ctx, ex, 1:1) + else + throw(LoweringError(ex, "Invalid function argument")) + end +end + +function Base.var"@nospecialize"(__context__::MacroContext, ex) + _apply_nospecialize(__context__, ex) +end + +function Base.var"@atomic"(__context__::MacroContext, ex) + @chk kind(ex) == K"Identifier" || kind(ex) == K"::" (ex, "Expected identifier or declaration") + @ast __context__ __context__.macrocall [K"atomic" ex] +end + +function Base.var"@label"(__context__::MacroContext, ex) + @chk kind(ex) == K"Identifier" + @ast __context__ ex ex=>K"symbolic_label" +end + +function Base.var"@goto"(__context__::MacroContext, ex) + @chk kind(ex) == K"Identifier" + @ast __context__ ex ex=>K"symbolic_goto" +end + +function Base.var"@locals"(__context__::MacroContext) + @ast __context__ __context__.macrocall [K"extension" "locals"::K"Symbol"] +end + +function Base.var"@isdefined"(__context__::MacroContext, ex) + @ast __context__ __context__.macrocall [K"isdefined" ex] +end + +function Base.var"@generated"(__context__::MacroContext) + @ast __context__ __context__.macrocall [K"generated"] +end +function Base.var"@generated"(__context__::MacroContext, ex) + if kind(ex) != K"function" + throw(LoweringError(ex, "Expected a function argument to `@generated`")) + end + @ast __context__ __context__.macrocall [K"function" + ex[1] + [K"if" [K"generated"] + ex[2] + [K"block" + [K"meta" "generated_only"::K"Symbol"] + [K"return"] + ] + ] + ] +end + +function Base.var"@cfunction"(__context__::MacroContext, callable, return_type, arg_types) + if kind(arg_types) != K"tuple" + throw(MacroExpansionError(arg_types, "@cfunction argument types must be a literal tuple")) + end + arg_types_svec = @ast __context__ arg_types [K"call" + "svec"::K"core" + children(arg_types)... + ] + if kind(callable) == K"$" + fptr = callable[1] + typ = Base.CFunction + else + # Kinda weird semantics here - without `$`, the callable is a top level + # expression which will be evaluated by `jl_resolve_globals_in_ir`, + # implicitly within the module where the `@cfunction` is expanded into. + # + # TODO: The existing flisp implementation is arguably broken because it + # ignores macro hygiene when `callable` is the result of a macro + # expansion within a different module. For now we've inherited this + # brokenness. + # + # Ideally we'd fix this by bringing the scoping rules for this + # expression back into lowering. One option may be to wrap the + # expression in a form which pushes it to top level - maybe as a whole + # separate top level thunk like closure lowering - then use the + # K"captured_local" mechanism to interpolate it back in. This scheme + # would make the complicated scope semantics explicit and let them be + # dealt with in the right place in the frontend rather than putting the + # rules into the runtime itself. + fptr = @ast __context__ callable QuoteNode(Expr(callable))::K"Value" + typ = Ptr{Cvoid} + end + @ast __context__ __context__.macrocall [K"cfunction" + typ::K"Value" + fptr + return_type + arg_types_svec + "ccall"::K"Symbol" + ] +end + +function Base.GC.var"@preserve"(__context__::MacroContext, exs...) + idents = exs[1:end-1] + for e in idents + if kind(e) != K"Identifier" + throw(MacroExpansionError(e, "Preserved variable must be a symbol")) + end + end + @ast __context__ __context__.macrocall [K"block" + [K"=" + "s"::K"Identifier" + [K"gc_preserve_begin" + idents... + ] + ] + [K"=" + "r"::K"Identifier" + exs[end] + ] + [K"gc_preserve_end" "s"::K"Identifier"] + "r"::K"Identifier" + ] +end + +function Base.Experimental.var"@opaque"(__context__::MacroContext, ex) + @chk kind(ex) == K"->" + @ast __context__ __context__.macrocall [K"opaque_closure" + "nothing"::K"core" + "nothing"::K"core" + "nothing"::K"core" + true::K"Bool" + ex + ] +end + +#-------------------------------------------------------------------------------- +# The following `@islocal` and `@inert` are macros for special syntax known to +# lowering which don't exist in Base but arguably should. +# +# For now we have our own versions +function var"@islocal"(__context__::MacroContext, ex) + @chk kind(ex) == K"Identifier" + @ast __context__ __context__.macrocall [K"extension" + "islocal"::K"Symbol" + ex + ] +end + +""" +A non-interpolating quoted expression. + +For example, + +```julia +@inert quote + \$x +end +``` + +does not take `x` from the surrounding scope - instead it leaves the +interpolation `\$x` intact as part of the expression tree. + +TODO: What is the correct way for `@inert` to work? ie which of the following +should work? + +```julia +@inert quote + body +end + +@inert begin + body +end + +@inert x + +@inert \$x +``` + +The especially tricky cases involve nested interpolation ... +```julia +quote + @inert \$x +end + +@inert quote + quote + \$x + end +end + +@inert quote + quote + \$\$x + end +end +``` + +etc. Needs careful thought - we should probably just copy what lisp does with +quote+quasiquote 😅 +""" +function var"@inert"(__context__::MacroContext, ex) + @chk kind(ex) == K"quote" + @ast __context__ __context__.macrocall [K"inert" ex] +end + diff --git a/src/vendored/JuliaLowering/src/utils.jl b/src/vendored/JuliaLowering/src/utils.jl new file mode 100644 index 00000000..ced8827e --- /dev/null +++ b/src/vendored/JuliaLowering/src/utils.jl @@ -0,0 +1,168 @@ +# Error handling + +TODO(msg::AbstractString) = throw(ErrorException("Lowering TODO: $msg")) +TODO(ex::SyntaxTree, msg="") = throw(LoweringError(ex, "Lowering TODO: $msg")) + +# Errors found during lowering will result in LoweringError being thrown to +# indicate the syntax causing the error. +struct LoweringError <: Exception + ex::SyntaxTree + msg::String +end + +function Base.showerror(io::IO, exc::LoweringError; show_detail=true) + print(io, "LoweringError:\n") + src = sourceref(exc.ex) + highlight(io, src; note=exc.msg) + + if show_detail + print(io, "\n\nDetailed provenance:\n") + showprov(io, exc.ex, tree=true) + end +end + +#------------------------------------------------------------------------------- +function _show_provtree(io::IO, ex::SyntaxTree, indent) + print(io, ex, "\n") + prov = provenance(ex) + for (i, e) in enumerate(prov) + islast = i == length(prov) + printstyled(io, "$indent$(islast ? "└─ " : "├─ ")", color=:light_black) + inner_indent = indent * (islast ? " " : "│ ") + _show_provtree(io, e, inner_indent) + end +end + +function _show_provtree(io::IO, prov, indent) + fn = filename(prov) + line, _ = source_location(prov) + printstyled(io, "@ $fn:$line\n", color=:light_black) +end + +function showprov(io::IO, exs::AbstractVector) + for (i,ex) in enumerate(Iterators.reverse(exs)) + sr = sourceref(ex) + if i > 1 + print(io, "\n\n") + end + k = kind(ex) + note = i > 1 && k == K"macrocall" ? "in macro expansion" : + i > 1 && k == K"$" ? "interpolated here" : + "in source" + highlight(io, sr, note=note) + + line, _ = source_location(sr) + locstr = "$(filename(sr)):$line" + JuliaSyntax._printstyled(io, "\n# @ $locstr", fgcolor=:light_black) + end +end + +function showprov(io::IO, ex::SyntaxTree; tree=false) + if tree + _show_provtree(io, ex, "") + else + showprov(io, flattened_provenance(ex)) + end +end + +function showprov(x; kws...) + showprov(stdout, x; kws...) +end + +function subscript_str(i) + replace(string(i), + "0"=>"₀", "1"=>"₁", "2"=>"₂", "3"=>"₃", "4"=>"₄", + "5"=>"₅", "6"=>"₆", "7"=>"₇", "8"=>"₈", "9"=>"₉") +end + +function _deref_ssa(stmts, ex) + while kind(ex) == K"SSAValue" + ex = stmts[ex.var_id] + end + ex +end + +function _find_method_lambda(ex, name) + @assert kind(ex) == K"code_info" + # Heuristic search through outer thunk for the method in question. + method_found = false + stmts = children(ex[1]) + for e in stmts + if kind(e) == K"method" && numchildren(e) >= 2 + sig = _deref_ssa(stmts, e[2]) + @assert kind(sig) == K"call" + arg_types = _deref_ssa(stmts, sig[2]) + @assert kind(arg_types) == K"call" + self_type = _deref_ssa(stmts, arg_types[2]) + if kind(self_type) == K"globalref" && occursin(name, self_type.name_val) + return e[3] + end + end + end +end + +function print_ir(io::IO, ex, method_filter=nothing) + @assert kind(ex) == K"code_info" + if !isnothing(method_filter) + filtered = _find_method_lambda(ex, method_filter) + if isnothing(filtered) + @warn "Method not found with method filter $method_filter" + else + ex = filtered + end + end + _print_ir(io, ex, "") +end + +function _print_ir(io::IO, ex, indent) + added_indent = " " + @assert (kind(ex) == K"lambda" || kind(ex) == K"code_info") && kind(ex[1]) == K"block" + if !ex.is_toplevel_thunk && kind(ex) == K"code_info" + slots = ex.slots + print(io, indent, "slots: [") + for (i,slot) in enumerate(slots) + print(io, "slot$(subscript_str(i))/$(slot.name)") + flags = String[] + slot.is_nospecialize && push!(flags, "nospecialize") + !slot.is_read && push!(flags, "!read") + slot.is_single_assign && push!(flags, "single_assign") + slot.is_maybe_undef && push!(flags, "maybe_undef") + slot.is_called && push!(flags, "called") + if !isempty(flags) + print(io, "($(join(flags, ",")))") + end + if i < length(slots) + print(io, " ") + end + end + println(io, "]") + end + stmts = children(ex[1]) + for (i, e) in enumerate(stmts) + lno = rpad(i, 3) + if kind(e) == K"method" && numchildren(e) == 3 + print(io, indent, lno, " --- method ", string(e[1]), " ", string(e[2])) + if kind(e[3]) == K"lambda" || kind(e[3]) == K"code_info" + println(io) + _print_ir(io, e[3], indent*added_indent) + else + println(io, " ", string(e[3])) + end + elseif kind(e) == K"opaque_closure_method" + @assert numchildren(e) == 5 + print(io, indent, lno, " --- opaque_closure_method ") + for i=1:4 + print(io, " ", e[i]) + end + println(io) + _print_ir(io, e[5], indent*added_indent) + elseif kind(e) == K"code_info" + println(io, indent, lno, " --- ", e.is_toplevel_thunk ? "thunk" : "code_info") + _print_ir(io, e, indent*added_indent) + else + code = string(e) + println(io, indent, lno, " ", code) + end + end +end + From 461ee1631324d079d176733a1b75ae4ebd2bccab Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 21 Jul 2025 22:00:41 +0200 Subject: [PATCH 23/35] rm sources --- Project.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Project.toml b/Project.toml index 8d4721cf..e1525640 100644 --- a/Project.toml +++ b/Project.toml @@ -9,10 +9,6 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -[sources] -JuliaLowering = {path = "dev/JuliaLowering"} -JuliaSyntax = {path = "dev/JuliaSyntax"} - [compat] Aqua = "0.8.4" Compat = "4.15" From 5e0682e5ed7b19333e8dd6e9e420a89c055a13b9 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 21 Jul 2025 22:09:26 +0200 Subject: [PATCH 24/35] wip --- test/lowering.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/lowering.jl b/test/lowering.jl index 3fb8d520..bb4cf430 100644 --- a/test/lowering.jl +++ b/test/lowering.jl @@ -1,9 +1,12 @@ -using JuliaLowering, JuliaSyntax -using JuliaLowering: SyntaxTree, ensure_attributes, showprov -using AbstractTrees +using ExplicitImports.Vendored.JuliaLowering, ExplicitImports.Vendored.JuliaSyntax +using .JuliaLowering: SyntaxTree, ensure_attributes, showprov +using ExplicitImports.Vendored.AbstractTrees + # piracy AbstractTrees.children(t::SyntaxTree) = something(JuliaSyntax.children(t), ()) +include("Exporter.jl") +using Compat # dep of test_mods.jl include("test_mods.jl") src = read("test_mods.jl", String) @@ -26,12 +29,9 @@ ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) leaf = collect(Leaves(ex_scoped))[end - 3] showprov(leaf) -binding_info = ctx3.bindings.info[leaf.var_id] -binding_info.kind == :global - global_bindings = filter(ctx3.bindings.info) do binding # want globals - keep = binding_info.kind == :global + keep = binding.kind == :global # internal ones seem non-interesting (`#self#` etc) keep &= !binding.is_internal From b7b5e9b123d221bd02fd9d20d210d855d6b2fd32 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 22 Jul 2025 00:19:28 +0200 Subject: [PATCH 25/35] use `JuliaLowering.SyntaxTree` --- src/ExplicitImports.jl | 12 +++++++----- src/get_names_used.jl | 15 +++++++-------- src/parse_utilities.jl | 24 ++++++++++++++++-------- test/lowering.jl | 38 +++++++++++++++++++++++++++++++++----- test/runtests.jl | 24 ++++++++++++++---------- 5 files changed, 77 insertions(+), 36 deletions(-) diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 2b58477a..98f2cd29 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -11,6 +11,8 @@ include(joinpath("vendored", "JuliaLowering", "src", "JuliaLowering.jl")) end #! explicit-imports: on +using .Vendored.JuliaLowering + using .Vendored.JuliaSyntax # suppress warning about Base.parse collision, even though parse is never used # this avoids a warning when loading the package while creating an unused explicit import @@ -431,10 +433,10 @@ end include("precompile.jl") -@setup_workload begin - @compile_workload begin - sprint(print_explicit_imports, ExplicitImports, @__FILE__; context=:color => true) - end -end +# @setup_workload begin +# @compile_workload begin +# sprint(print_explicit_imports, ExplicitImports, @__FILE__; context=:color => true) +# end +# end end diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 3530e3fd..dd740172 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -13,7 +13,7 @@ Base.@kwdef struct PerUsageInfo function_arg::Bool is_assignment::Bool module_path::Vector{Symbol} - scope_path::Vector{JuliaSyntax.SyntaxNode} + scope_path::Vector{JuliaLowering.SyntaxTree} struct_field_or_type_param::Bool for_loop_index::Bool generator_index::Bool @@ -357,7 +357,7 @@ function analyze_name(leaf; debug=false) generator_index = is_generator_arg(leaf) catch_arg = is_catch_arg(leaf) module_path = Symbol[] - scope_path = JuliaSyntax.SyntaxNode[] + scope_path = JuliaLowering.SyntaxTree[] is_assignment = false node = leaf idx = 1 @@ -365,11 +365,9 @@ function analyze_name(leaf; debug=false) prev_node = nothing while true # update our state - val = get_val(node) k = kind(node) - args = nodevalue(node).node.raw.children + args = children(nodevalue(node).node) - debug && println(val, ": ", k) # Constructs that start a new local scope. Note `let` & `macro` *arguments* are not explicitly supported/tested yet, # but we can at least keep track of scope properly. if k in @@ -389,7 +387,7 @@ function analyze_name(leaf; debug=false) return kind(arg.node) == K"Identifier" end if !isempty(ids) - push!(module_path, first(ids).node.val) + push!(module_path, get_val(first(ids).node)) end push!(scope_path, nodevalue(node).node) end @@ -444,7 +442,7 @@ function analyze_all_names(file) function_arg::Bool, is_assignment::Bool, module_path::Vector{Symbol}, - scope_path::Vector{JuliaSyntax.SyntaxNode}, + scope_path::Vector{JuliaLowering.SyntaxTree}, struct_field_or_type_param::Bool, for_loop_index::Bool, generator_index::Bool, @@ -495,6 +493,7 @@ function analyze_all_names(file) end ret = analyze_name(leaf) push!(seen_modules, ret.module_path) + # @show name, qualified_by, import_type, explicitly_imported_by, location, ret push!(per_usage_info, (; name, qualified_by, import_type, explicitly_imported_by, location, ret...)) end @@ -532,7 +531,7 @@ end # in JuliaSyntax 1.0. This is very slow and also not quite the semantics we want anyway. # Here, we wrap our nodes in a custom type that only compares object identity. struct SyntaxNodeList - nodes::Vector{JuliaSyntax.SyntaxNode} + nodes::Vector{JuliaLowering.SyntaxTree} end function Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index eb28cb2d..96a4732b 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -8,7 +8,7 @@ # We define a new tree that wraps a `SyntaxNode`. # For this tree, we we add an `AbstractTrees` `children` method to traverse `include` statements to span our tree across files. struct SyntaxNodeWrapper - node::JuliaSyntax.SyntaxNode + node::JuliaLowering.SyntaxTree file::String bad_locations::Set{String} end @@ -33,7 +33,7 @@ function SyntaxNodeWrapper(file::AbstractString; bad_locations=Set{String}()) end end contents = String(take!(stripped)) - parsed = JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, contents; ignore_warnings=true) + parsed = JuliaSyntax.parseall(JuliaLowering.SyntaxTree, contents; ignore_warnings=true) return SyntaxNodeWrapper(parsed, file, bad_locations) end @@ -71,8 +71,8 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) if JuliaSyntax.kind(node) == K"call" children = js_children(node) if length(children) == 2 - f, arg = children::Vector{JuliaSyntax.SyntaxNode} # make JET happy - if f.val === :include + f, arg = children #::Vector{JuliaSyntax.SyntaxNode} # make JET happy + if try_get_val(f) === :include location = location_str(wrapper) if location in wrapper.bad_locations return [SkippedFile(location)] @@ -85,7 +85,8 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) # if we have interpolation, this might not be a string kind(c) == K"String" || @goto dynamic # The children of a static include statement is the entire file being included - new_file = joinpath(dirname(wrapper.file), c.val) + # @show c typeof(c) propertynames(c) + new_file = joinpath(dirname(wrapper.file), c.value) if isfile(new_file) # @debug "Recursing into `$new_file`" node wrapper.file new_wrapper = try_parse_wrapper(new_file; wrapper.bad_locations) @@ -116,12 +117,14 @@ end js_children(n::Union{TreeCursor,SyntaxNodeWrapper}) = js_children(js_node(n)) # https://github.com/JuliaLang/JuliaSyntax.jl/issues/557 -js_children(n::Union{JuliaSyntax.SyntaxNode}) = something(JuliaSyntax.children(n), ()) +js_children(n::JuliaSyntax.SyntaxNode) = something(JuliaSyntax.children(n), ()) + +js_children(n::JuliaLowering.SyntaxTree) = something(JuliaSyntax.children(n), ()) js_node(n::SyntaxNodeWrapper) = n.node js_node(n::TreeCursor) = js_node(nodevalue(n)) -function kind(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode,JuliaSyntax.SyntaxHead}) +function kind(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode,JuliaSyntax.SyntaxHead, JuliaLowering.SyntaxTree}) return JuliaSyntax.kind(n) end kind(n::Union{TreeCursor,SyntaxNodeWrapper}) = kind(js_node(n)) @@ -129,10 +132,15 @@ kind(n::Union{TreeCursor,SyntaxNodeWrapper}) = kind(js_node(n)) head(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode}) = JuliaSyntax.head(n) head(n::Union{TreeCursor,SyntaxNodeWrapper}) = head(js_node(n)) +get_val(n::JuliaLowering.SyntaxTree) = Symbol(n.name_val) get_val(n::JuliaSyntax.SyntaxNode) = n.val get_val(n::Union{TreeCursor,SyntaxNodeWrapper}) = get_val(js_node(n)) -function has_flags(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode}, args...) +try_get_val(n::JuliaLowering.SyntaxTree) = hasproperty(n, :name_val) ? Symbol(n.name_val) : nothing +try_get_val(n::JuliaSyntax.SyntaxNode) = n.val +try_get_val(n::Union{TreeCursor,SyntaxNodeWrapper}) = try_get_val(js_node(n)) + +function has_flags(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode, JuliaLowering.SyntaxTree}, args...) return JuliaSyntax.has_flags(n, args...) end has_flags(n::Union{TreeCursor,SyntaxNodeWrapper}, args...) = has_flags(js_node(n), args...) diff --git a/test/lowering.jl b/test/lowering.jl index bb4cf430..5649801f 100644 --- a/test/lowering.jl +++ b/test/lowering.jl @@ -1,6 +1,9 @@ +# Scratch work - this file is not included anywhere + using ExplicitImports.Vendored.JuliaLowering, ExplicitImports.Vendored.JuliaSyntax using .JuliaLowering: SyntaxTree, ensure_attributes, showprov using ExplicitImports.Vendored.AbstractTrees +using ExplicitImports.Vendored.AbstractTrees: parent # piracy AbstractTrees.children(t::SyntaxTree) = something(JuliaSyntax.children(t), ()) @@ -10,14 +13,30 @@ using Compat # dep of test_mods.jl include("test_mods.jl") src = read("test_mods.jl", String) -tree = parseall(JuliaLowering.SyntaxTree, src; filename="tests_mods.jl") +tree = TreeCursor(parseall(JuliaLowering.SyntaxTree, src; filename="tests_mods.jl")) + +cchildren(x) = collect(children(x)) +testmod1_code = cchildren(cchildren(tree)[2])[2] +func = cchildren(testmod1_code)[end - 1] + +leaf = cchildren(func)[2] +nodevalue(leaf) # print_explicit_imports + +nodevalue(AbstractTrees.parent(leaf)) # call defining f + -testmod1_code = JuliaSyntax.children(JuliaSyntax.children(tree)[2])[2] -func = JuliaSyntax.children(testmod1_code)[end - 1] -leaf = JuliaSyntax.children(func)[2] +cchildren(x) = collect(children(x)) +testmod1_code = cchildren(cchildren(tree)[2])[2] +func = cchildren(testmod1_code)[end] -ex = testmod1_code +leaf = cchildren(cchildren(func)[2])[2] +nodevalue(leaf) # check_no_implicit_imports as an Identifier + +nodevalue(AbstractTrees.parent(leaf)) # . with ExplicitImports and check_no_implicit_imports + + +ex = nodevalue(testmod1_code) ex = ensure_attributes(ex; var_id=Int) in_mod = TestMod1 @@ -56,3 +75,12 @@ end # so if we want to check you are calling it from the "right" module, we need to follow the tree, # find this pattern, then check the module against the symbol. # That's what we already do, but now we should have more precision in knowing the module I think + +# Ok, so something we could do is basically like what we do now in `get_names_used`: +# 1. find the leaves. Throw out anything whose `kind` isn't a BindingId or a Symbol (I think) +# 2. Symbols may be qualified; check if the parent is `.`. Then the first parent is the module(?). If not qualified then not interesting (?) +# 3. BindingIds are potential globals, and are potentially qualifying another name + +leaf = collect(Leaves(TreeCursor(ex_scoped)))[end-3] +nodevalue(leaf) +nodevalue(AbstractTrees.parent(leaf)) diff --git a/test/runtests.jl b/test/runtests.jl index 71318408..8d9bc72b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,7 +13,7 @@ using DataFrames using Aqua using Logging, UUIDs using ExplicitImports.Vendored.AbstractTrees -using ExplicitImports: is_function_definition_arg, SyntaxNodeWrapper, get_val +using ExplicitImports: is_function_definition_arg, SyntaxNodeWrapper, try_get_val using ExplicitImports: is_struct_type_param, is_struct_field_name, is_for_arg, is_generator_arg, analyze_qualified_names using TestPkg, Markdown @@ -74,12 +74,16 @@ include("script.jl") include("imports.jl") include("test_qualified_access.jl") include("test_explicit_imports.jl") -include("main.jl") include("Test_Mod_Underscores.jl") include("module_alias.jl") include("issue_129.jl") @testset "ExplicitImports" begin + + @testset "main() function" begin + include("main.jl") + end + @testset "deprecations" begin include("deprecated.jl") end @@ -145,7 +149,7 @@ include("issue_129.jl") @testset "imports" begin cursor = TreeCursor(SyntaxNodeWrapper("imports.jl")) leaves = collect(Leaves(cursor)) - import_type_pairs = get_val.(leaves) .=> analyze_import_type.(leaves) + import_type_pairs = try_get_val.(leaves) .=> analyze_import_type.(leaves) filter!(import_type_pairs) do (k, v) return v !== :not_import end @@ -181,7 +185,7 @@ include("issue_129.jl") :f => :import_RHS] inds = findall(==(:import_RHS), analyze_import_type.(leaves)) - lhs_rhs_pairs = get_import_lhs.(leaves[inds]) .=> get_val.(leaves[inds]) + lhs_rhs_pairs = get_import_lhs.(leaves[inds]) .=> try_get_val.(leaves[inds]) @test lhs_rhs_pairs == [[:., :., :Exporter] => :exported_a, [:., :., :Exporter] => :exported_c, [:., :., :Exporter] => :exported_c, @@ -494,9 +498,9 @@ include("issue_129.jl") @testset "structs" begin cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) leaves = collect(Leaves(cursor)) - @test map(get_val, filter(is_struct_type_param, leaves)) == [:X, :Y, :QR] + @test map(try_get_val, filter(is_struct_type_param, leaves)) == [:X, :Y, :QR] - @test map(get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] + @test map(try_get_val, filter(is_struct_field_name, leaves)) == [:x, :x, :x, :qr, :qr] df = DataFrame(get_names_used("test_mods.jl").per_usage_info) subset!(df, :name => ByRow(==(:QR)), :module_path => ByRow(==([:TestMod5]))) @@ -513,7 +517,7 @@ include("issue_129.jl") @testset "loops" begin cursor = TreeCursor(SyntaxNodeWrapper("test_mods.jl")) leaves = collect(Leaves(cursor)) - @test map(get_val, filter(is_for_arg, leaves)) == + @test map(try_get_val, filter(is_for_arg, leaves)) == [:i, :I, :j, :k, :k, :j, :xi, :yi] # Tests #35 @@ -541,7 +545,7 @@ include("issue_129.jl") v = [:i1, :I, :i2, :I, :i3, :I, :i4, :I] w = [:i1, :I] - @test map(get_val, filter(is_generator_arg, leaves)) == + @test map(try_get_val, filter(is_generator_arg, leaves)) == [v; v; w; w; w; w; w] @test using_statement.(explicit_imports_nonrecursive(TestMod9, "test_mods.jl")) == @@ -673,13 +677,13 @@ include("issue_129.jl") purported_function_args = filter(is_function_definition_arg, leaves) # written this way to get clearer test failure messages - vals = unique(get_val.(purported_function_args)) + vals = unique(try_get_val.(purported_function_args)) @test vals == [:a] # we have 9*4 functions with one argument `a`, plus 2 macros @test length(purported_function_args) == 9 * 4 + 2 non_function_args = filter(!is_function_definition_arg, leaves) - missed = filter(x -> get_val(x) === :a, non_function_args) + missed = filter(x -> try_get_val(x) === :a, non_function_args) @test isempty(missed) # https://github.com/JuliaTesting/ExplicitImports.jl/issues/129 From 147d58a883a54d832db16dd2f928130451bb0dbf Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 22 Jul 2025 00:27:52 +0200 Subject: [PATCH 26/35] do some lowering --- src/parse_utilities.jl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index 96a4732b..3d692387 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -11,6 +11,7 @@ struct SyntaxNodeWrapper node::JuliaLowering.SyntaxTree file::String bad_locations::Set{String} + context::Any end const OFF = "#! explicit-imports: off" @@ -34,7 +35,15 @@ function SyntaxNodeWrapper(file::AbstractString; bad_locations=Set{String}()) end contents = String(take!(stripped)) parsed = JuliaSyntax.parseall(JuliaLowering.SyntaxTree, contents; ignore_warnings=true) - return SyntaxNodeWrapper(parsed, file, bad_locations) + + # Perform lowering on the parse tree until scoping + ex = JuliaLowering.ensure_attributes(parsed; var_id=Int) + in_mod = Main + ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex) + ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) + ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) + + return SyntaxNodeWrapper(ex_scoped, file, bad_locations, ctx3) end function try_parse_wrapper(file::AbstractString; bad_locations) @@ -110,7 +119,7 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) end end end - return map(n -> SyntaxNodeWrapper(n, wrapper.file, wrapper.bad_locations), + return map(n -> SyntaxNodeWrapper(n, wrapper.file, wrapper.bad_locations, wrapper.context), js_children(node)) end From 30f9fab3e8ba65958771d0700018deef3e1738b2 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 22 Jul 2025 00:46:16 +0200 Subject: [PATCH 27/35] add issue 120 --- test/issue_120.jl | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/issue_120.jl diff --git a/test/issue_120.jl b/test/issue_120.jl new file mode 100644 index 00000000..a699e4cb --- /dev/null +++ b/test/issue_120.jl @@ -0,0 +1,9 @@ +module Foo120 + +using Base: wrap_string + +function f(wrap_string = wrap_string("foo", UInt32(1))) + print(wrap_string) +end + +end From 92790fd1a894182a3832d3138ec4b2985803568e Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 22 Jul 2025 16:22:44 +0200 Subject: [PATCH 28/35] wip --- src/ExplicitImports.jl | 8 ++++---- src/deprecated.jl | 4 ++-- src/get_names_used.jl | 20 ++++++++++++-------- src/improper_explicit_imports.jl | 6 +++--- src/improper_qualified_accesses.jl | 6 +++--- src/parse_utilities.jl | 28 ++++++++++++++++------------ 6 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 98f2cd29..10e52da5 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -142,7 +142,7 @@ function explicit_imports(mod::Module, file=pathof(mod); skip=(mod, Base, Core), end submodules = find_submodules(mod, file) - fill_cache!(file_analysis, last.(submodules)) + fill_cache!(file_analysis, last.(submodules), mod) return [submodule => explicit_imports_nonrecursive(submodule, path; skip, warn_stale, file_analysis=file_analysis[path], strict) @@ -209,7 +209,7 @@ function explicit_imports_nonrecursive(mod::Module, file=pathof(mod); # deprecated warn_stale=nothing, # private undocumented kwarg for hoisting this analysis - file_analysis=get_names_used(file)) + file_analysis=get_names_used(file, mod)) check_file(file) if warn_stale !== nothing @warn "[explicit_imports_nonrecursive] keyword argument `warn_stale` is deprecated and does nothing" _id = :explicit_imports_explicit_imports_warn_stale maxlog = 1 @@ -405,10 +405,10 @@ function find_submodule_path(file, submodule) return path end -function fill_cache!(file_analysis::Dict, files) +function fill_cache!(file_analysis::Dict, files, mod) for _file in files if !haskey(file_analysis, _file) - file_analysis[_file] = get_names_used(_file) + file_analysis[_file] = get_names_used(_file, mod) end end return file_analysis diff --git a/src/deprecated.jl b/src/deprecated.jl index 18907e2a..a21ec975 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -3,7 +3,7 @@ function stale_explicit_imports(mod::Module, file=pathof(mod); strict=true) @warn "[stale_explicit_imports] deprecated in favor of `improper_explicit_imports`" _id = :explicit_imports_stale_explicit_imports maxlog = 1 submodules = find_submodules(mod, file) file_analysis = Dict{String,FileAnalysis}() - fill_cache!(file_analysis, last.(submodules)) + fill_cache!(file_analysis, last.(submodules), mod) return [submodule => stale_explicit_imports_nonrecursive(submodule, path; file_analysis=file_analysis[path], strict) @@ -13,7 +13,7 @@ end function stale_explicit_imports_nonrecursive(mod::Module, file=pathof(mod); strict=true, # private undocumented kwarg for hoisting this analysis - file_analysis=get_names_used(file)) + file_analysis=get_names_used(file, mod)) check_file(file) @warn "[stale_explicit_imports_nonrecursive] deprecated in favor of `improper_explicit_imports_nonrecursive`" _id = :explicit_imports_stale_explicit_imports maxlog = 1 diff --git a/src/get_names_used.jl b/src/get_names_used.jl index dd740172..4c85166f 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -349,7 +349,7 @@ end # a leaf and follow the parents up to see what scopes our leaf is in. # TODO-someday- cleanup. This basically has two jobs: check is function arg etc, and figure out the scope/module path. # We could do these two things separately for more clarity. -function analyze_name(leaf; debug=false) +function analyze_name(leaf) # Ok, we have a "name". Let us work our way up and try to figure out if it is in local scope or not function_arg = is_function_definition_arg(leaf) struct_field_or_type_param = is_struct_type_param(leaf) || is_struct_field_name(leaf) @@ -359,9 +359,13 @@ function analyze_name(leaf; debug=false) module_path = Symbol[] scope_path = JuliaLowering.SyntaxTree[] is_assignment = false + is_global = false node = leaf idx = 1 - + @show try_get_val(nodevalue(leaf)) + @show kind(nodevalue(get_parent(leaf, 1))) + global CTX = nodevalue(leaf).context + global LEAF = leaf prev_node = nothing while true # update our state @@ -415,17 +419,17 @@ function analyze_name(leaf; debug=false) end """ - analyze_all_names(file) + analyze_all_names(file, in_mod::Module) Returns a tuple of two items: * `per_usage_info`: a table containing information about each name each time it was used * `untainted_modules`: a set containing modules found and analyzed successfully """ -function analyze_all_names(file) +function analyze_all_names(file, in_mod::Module) # we don't use `try_parse_wrapper` here, since there's no recovery possible # (no other files we know about to look at) - tree = SyntaxNodeWrapper(file) + tree = SyntaxNodeWrapper(file, in_mod) # in local scope, a name refers to a global if it is read from before it is assigned to, OR if the global keyword is used # a name refers to a local otherwise # so we need to traverse the tree, keeping track of state like: which scope are we in, and for each name, in each scope, has it been used @@ -647,7 +651,7 @@ function setdiff_no_metadata(set1, set2) end """ - get_names_used(file) -> FileAnalysis + get_names_used(file, in_mod::Module) -> FileAnalysis Figures out which global names are used in `file`, and what modules they are used within. @@ -655,10 +659,10 @@ Traverses static `include` statements. Returns a `FileAnalysis` object. """ -function get_names_used(file) +function get_names_used(file, in_mod::Module) check_file(file) # Here we get 1 row per name per usage - per_usage_info, untainted_modules = analyze_all_names(file) + per_usage_info, untainted_modules = analyze_all_names(file, in_mod) names_used_for_global_bindings = get_global_names(per_usage_info) explicit_imports = get_explicit_imports(per_usage_info) diff --git a/src/improper_explicit_imports.jl b/src/improper_explicit_imports.jl index 34d10566..5262ed59 100644 --- a/src/improper_explicit_imports.jl +++ b/src/improper_explicit_imports.jl @@ -1,6 +1,6 @@ function analyze_explicitly_imported_names(mod::Module, file=pathof(mod); # private undocumented kwarg for hoisting this analysis - file_analysis=get_names_used(file)) + file_analysis=get_names_used(file, mod)) check_file(file) (; per_usage_info, unnecessary_explicit_import, tainted) = filter_to_module(file_analysis, mod) @@ -174,7 +174,7 @@ function improper_explicit_imports_nonrecursive(mod::Module, file=pathof(mod); strict=true, allow_internal_imports=true, # private undocumented kwarg for hoisting this analysis - file_analysis=get_names_used(file)) + file_analysis=get_names_used(file, mod)) check_file(file) problematic, tainted = analyze_explicitly_imported_names(mod, file; file_analysis) @@ -255,7 +255,7 @@ function improper_explicit_imports(mod::Module, file=pathof(mod); strict=true, check_file(file) submodules = find_submodules(mod, file) file_analysis = Dict{String,FileAnalysis}() - fill_cache!(file_analysis, last.(submodules)) + fill_cache!(file_analysis, last.(submodules), mod) return [submodule => improper_explicit_imports_nonrecursive(submodule, path; strict, file_analysis=file_analysis[path], skip, diff --git a/src/improper_qualified_accesses.jl b/src/improper_qualified_accesses.jl index 0b29685e..b04de556 100644 --- a/src/improper_qualified_accesses.jl +++ b/src/improper_qualified_accesses.jl @@ -3,7 +3,7 @@ function analyze_qualified_names(mod::Module, file=pathof(mod); # private undocumented kwarg for hoisting this analysis - file_analysis=get_names_used(file)) + file_analysis=get_names_used(file, mod)) check_file(file) (; per_usage_info, tainted) = filter_to_module(file_analysis, mod) # Do we want to do anything with `tainted`? This means there is unanalyzable code here @@ -128,7 +128,7 @@ function improper_qualified_accesses_nonrecursive(mod::Module, file=pathof(mod); # deprecated, does nothing require_submodule_access=nothing, # private undocumented kwarg for hoisting this analysis - file_analysis=get_names_used(file)) + file_analysis=get_names_used(file, mod)) check_file(file) if require_submodule_access !== nothing @warn "[improper_qualified_accesses_nonrecursive] `require_submodule_access` is deprecated and unused" _id = :explicit_imports_improper_qualified_accesses_require_submodule_access maxlog = 1 @@ -229,7 +229,7 @@ function improper_qualified_accesses(mod::Module, file=pathof(mod); end submodules = find_submodules(mod, file) file_analysis = Dict{String,FileAnalysis}() - fill_cache!(file_analysis, last.(submodules)) + fill_cache!(file_analysis, last.(submodules), mod) return [submodule => improper_qualified_accesses_nonrecursive(submodule, path; file_analysis=file_analysis[path], skip, diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index 3d692387..72a0ef17 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -12,12 +12,13 @@ struct SyntaxNodeWrapper file::String bad_locations::Set{String} context::Any + in_mod::Module end const OFF = "#! explicit-imports: off" const ON = "#! explicit-imports: on" -function SyntaxNodeWrapper(file::AbstractString; bad_locations=Set{String}()) +function SyntaxNodeWrapper(file::AbstractString, in_mod::Module; bad_locations=Set{String}()) stripped = IOBuffer() on = true for line in eachline(file; keep=true) @@ -25,7 +26,7 @@ function SyntaxNodeWrapper(file::AbstractString; bad_locations=Set{String}()) on = false end - if strip(line) == ON + if strip(line) == ON on = true end @@ -38,17 +39,16 @@ function SyntaxNodeWrapper(file::AbstractString; bad_locations=Set{String}()) # Perform lowering on the parse tree until scoping ex = JuliaLowering.ensure_attributes(parsed; var_id=Int) - in_mod = Main ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex) ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) - return SyntaxNodeWrapper(ex_scoped, file, bad_locations, ctx3) + return SyntaxNodeWrapper(ex_scoped, file, bad_locations, ctx3, in_mod) end -function try_parse_wrapper(file::AbstractString; bad_locations) +function try_parse_wrapper(file::AbstractString, mod::Module; bad_locations) return try - SyntaxNodeWrapper(file; bad_locations) + SyntaxNodeWrapper(file, mod; bad_locations) catch e msg = "Error when parsing file. Skipping this file." @error msg file exception = (e, catch_backtrace()) @@ -98,7 +98,7 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) new_file = joinpath(dirname(wrapper.file), c.value) if isfile(new_file) # @debug "Recursing into `$new_file`" node wrapper.file - new_wrapper = try_parse_wrapper(new_file; wrapper.bad_locations) + new_wrapper = try_parse_wrapper(new_file, wrapper.in_mod; wrapper.bad_locations) if new_wrapper === nothing push!(wrapper.bad_locations, location) return [SkippedFile(location)] @@ -119,7 +119,8 @@ function AbstractTrees.children(wrapper::SyntaxNodeWrapper) end end end - return map(n -> SyntaxNodeWrapper(n, wrapper.file, wrapper.bad_locations, wrapper.context), + return map(n -> SyntaxNodeWrapper(n, wrapper.file, wrapper.bad_locations, + wrapper.context, wrapper.in_mod), js_children(node)) end @@ -133,7 +134,8 @@ js_children(n::JuliaLowering.SyntaxTree) = something(JuliaSyntax.children(n), () js_node(n::SyntaxNodeWrapper) = n.node js_node(n::TreeCursor) = js_node(nodevalue(n)) -function kind(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode,JuliaSyntax.SyntaxHead, JuliaLowering.SyntaxTree}) +function kind(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode,JuliaSyntax.SyntaxHead, + JuliaLowering.SyntaxTree}) return JuliaSyntax.kind(n) end kind(n::Union{TreeCursor,SyntaxNodeWrapper}) = kind(js_node(n)) @@ -145,11 +147,14 @@ get_val(n::JuliaLowering.SyntaxTree) = Symbol(n.name_val) get_val(n::JuliaSyntax.SyntaxNode) = n.val get_val(n::Union{TreeCursor,SyntaxNodeWrapper}) = get_val(js_node(n)) -try_get_val(n::JuliaLowering.SyntaxTree) = hasproperty(n, :name_val) ? Symbol(n.name_val) : nothing +function try_get_val(n::JuliaLowering.SyntaxTree) + return hasproperty(n, :name_val) ? Symbol(n.name_val) : nothing +end try_get_val(n::JuliaSyntax.SyntaxNode) = n.val try_get_val(n::Union{TreeCursor,SyntaxNodeWrapper}) = try_get_val(js_node(n)) -function has_flags(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode, JuliaLowering.SyntaxTree}, args...) +function has_flags(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode, + JuliaLowering.SyntaxTree}, args...) return JuliaSyntax.has_flags(n, args...) end has_flags(n::Union{TreeCursor,SyntaxNodeWrapper}, args...) = has_flags(js_node(n), args...) @@ -174,7 +179,6 @@ function parents_match(n::TreeCursor, kinds::Tuple) return parents_match(p, Base.tail(kinds)) end - function parent_kinds(n::TreeCursor) kinds = [] while true From a48b1ad322950a19052902f3d735317098cb3501 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 22 Jul 2025 16:39:12 +0200 Subject: [PATCH 29/35] wip --- src/get_names_used.jl | 4 ++-- src/parse_utilities.jl | 21 +++++++++++++++++++ .../JuliaLowering/src/JuliaLowering.jl | 2 +- vendor/run.jl | 4 +++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/get_names_used.jl b/src/get_names_used.jl index 4c85166f..c84be097 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -345,6 +345,7 @@ function is_double_colon_LHS(leaf) return child_index(leaf) == 1 end +DEBUG = [] # Here we use the magic of AbstractTrees' `TreeCursor` so we can start at # a leaf and follow the parents up to see what scopes our leaf is in. # TODO-someday- cleanup. This basically has two jobs: check is function arg etc, and figure out the scope/module path. @@ -364,8 +365,7 @@ function analyze_name(leaf) idx = 1 @show try_get_val(nodevalue(leaf)) @show kind(nodevalue(get_parent(leaf, 1))) - global CTX = nodevalue(leaf).context - global LEAF = leaf + push!(DEBUG, leaf) prev_node = nothing while true # update our state diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index 72a0ef17..388a825b 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -15,6 +15,17 @@ struct SyntaxNodeWrapper in_mod::Module end +function Base.show(io::IO, n::SyntaxNodeWrapper) + print(io, "SyntaxNodeWrapper: ") + show(io, n.node) +end + +function Base.show(io::IO, mime::MIME"text/plain", n::SyntaxNodeWrapper) + print(io, "SyntaxNodeWrapper: ") + show(io, mime, n.node) + print(io, "File: ", n.file) +end + const OFF = "#! explicit-imports: off" const ON = "#! explicit-imports: on" @@ -204,3 +215,13 @@ function has_parent(n, i=1) end return true end + +# these would be piracy, but we've vendored AbstractTrees so it's technically fine +function Base.show(io::IO, cursor::AbstractTrees.ImplicitCursor) + print(io, "ImplicitCursor: ") + show(io, nodevalue(cursor)) +end +function Base.show(io::IO, mime::MIME"text/plain", cursor::AbstractTrees.ImplicitCursor) + print(io, "ImplicitCursor: ") + show(io, mime, nodevalue(cursor)) +end diff --git a/src/vendored/JuliaLowering/src/JuliaLowering.jl b/src/vendored/JuliaLowering/src/JuliaLowering.jl index fba6ec19..d02b3294 100644 --- a/src/vendored/JuliaLowering/src/JuliaLowering.jl +++ b/src/vendored/JuliaLowering/src/JuliaLowering.jl @@ -4,7 +4,7 @@ baremodule JuliaLowering using Base # We define a separate _include() for use in this module to avoid mixing method # tables with the public `JuliaLowering.include()` API -_include(path::AbstractString) = Base.include(JuliaLowering, path) +_include(path::AbstractString) = Base.include(JuliaLowering, joinpath(@__DIR__, path)) using Core: eval using ..JuliaSyntax diff --git a/vendor/run.jl b/vendor/run.jl index 99b3311a..6631a360 100644 --- a/vendor/run.jl +++ b/vendor/run.jl @@ -28,7 +28,9 @@ for pkg in deps contents = replace(read(joinpath(root, file), String), "using JuliaSyntax" => "using ..JuliaSyntax", # remove unnecessary `using JuliaLowering` from src/hooks.jl - "using JuliaLowering" => "") + "using JuliaLowering" => "", + # for some reason Revise seems to need this: + "_include(path::AbstractString) = Base.include(JuliaLowering, path)" => "_include(path::AbstractString) = Base.include(JuliaLowering, joinpath(@__DIR__, path))") chmod(joinpath(root, file), 0o666) # make writable write(abspath(joinpath(root, file)), contents) chmod(joinpath(root, file), 0o444) # back to read-only From b8f35de9a029db8c30716ec2dec36b1e88c8d9e9 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Wed, 30 Jul 2025 23:07:17 +0200 Subject: [PATCH 30/35] wip --- src/ExplicitImports.jl | 3 + src/get_names_used.jl | 4 +- src/parse_utilities.jl | 25 +++++-- test/lowering.jl | 32 +++++++-- test/wip.jl | 32 +++++++++ test/wip2.jl | 67 ++++++++++++++++++ test/wip3.jl | 153 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 test/wip.jl create mode 100644 test/wip2.jl create mode 100644 test/wip3.jl diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 10e52da5..64468c32 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -26,6 +26,9 @@ using Markdown: Markdown using PrecompileTools: @setup_workload, @compile_workload using Pkg: Pkg +# debug +parsefile + # we'll borrow their `@_public` macro; if this goes away, we can get our own JuliaSyntax.@_public ignore_submodules diff --git a/src/get_names_used.jl b/src/get_names_used.jl index c84be097..9862400e 100644 --- a/src/get_names_used.jl +++ b/src/get_names_used.jl @@ -363,8 +363,8 @@ function analyze_name(leaf) is_global = false node = leaf idx = 1 - @show try_get_val(nodevalue(leaf)) - @show kind(nodevalue(get_parent(leaf, 1))) + # @show try_get_val(nodevalue(leaf)) + # @show kind(nodevalue(get_parent(leaf, 1))) push!(DEBUG, leaf) prev_node = nothing while true diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index 388a825b..50ebfb5a 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -138,9 +138,9 @@ end js_children(n::Union{TreeCursor,SyntaxNodeWrapper}) = js_children(js_node(n)) # https://github.com/JuliaLang/JuliaSyntax.jl/issues/557 -js_children(n::JuliaSyntax.SyntaxNode) = something(JuliaSyntax.children(n), ()) - -js_children(n::JuliaLowering.SyntaxTree) = something(JuliaSyntax.children(n), ()) +function js_children(n::Union{JuliaSyntax.SyntaxNode,JuliaLowering.SyntaxTree}) + return something(JuliaSyntax.children(n), ()) +end js_node(n::SyntaxNodeWrapper) = n.node js_node(n::TreeCursor) = js_node(nodevalue(n)) @@ -164,7 +164,8 @@ end try_get_val(n::JuliaSyntax.SyntaxNode) = n.val try_get_val(n::Union{TreeCursor,SyntaxNodeWrapper}) = try_get_val(js_node(n)) -function has_flags(n::Union{JuliaSyntax.SyntaxNode,JuliaSyntax.GreenNode, +function has_flags(n::Union{JuliaSyntax.SyntaxNode, + JuliaSyntax.GreenNode, JuliaLowering.SyntaxTree}, args...) return JuliaSyntax.has_flags(n, args...) end @@ -219,9 +220,21 @@ end # these would be piracy, but we've vendored AbstractTrees so it's technically fine function Base.show(io::IO, cursor::AbstractTrees.ImplicitCursor) print(io, "ImplicitCursor: ") - show(io, nodevalue(cursor)) + return show(io, nodevalue(cursor)) end function Base.show(io::IO, mime::MIME"text/plain", cursor::AbstractTrees.ImplicitCursor) print(io, "ImplicitCursor: ") - show(io, mime, nodevalue(cursor)) + return show(io, mime, nodevalue(cursor)) +end + +function Base.show(io::IO, mime::MIME"text/plain", + ctx::JuliaLowering.VariableAnalysisContext) + println(io, + """VariableAnalysisContext with module $(ctx.mod) and + - $(length(ctx.bindings.info)) bindings + - $(length(ctx.closure_bindings)) closure bindings + - $(length(ctx.closure_bindings)) lambda bindings + - $(length(ctx.method_def_stack))-long method def stack + and graph:""") + return show(io, mime, ctx.graph) end diff --git a/test/lowering.jl b/test/lowering.jl index 5649801f..26f23b41 100644 --- a/test/lowering.jl +++ b/test/lowering.jl @@ -13,7 +13,24 @@ using Compat # dep of test_mods.jl include("test_mods.jl") src = read("test_mods.jl", String) -tree = TreeCursor(parseall(JuliaLowering.SyntaxTree, src; filename="tests_mods.jl")) +tree = parseall(JuliaLowering.SyntaxTree, src; filename="tests_mods.jl") + +src = """ +module Foo129 +foo() = 3 +h(f) = 4 +h(f, f2) = 4 +module Bar +using ..Foo129: foo, h +bar() = h(foo) + +# we will test that the LHS foo is a function arg and the RHS ones are not +bar2(x, foo) = h(foo, foo) +end # Bar +end # Foo129 +""" + +tree = parseall(JuliaLowering.SyntaxTree, src; filename="tests_mods.jl") cchildren(x) = collect(children(x)) testmod1_code = cchildren(cchildren(tree)[2])[2] @@ -41,9 +58,9 @@ ex = ensure_attributes(ex; var_id=Int) in_mod = TestMod1 # in_mod=Main -ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex) -ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) -ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) +ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex); +ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand); +ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar); leaf = collect(Leaves(ex_scoped))[end - 3] showprov(leaf) @@ -84,3 +101,10 @@ end leaf = collect(Leaves(TreeCursor(ex_scoped)))[end-3] nodevalue(leaf) nodevalue(AbstractTrees.parent(leaf)) + + +## +using ExplicitImports.Vendored.JuliaLowering, ExplicitImports.Vendored.JuliaSyntax + +src = read("issue_120.jl", String) +tree = JuliaSyntax.parseall(JuliaLowering.SyntaxTree, src; filename="issue_120.jl") diff --git a/test/wip.jl b/test/wip.jl new file mode 100644 index 00000000..5d64b30d --- /dev/null +++ b/test/wip.jl @@ -0,0 +1,32 @@ + +using .JuliaLowering: SyntaxTree, ensure_attributes, showprov +using .JuliaSyntax, .JuliaLowering + +src = """ +module Foo129 +foo() = 3 +h(f) = 4 +h(f, f2) = 4 +module Bar +using ..Foo129: foo, h +bar() = h(foo) + +# we will test that the LHS foo is a function arg and the RHS ones are not +bar2(x, foo) = h(foo, foo) +end # Bar +end # Foo129 +""" + +tree = parseall(JuliaLowering.SyntaxTree, src; filename="file.jl") + +ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(Main, tree); +ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand); +ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar); +ex_scoped + +ex = JuliaSyntax.children(JuliaSyntax.children(tree)[1])[2] + +ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(Main, ex); +ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand); +ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar); +ex_scoped diff --git a/test/wip2.jl b/test/wip2.jl new file mode 100644 index 00000000..b6e992d4 --- /dev/null +++ b/test/wip2.jl @@ -0,0 +1,67 @@ +using ExplicitImports +using ExplicitImports.Vendored.JuliaLowering, ExplicitImports.Vendored.JuliaSyntax + +using .JuliaLowering: SyntaxTree, MacroExpansionContext, DesugaringContext, reparent +using .JuliaSyntax, .JuliaLowering + +using .JuliaSyntax: children + +src = """ +module Foo129 +foo() = 3 +h(f) = 4 +module Bar + using ..Foo129: foo, h + bar() = h(foo) + bar2(x, foo) = h(foo) +end # Bar +end # Foo129 +""" + +# 1. Parse the entire file content. +tree = parseall(SyntaxTree, src; filename="file.jl") + +# 2. Set up an initial context. This context will be updated as we process +# each top-level statement. +graph = JuliaLowering.ensure_attributes(JuliaLowering.syntax_graph(tree), + var_id=Int, scope_layer=Int, + # and any other attributes your passes need... + lambda_bindings=JuliaLowering.LambdaBindings) +layers = [JuliaLowering.ScopeLayer(1, Main, false)] +bindings = JuliaLowering.Bindings() +# This `macro_ctx` will be updated after each statement. +macro_ctx = MacroExpansionContext(graph, bindings, layers, layers[1]) + +resolved_expressions = [] + +# 3. Iterate through top-level statements, threading the context. +@show children(tree) +for stmt in children(tree) + println("hi") + # Use the context from the previous step. + current_macro_ctx = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + ex1 = JuliaLowering.expand_forms_1(current_macro_ctx, stmt) + ctx2 = DesugaringContext(current_macro_ctx) + ex2 = JuliaLowering.expand_forms_2(ctx2, ex1) + + # This is the key step that resolves bindings. + ctx3, ex3 = JuliaLowering.resolve_scopes(ctx2, ex2) + push!(resolved_expressions, ex3) + + # 4. Update the main context with the results from the statement we just processed. + # The `bindings` and `scope_layers` tables have been mutated by the lowering passes. + global macro_ctx + macro_ctx = MacroExpansionContext(ctx3.graph, ctx3.bindings, current_macro_ctx.scope_layers, + current_macro_ctx.current_layer) # Top-level layer doesn't change + global tree + tree = reparent(macro_ctx, tree) + @show children(tree) +end + +# Now, `resolved_expressions` is a vector of fully-scoped trees for each top-level statement, +# and `macro_ctx.bindings` contains the complete binding table for the entire file. +# You can now analyze these resolved trees. +combined_tree = JuliaLowering.makenode(macro_ctx, tree, K"toplevel", + resolved_expressions...) diff --git a/test/wip3.jl b/test/wip3.jl new file mode 100644 index 00000000..27dd7091 --- /dev/null +++ b/test/wip3.jl @@ -0,0 +1,153 @@ +using ExplicitImports +using ExplicitImports.Vendored.JuliaLowering +using ExplicitImports.Vendored.JuliaSyntax +using .JuliaSyntax: parseall, children, numchildren +using .JuliaLowering: SyntaxTree, MacroExpansionContext, DesugaringContext, + ensure_attributes, expand_forms_1, expand_forms_2, resolve_scopes, + reparent, ScopeLayer, + Bindings, IdTag, LayerId, LambdaBindings, eval_module, + syntax_graph, children, kind, makenode, is_leaf, @K_str, + Bindings + +""" + lower_file(tree, into_mod=Main) -> scoped_toplevel + +Statically lowers `tree` (the result of `parseall`) **and every nested module +body it contains**, returning a single `K"toplevel"` that carries full +`BindingId` information. +""" +function lower_file(tree::SyntaxTree, into_mod::Module=Main) + # --- initial context -------------------------------------------------- + graph = ensure_attributes(syntax_graph(tree); + var_id=Int, + scope_layer=Int, + lambda_bindings=LambdaBindings, + bindings=Bindings) + + layers = [ScopeLayer(1, into_mod, false)] + bindings = Bindings() + ctx = MacroExpansionContext(graph, bindings, layers, layers[1]) + + resolved = SyntaxTree[] + last_ctx = _process_block!(ctx, tree, resolved) + + # stitch everything back together so callers get one tree + return makenode(last_ctx, tree, K"toplevel", resolved...), last_ctx +end + +# ------------------------------------------------------------------------- +# Internal helpers +# ------------------------------------------------------------------------- + +function _process_block!(macro_ctx, blk, out) + for stmt in children(blk) + macro_ctx = _process_stmt!(macro_ctx, stmt, out) # ← keep the new ctx + end + return macro_ctx +end + +function _process_stmt!(macro_ctx, stmt, out) + ctx1 = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + ex1 = expand_forms_1(ctx1, stmt) + ctx2 = DesugaringContext(ctx1) + ex2 = expand_forms_2(ctx2, ex1) + ctx3, ex3 = resolve_scopes(ctx2, ex2) + + push!(out, ex3) + + # Wrap the *richer* graph / bindings in a fresh context + next_ctx = MacroExpansionContext(ctx3.graph, ctx3.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + # Depth-first search for nested modules + return _visit_nested!(next_ctx, ex3, out) # returns an updated context +end + +function _visit_nested!(macro_ctx, ex, out) + k = kind(ex) + + if k == K"inert" + for c in children(ex) + macro_ctx = _visit_nested!(macro_ctx, c, out) + end + return macro_ctx + end + + if k == K"call" && + kind(ex[1]) == K"Value" && ex[1].value === eval_module && + kind(ex[4]) == K"inert" + + parent = (kind(ex[2]) == K"Value" && ex[2].value isa Module) ? + ex[2].value : Main + modname = Symbol(ex[3].value) + + childmod = Base.isdefined(parent, modname) ? + getfield(parent, modname) : + ( @warn "Module $modname not found in $(parent); using dummy." ; + Module(modname) ) + + new_layer = ScopeLayer(length(macro_ctx.scope_layers)+1, childmod, false) + scope_layers′ = [macro_ctx.scope_layers; new_layer] + + inner_ctx = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, + scope_layers′, new_layer) + + # recurse into the body of the nested module + inner_ctx = _process_block!(inner_ctx, children(ex[4])[1], out) + + # bring back the (possibly) mutated graph/bindings but restore + # the outer scope stack / current layer + macro_ctx = MacroExpansionContext(inner_ctx.graph, inner_ctx.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + # walk any remaining children of the original call node + for i in 2:numchildren(ex) + macro_ctx = _visit_nested!(macro_ctx, ex[i], out) + end + return macro_ctx + end + + is_leaf(ex) && return macro_ctx + for c in children(ex) + macro_ctx = _visit_nested!(macro_ctx, c, out) + end + return macro_ctx +end + +src = """ +module Foo129 +foo() = 3 +h(f) = 4 + +global_xyz = 1 + +module Bar + using ..Foo129: foo, h, global_xyz + bar() = h(foo) + bar2(x, foo) = h(foo) + bar3() = global_xyz + 1 +end # Bar +end # Foo129 +""" + +eval(Meta.parse(src)) + +# 1. Parse the entire file content. +tree = parseall(SyntaxTree, src; filename="file.jl") + +scoped, ctx = lower_file(tree, Main) + + +global_bindings = filter(ctx.bindings.info) do binding + # want globals + keep = binding.kind == :global + + # internal ones seem non-interesting (`#self#` etc) + keep &= !binding.is_internal + + # I think we want ones that aren't assigned to? otherwise we are _defining_ the global here, not using it + keep &= binding.n_assigned == 0 + return keep +end From 23ad1b04a91f1c8ee2fc74e73700749a88eab08f Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Wed, 30 Jul 2025 23:22:25 +0200 Subject: [PATCH 31/35] wip --- Project.toml | 2 +- src/ExplicitImports.jl | 3 ++ src/lower.jl | 117 +++++++++++++++++++++++++++++++++++++++++ src/parse_utilities.jl | 19 ++++--- test/wip3.jl | 117 ++--------------------------------------- 5 files changed, 137 insertions(+), 121 deletions(-) create mode 100644 src/lower.jl diff --git a/Project.toml b/Project.toml index e1525640..a2e26f3b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ExplicitImports" uuid = "7d51a73a-1435-4ff3-83d9-f097790105c7" -authors = ["Eric P. Hanson"] version = "1.13.1" +authors = ["Eric P. Hanson"] [deps] Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 64468c32..61c19e34 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -1,5 +1,7 @@ module ExplicitImports +MUST_USE_JULIA_LOWERING::Bool = false + #! explicit-imports: off # We vendor some dependencies to avoid compatibility problems. We tell ExplicitImports to ignore # these as we don't want it to recurse into vendored dependencies. @@ -66,6 +68,7 @@ const STRICT_PRINTING_KWARG = """ const STRICT_NONRECURSIVE_KWARG = """ * `strict=true`: when `strict=true`, results will be `nothing` in the case that the analysis could not be performed accurately, due to e.g. dynamic `include` statements. When `strict=false`, results are returned in all cases, but may be inaccurate.""" +include("lower.jl") include("parse_utilities.jl") include("find_implicit_imports.jl") include("get_names_used.jl") diff --git a/src/lower.jl b/src/lower.jl new file mode 100644 index 00000000..ec064836 --- /dev/null +++ b/src/lower.jl @@ -0,0 +1,117 @@ +# NOTE: this file was written mostly by o3, needs checking + +using .JuliaSyntax: parseall, numchildren +using .JuliaLowering: SyntaxTree, MacroExpansionContext, DesugaringContext, + ensure_attributes, expand_forms_1, expand_forms_2, resolve_scopes, + reparent, ScopeLayer, + Bindings, IdTag, LayerId, LambdaBindings, eval_module, + syntax_graph, makenode, is_leaf, @K_str, + Bindings + +""" + lower_tree(tree, into_mod=Main) -> scoped_toplevel + +Statically lowers `tree` (the result of `parseall`) **and every nested module +body it contains**, returning a single `K"toplevel"` that carries full +`BindingId` information. +""" +function lower_tree(tree::SyntaxTree, into_mod::Module=Main) + # --- initial context -------------------------------------------------- + graph = ensure_attributes(syntax_graph(tree); + var_id=Int, + scope_layer=Int, + lambda_bindings=LambdaBindings, + bindings=Bindings) + + layers = [ScopeLayer(1, into_mod, false)] + bindings = Bindings() + # TODO: should the layer[1] be repeated like this? or should it be absent from layers? + ctx = MacroExpansionContext(graph, bindings, layers, layers[1]) + + resolved = SyntaxTree[] + last_ctx = _process_block!(ctx, tree, resolved) + + # stitch everything back together so callers get one tree + return makenode(last_ctx, tree, K"toplevel", resolved...), last_ctx +end + +# ------------------------------------------------------------------------- +# Internal helpers +# ------------------------------------------------------------------------- + +function _process_block!(macro_ctx, blk, out) + for stmt in js_children(blk) + macro_ctx = _process_stmt!(macro_ctx, stmt, out) # ← keep the new ctx + end + return macro_ctx +end + +function _process_stmt!(macro_ctx, stmt, out) + ctx1 = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + ex1 = expand_forms_1(ctx1, stmt) + ctx2 = DesugaringContext(ctx1) + ex2 = expand_forms_2(ctx2, ex1) + ctx3, ex3 = resolve_scopes(ctx2, ex2) + + push!(out, ex3) + + # Wrap the *richer* graph / bindings in a fresh context + next_ctx = MacroExpansionContext(ctx3.graph, ctx3.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + # Depth-first search for nested modules + return _visit_nested!(next_ctx, ex3, out) # returns an updated context +end + +function _visit_nested!(macro_ctx, ex, out) + k = kind(ex) + + if k == K"inert" + for c in js_children(ex) + macro_ctx = _visit_nested!(macro_ctx, c, out) + end + return macro_ctx + end + + if k == K"call" && + kind(ex[1]) == K"Value" && ex[1].value === eval_module && + kind(ex[4]) == K"inert" + + parent = (kind(ex[2]) == K"Value" && ex[2].value isa Module) ? + ex[2].value : Main + modname = Symbol(ex[3].value) + + childmod = Base.isdefined(parent, modname) ? + getfield(parent, modname) : + ( @warn "Module $modname not found in $(parent); using dummy." ; + Module(modname) ) + + new_layer = ScopeLayer(length(macro_ctx.scope_layers)+1, childmod, false) + scope_layers′ = [macro_ctx.scope_layers; new_layer] + + inner_ctx = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, + scope_layers′, new_layer) + + # recurse into the body of the nested module + inner_ctx = _process_block!(inner_ctx, js_children(ex[4])[1], out) + + # bring back the (possibly) mutated graph/bindings but restore + # the outer scope stack / current layer + macro_ctx = MacroExpansionContext(inner_ctx.graph, inner_ctx.bindings, + macro_ctx.scope_layers, macro_ctx.current_layer) + + # walk any remaining js_children of the original call node + for i in 2:numchildren(ex) + macro_ctx = _visit_nested!(macro_ctx, ex[i], out) + end + return macro_ctx + end + + is_leaf(ex) && return macro_ctx + for c in js_children(ex) + macro_ctx = _visit_nested!(macro_ctx, c, out) + end + return macro_ctx +end diff --git a/src/parse_utilities.jl b/src/parse_utilities.jl index 50ebfb5a..562f08cd 100644 --- a/src/parse_utilities.jl +++ b/src/parse_utilities.jl @@ -48,13 +48,20 @@ function SyntaxNodeWrapper(file::AbstractString, in_mod::Module; bad_locations=S contents = String(take!(stripped)) parsed = JuliaSyntax.parseall(JuliaLowering.SyntaxTree, contents; ignore_warnings=true) - # Perform lowering on the parse tree until scoping - ex = JuliaLowering.ensure_attributes(parsed; var_id=Int) - ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex) - ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) - ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) + scoped = parsed + ctx = nothing + try + scoped, ctx = lower_tree(parsed, in_mod) + catch e + @show in_mod + if MUST_USE_JULIA_LOWERING + rethrow() + else + @warn "JuliaLowering scope resolution error; may need newer Julia" exception=(e, catch_backtrace()) maxlog=1 + end + end - return SyntaxNodeWrapper(ex_scoped, file, bad_locations, ctx3, in_mod) + return SyntaxNodeWrapper(scoped, file, bad_locations, ctx, in_mod) end function try_parse_wrapper(file::AbstractString, mod::Module; bad_locations) diff --git a/test/wip3.jl b/test/wip3.jl index 27dd7091..a466e56f 100644 --- a/test/wip3.jl +++ b/test/wip3.jl @@ -1,120 +1,9 @@ using ExplicitImports +using ExplicitImports: lower_tree using ExplicitImports.Vendored.JuliaLowering using ExplicitImports.Vendored.JuliaSyntax using .JuliaSyntax: parseall, children, numchildren -using .JuliaLowering: SyntaxTree, MacroExpansionContext, DesugaringContext, - ensure_attributes, expand_forms_1, expand_forms_2, resolve_scopes, - reparent, ScopeLayer, - Bindings, IdTag, LayerId, LambdaBindings, eval_module, - syntax_graph, children, kind, makenode, is_leaf, @K_str, - Bindings - -""" - lower_file(tree, into_mod=Main) -> scoped_toplevel - -Statically lowers `tree` (the result of `parseall`) **and every nested module -body it contains**, returning a single `K"toplevel"` that carries full -`BindingId` information. -""" -function lower_file(tree::SyntaxTree, into_mod::Module=Main) - # --- initial context -------------------------------------------------- - graph = ensure_attributes(syntax_graph(tree); - var_id=Int, - scope_layer=Int, - lambda_bindings=LambdaBindings, - bindings=Bindings) - - layers = [ScopeLayer(1, into_mod, false)] - bindings = Bindings() - ctx = MacroExpansionContext(graph, bindings, layers, layers[1]) - - resolved = SyntaxTree[] - last_ctx = _process_block!(ctx, tree, resolved) - - # stitch everything back together so callers get one tree - return makenode(last_ctx, tree, K"toplevel", resolved...), last_ctx -end - -# ------------------------------------------------------------------------- -# Internal helpers -# ------------------------------------------------------------------------- - -function _process_block!(macro_ctx, blk, out) - for stmt in children(blk) - macro_ctx = _process_stmt!(macro_ctx, stmt, out) # ← keep the new ctx - end - return macro_ctx -end - -function _process_stmt!(macro_ctx, stmt, out) - ctx1 = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, - macro_ctx.scope_layers, macro_ctx.current_layer) - - ex1 = expand_forms_1(ctx1, stmt) - ctx2 = DesugaringContext(ctx1) - ex2 = expand_forms_2(ctx2, ex1) - ctx3, ex3 = resolve_scopes(ctx2, ex2) - - push!(out, ex3) - - # Wrap the *richer* graph / bindings in a fresh context - next_ctx = MacroExpansionContext(ctx3.graph, ctx3.bindings, - macro_ctx.scope_layers, macro_ctx.current_layer) - - # Depth-first search for nested modules - return _visit_nested!(next_ctx, ex3, out) # returns an updated context -end - -function _visit_nested!(macro_ctx, ex, out) - k = kind(ex) - - if k == K"inert" - for c in children(ex) - macro_ctx = _visit_nested!(macro_ctx, c, out) - end - return macro_ctx - end - - if k == K"call" && - kind(ex[1]) == K"Value" && ex[1].value === eval_module && - kind(ex[4]) == K"inert" - - parent = (kind(ex[2]) == K"Value" && ex[2].value isa Module) ? - ex[2].value : Main - modname = Symbol(ex[3].value) - - childmod = Base.isdefined(parent, modname) ? - getfield(parent, modname) : - ( @warn "Module $modname not found in $(parent); using dummy." ; - Module(modname) ) - - new_layer = ScopeLayer(length(macro_ctx.scope_layers)+1, childmod, false) - scope_layers′ = [macro_ctx.scope_layers; new_layer] - - inner_ctx = MacroExpansionContext(macro_ctx.graph, macro_ctx.bindings, - scope_layers′, new_layer) - - # recurse into the body of the nested module - inner_ctx = _process_block!(inner_ctx, children(ex[4])[1], out) - - # bring back the (possibly) mutated graph/bindings but restore - # the outer scope stack / current layer - macro_ctx = MacroExpansionContext(inner_ctx.graph, inner_ctx.bindings, - macro_ctx.scope_layers, macro_ctx.current_layer) - - # walk any remaining children of the original call node - for i in 2:numchildren(ex) - macro_ctx = _visit_nested!(macro_ctx, ex[i], out) - end - return macro_ctx - end - - is_leaf(ex) && return macro_ctx - for c in children(ex) - macro_ctx = _visit_nested!(macro_ctx, c, out) - end - return macro_ctx -end +using .JuliaLowering: SyntaxTree src = """ module Foo129 @@ -137,7 +26,7 @@ eval(Meta.parse(src)) # 1. Parse the entire file content. tree = parseall(SyntaxTree, src; filename="file.jl") -scoped, ctx = lower_file(tree, Main) +scoped, ctx = lower_tree(tree, Main) global_bindings = filter(ctx.bindings.info) do binding From 7e73b5ecc392110b0c740034d33d272897f5b7de Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Wed, 30 Jul 2025 23:55:00 +0200 Subject: [PATCH 32/35] wip --- src/ExplicitImports.jl | 2 +- .../JuliaLowering/src/JuliaLowering.jl | 4 +- src/vendored/JuliaLowering/src/bindings.jl | 38 +++++ .../JuliaLowering/src/closure_conversion.jl | 12 +- src/vendored/JuliaLowering/src/desugaring.jl | 83 ++++------ src/vendored/JuliaLowering/src/eval.jl | 19 ++- src/vendored/JuliaLowering/src/hooks.jl | 153 ------------------ src/vendored/JuliaLowering/src/kinds.jl | 4 +- src/vendored/JuliaLowering/src/linear_ir.jl | 9 +- src/vendored/JuliaLowering/src/precompile.jl | 27 ++++ .../JuliaLowering/src/scope_analysis.jl | 13 +- .../JuliaLowering/src/syntax_graph.jl | 4 +- .../JuliaLowering/src/syntax_macros.jl | 8 +- src/vendored/JuliaLowering/src/utils.jl | 26 +-- vendor/run.jl | 15 +- 15 files changed, 164 insertions(+), 253 deletions(-) delete mode 100644 src/vendored/JuliaLowering/src/hooks.jl create mode 100644 src/vendored/JuliaLowering/src/precompile.jl diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 61c19e34..4a174303 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -1,6 +1,6 @@ module ExplicitImports -MUST_USE_JULIA_LOWERING::Bool = false +MUST_USE_JULIA_LOWERING::Bool = true #! explicit-imports: off # We vendor some dependencies to avoid compatibility problems. We tell ExplicitImports to ignore diff --git a/src/vendored/JuliaLowering/src/JuliaLowering.jl b/src/vendored/JuliaLowering/src/JuliaLowering.jl index d02b3294..6f604d85 100644 --- a/src/vendored/JuliaLowering/src/JuliaLowering.jl +++ b/src/vendored/JuliaLowering/src/JuliaLowering.jl @@ -33,10 +33,10 @@ _include("syntax_macros.jl") _include("eval.jl") -_include("hooks.jl") - function __init__() _register_kinds() end +_include("precompile.jl") + end diff --git a/src/vendored/JuliaLowering/src/bindings.jl b/src/vendored/JuliaLowering/src/bindings.jl index e6fda3c2..f35a61c0 100644 --- a/src/vendored/JuliaLowering/src/bindings.jl +++ b/src/vendored/JuliaLowering/src/bindings.jl @@ -34,6 +34,44 @@ function BindingInfo(id::IdTag, name::AbstractString, kind::Symbol, node_id::Int is_internal, is_ambiguous_local, is_nospecialize) end +function Base.show(io::IO, binfo::BindingInfo) + print(io, "BindingInfo(", binfo.id, ", ", + repr(binfo.name), ", ", + repr(binfo.kind), ", ", + binfo.node_id) + if !isnothing(binfo.mod) + print(io, ", mod=", binfo.mod) + end + if !isnothing(binfo.type) + print(io, ", type=", binfo.type) + end + if binfo.n_assigned != 0 + print(io, ", n_assigned=", binfo.n_assigned) + end + if binfo.is_const + print(io, ", is_const=", binfo.is_const) + end + if binfo.is_ssa + print(io, ", is_ssa=", binfo.is_ssa) + end + if binfo.is_captured + print(io, ", is_captured=", binfo.is_captured) + end + if binfo.is_always_defined != binfo.is_ssa + print(io, ", is_always_defined=", binfo.is_always_defined) + end + if binfo.is_internal + print(io, ", is_internal=", binfo.is_internal) + end + if binfo.is_ambiguous_local + print(io, ", is_ambiguous_local=", binfo.is_ambiguous_local) + end + if binfo.is_nospecialize + print(io, ", is_nospecialize=", binfo.is_nospecialize) + end + print(io, ")") +end + """ Metadata about "entities" (variables, constants, etc) in the program. Each entity is associated to a unique integer id, the BindingId. A binding will be diff --git a/src/vendored/JuliaLowering/src/closure_conversion.jl b/src/vendored/JuliaLowering/src/closure_conversion.jl index cbcdf16b..79bde082 100644 --- a/src/vendored/JuliaLowering/src/closure_conversion.jl +++ b/src/vendored/JuliaLowering/src/closure_conversion.jl @@ -352,11 +352,13 @@ function _convert_closures(ctx::ClosureConversionCtx, ex) @assert kind(ex[1]) == K"BindingId" binfo = lookup_binding(ctx, ex[1]) if binfo.kind == :global - @ast ctx ex [ - K"globaldecl" - ex[1] - _convert_closures(ctx, ex[2]) - ] + @ast ctx ex [K"block" + # flisp has this, but our K"assert" handling is in a previous pass + # [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]] + [K"globaldecl" + ex[1] + _convert_closures(ctx, ex[2])] + "nothing"::K"core"] else makeleaf(ctx, ex, K"TOMBSTONE") end diff --git a/src/vendored/JuliaLowering/src/desugaring.jl b/src/vendored/JuliaLowering/src/desugaring.jl index 9d9a3c03..4702bb5e 100644 --- a/src/vendored/JuliaLowering/src/desugaring.jl +++ b/src/vendored/JuliaLowering/src/desugaring.jl @@ -814,8 +814,7 @@ function expand_generator(ctx, ex) outervars_by_key = Dict{NameKey,typeof(ex)}() for iterspecs in ex[2:end-1] for iterspec in children(iterspecs) - lhs = iterspec[1] - foreach_lhs_var(lhs) do var + for var in lhs_bound_names(iterspec[1]) @assert kind(var) == K"Identifier" # Todo: K"BindingId"? outervars_by_key[NameKey(var)] = var end @@ -1170,15 +1169,13 @@ function expand_unionall_def(ctx, srcref, lhs, rhs, is_const=true) throw(LoweringError(lhs, "empty type parameter list in type alias")) end name = lhs[1] - rr = ssavar(ctx, srcref) expand_forms_2( ctx, - @ast ctx srcref [ - K"block" - [K"=" rr [K"where" rhs lhs[2:end]...]] - [is_const ? K"constdecl" : K"assign_const_if_global" name rr] + @ast ctx srcref [K"block" + rr := [K"where" rhs lhs[2:end]...] + [is_const ? K"constdecl" : K"assign_or_constdecl_if_global" name rr] [K"latestworld_if_toplevel"] - rr + [K"removable" rr] ] ) end @@ -1229,10 +1226,9 @@ function expand_assignment(ctx, ex, is_const=false) ) elseif is_identifier_like(lhs) if is_const - rr = ssavar(ctx, rhs) @ast ctx ex [ K"block" - sink_assignment(ctx, ex, rr, expand_forms_2(ctx, rhs)) + rr := expand_forms_2(ctx, rhs) [K"constdecl" lhs rr] [K"latestworld"] [K"removable" rr] @@ -1467,7 +1463,7 @@ function expand_let(ctx, ex) ] elseif kind(lhs) == K"tuple" lhs_locals = SyntaxList(ctx) - foreach_lhs_var(lhs) do var + for var in lhs_bound_names(lhs) push!(lhs_locals, @ast ctx var [K"local" var]) push!(lhs_locals, @ast ctx var [K"always_defined" var]) end @@ -1904,23 +1900,6 @@ end #------------------------------------------------------------------------------- # Expand for loops -# Extract the variable names assigned to from a "fancy assignment left hand -# side" such as nested tuple destructuring. -function foreach_lhs_var(f::Function, ex) - k = kind(ex) - if k == K"Identifier" || k == K"BindingId" - f(ex) - elseif k == K"::" && numchildren(ex) == 2 - foreach_lhs_var(f, ex[1]) - elseif k == K"tuple" || k == K"parameters" - for e in children(ex) - foreach_lhs_var(f, e) - end - end - # k == K"Placeholder" ignored, along with everything else - we assume - # validation is done elsewhere. -end - function expand_for(ctx, ex) iterspecs = ex[1] @@ -1936,7 +1915,7 @@ function expand_for(ctx, ex) @chk kind(iterspec) == K"in" lhs = iterspec[1] if kind(lhs) != K"outer" - foreach_lhs_var(lhs) do var + for var in lhs_bound_names(lhs) push!(copied_vars, @ast ctx var [K"=" var var]) end end @@ -1953,7 +1932,7 @@ function expand_for(ctx, ex) if outer lhs = lhs[1] end - foreach_lhs_var(lhs) do var + for var in lhs_bound_names(lhs) if outer push!(lhs_outer_defs, @ast ctx var var) else @@ -2168,49 +2147,37 @@ function expand_decls(ctx, ex) makenode(ctx, ex, K"block", stmts) end -# Return all the names that will be bound by the assignment LHS, including -# curlies and calls. -function lhs_bound_names(ex) +# Extract the variable names assigned to from a "fancy assignment left hand +# side" such as nested tuple destructuring, curlies, and calls. +function lhs_bound_names(ex, out=SyntaxList(ex)) k = kind(ex) if k == K"Placeholder" - [] + # Ignored elseif is_identifier_like(ex) - [ex] - elseif k in KSet"call curly where ::" - lhs_bound_names(ex[1]) + push!(out, ex) + elseif (k === K"::" && numchildren(ex) === 2) || k in KSet"call curly where" + lhs_bound_names(ex[1], out) elseif k in KSet"tuple parameters" - vcat(map(lhs_bound_names, children(ex))...) - else - [] + map(c->lhs_bound_names(c, out), children(ex)) end + return out end function expand_const_decl(ctx, ex) - function check_assignment(asgn) - @chk (kind(asgn) == K"=") (ex, "expected assignment after `const`") - end - k = kind(ex[1]) - if numchildren(ex) == 2 - @ast ctx ex [ - K"constdecl" - ex[1] - expand_forms_2(ctx, ex[2]) - ] - elseif k == K"global" + if k == K"global" asgn = ex[1][1] - check_assignment(asgn) + @chk (kind(asgn) == K"=") (ex, "expected assignment after `const`") globals = map(lhs_bound_names(asgn[1])) do x @ast ctx ex [K"global" x] end - @ast ctx ex [ - K"block" + @ast ctx ex [K"block" globals... - expand_assignment(ctx, ex[1], true) + expand_assignment(ctx, asgn, true) ] elseif k == K"=" if numchildren(ex[1]) >= 1 && kind(ex[1][1]) == K"tuple" - throw(LoweringError(ex[1][1], "unsupported `const` tuple")) + TODO(ex[1][1], "`const` tuple assignment desugaring") end expand_assignment(ctx, ex[1], true) elseif k == K"local" @@ -2872,6 +2839,7 @@ function expand_function_def(ctx, ex, docs, rewrite_call=identity, rewrite_body= end return @ast ctx ex [K"block" [K"function_decl" name] + [K"latestworld"] name ] end @@ -4402,6 +4370,9 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) if numchildren(ex) == 1 && kind(ex[1]) == K"Identifier" # Don't recurse when already simplified - `local x`, etc ex + elseif kind(ex[1]) == K"const" + # Normalize global const to const global + expand_const_decl(ctx, @ast ctx ex [K"global" ex[1][1]]) else expand_forms_2(ctx, expand_decls(ctx, ex)) end diff --git a/src/vendored/JuliaLowering/src/eval.jl b/src/vendored/JuliaLowering/src/eval.jl index ccf2ac6a..df2da95f 100644 --- a/src/vendored/JuliaLowering/src/eval.jl +++ b/src/vendored/JuliaLowering/src/eval.jl @@ -12,6 +12,18 @@ function macroexpand(mod::Module, ex) ex1 end +function codeinfo_has_image_globalref(@nospecialize(e)) + if e isa GlobalRef + return 0x00 !== @ccall jl_object_in_image(e.mod::Any)::UInt8 + elseif e isa Core.CodeInfo + return any(codeinfo_has_image_globalref, e.code) + elseif e isa Expr + return any(codeinfo_has_image_globalref, e.args) + else + return false + end +end + _CodeInfo_need_ver = v"1.12.0-DEV.512" if VERSION < _CodeInfo_need_ver function _CodeInfo(args...) @@ -152,6 +164,8 @@ function to_code_info(ex, mod, funcname, slots) debuginfo = finish_ir_debug_info!(current_codelocs_stack) + has_image_globalref = any(codeinfo_has_image_globalref, stmts) + # TODO: Set ssaflags based on call site annotations: # - @inbounds annotations # - call site @inline / @noinline @@ -171,10 +185,6 @@ function to_code_info(ex, mod, funcname, slots) # TODO: Set based on Base.@assume_effects purity = 0x0000 - # TODO: Should we set these? - rettype = Any - has_image_globalref = false - # The following CodeInfo fields always get their default values for # uninferred code. ssavaluetypes = length(stmts) # Why does the runtime code do this? @@ -186,6 +196,7 @@ function to_code_info(ex, mod, funcname, slots) max_world = typemax(Csize_t) isva = false inlining_cost = 0xffff + rettype = Any _CodeInfo( stmts, diff --git a/src/vendored/JuliaLowering/src/hooks.jl b/src/vendored/JuliaLowering/src/hooks.jl deleted file mode 100644 index 4b3748e7..00000000 --- a/src/vendored/JuliaLowering/src/hooks.jl +++ /dev/null @@ -1,153 +0,0 @@ -using ..JuliaSyntax - - -# Becomes `Core._lower()` upon activating JuliaLowering. Returns an svec with -# the lowered code (usually expr) as its first element, and whatever we want -# after it, until the API stabilizes -function core_lowerer_hook(code, mod::Module, file="none", line=0, world=typemax(Csize_t), warn=false) - if Base.isexpr(code, :syntaxtree) - # Getting toplevel.c to check for types it doesn't know about is hard. - # We wrap SyntaxTrees with this random expr head so that the call to - # `jl_needs_lowering` in `jl_toplevel_eval_flex` returns true; this way - # the SyntaxTree is handed back to us, unwraped here, and lowered. - code = code.args[1] - end - if code isa Expr - @warn("""JuliaLowering received an Expr instead of a SyntaxTree. - This is currently expected when evaluating modules. - Falling back to flisp...""", - code=code, file=file, line=line, mod=mod) - return Base.fl_lower(code, mod, file, line, world, warn) - elseif !(code isa SyntaxTree) - # LineNumberNode, Symbol, probably others... - return Core.svec(code) - end - try - ctx1, st1 = expand_forms_1( mod, code) - ctx2, st2 = expand_forms_2( ctx1, st1) - ctx3, st3 = resolve_scopes( ctx2, st2) - ctx4, st4 = convert_closures(ctx3, st3) - ctx5, st5 = linearize_ir( ctx4, st4) - ex = to_lowered_expr(mod, st5) - return Core.svec(ex, st5, ctx5) - catch exc - @error("JuliaLowering failed — falling back to flisp!", - exception=(exc,catch_backtrace()), - code=code, file=file, line=line, mod=mod) - return Base.fl_lower(st0, mod, file, line, world, warn) - end -end - -# TODO: This is code copied from JuliaSyntax, adapted to produce -# `Expr(:syntaxtree, st::SyntaxTree)`. -function core_parse_for_lowering_hook(code, filename::String, lineno::Int, offset::Int, options::Symbol) - if Core._lower != core_lowerer_hook - # If lowering can't handle SyntaxTree, return Expr. - # (assumes no Core._lower function other than our core_lowerer_hook can handle SyntaxTree) - return JuliaSyntax.core_parser_hook(code, filename, lineno, offset, options) - end - try - # TODO: Check that we do all this input wrangling without copying the - # code buffer - if code isa Core.SimpleVector - # The C entry points will pass us this form. - (ptr,len) = code - code = String(unsafe_wrap(Array, ptr, len)) - elseif !(code isa String || code isa SubString || code isa Vector{UInt8}) - # For non-Base string types, convert to UTF-8 encoding, using an - # invokelatest to avoid world age issues. - code = Base.invokelatest(String, code) - end - stream = JuliaSyntax.ParseStream(code, offset+1) - if options === :statement || options === :atom - # To copy the flisp parser driver: - # * Parsing atoms consumes leading trivia - # * Parsing statements consumes leading+trailing trivia - JuliaSyntax.bump_trivia(stream) - if peek(stream) == K"EndMarker" - # If we're at the end of stream after skipping whitespace, just - # return `nothing` to indicate this rather than attempting to - # parse a statement or atom and failing. - return Core.svec(nothing, last_byte(stream)) - end - end - JuliaSyntax.parse!(stream; rule=options) - if options === :statement - JuliaSyntax.bump_trivia(stream; skip_newlines=false) - if peek(stream) == K"NewlineWs" - JuliaSyntax.bump(stream) - end - end - - if JuliaSyntax.any_error(stream) - pos_before_comments = JuliaSyntax.last_non_whitespace_byte(stream) - tree = JuliaSyntax.build_tree(SyntaxNode, stream, first_line=lineno, filename=filename) - tag = JuliaSyntax._incomplete_tag(tree, pos_before_comments) - exc = JuliaSyntax.ParseError(stream, filename=filename, first_line=lineno, - incomplete_tag=tag) - msg = sprint(showerror, exc) - error_ex = Expr(tag === :none ? :error : :incomplete, - Meta.ParseError(msg, exc)) - ex = if options === :all - # When encountering a toplevel error, the reference parser - # * truncates the top level expression arg list before that error - # * includes the last line number - # * appends the error message - topex = Expr(tree) - @assert topex.head == :toplevel - i = findfirst(JuliaSyntax._has_nested_error, topex.args) - if i > 1 && topex.args[i-1] isa LineNumberNode - i -= 1 - end - resize!(topex.args, i-1) - _,errort = JuliaSyntax._first_error(tree) - push!(topex.args, LineNumberNode(JuliaSyntax.source_line(errort), filename)) - push!(topex.args, error_ex) - topex - else - error_ex - end - else - # See unwrapping of `:syntaxtree` above. - ex = Expr(:syntaxtree, JuliaSyntax.build_tree(SyntaxTree, stream; filename=filename, first_line=lineno)) - end - - # Note the next byte in 1-based indexing is `last_byte(stream) + 1` but - # the Core hook must return an offset (ie, it's 0-based) so the factors - # of one cancel here. - last_offset = last_byte(stream) - - # Rewrap result in an svec for use by the C code - return Core.svec(ex, last_offset) - catch exc - @error("""JuliaSyntax parser failed — falling back to flisp! - This is not your fault. Please submit a bug report to https://github.com/JuliaLang/JuliaSyntax.jl/issues""", - exception=(exc,catch_backtrace()), - offset=offset, - code=code) - - Base.fl_parse(code, filename, lineno, offset, options) - end -end - -const _has_v1_13_hooks = isdefined(Core, :_lower) - -function activate!(enable=true) - if !_has_v1_13_hooks - error("Cannot use JuliaLowering without `Core._lower` binding or in $VERSION < 1.13") - end - - if enable - if !isnothing(Base.active_repl_backend) - # TODO: These act on parsed exprs, which we don't have. - # Reimplementation needed (e.g. for scoping rules). - empty!(Base.active_repl_backend.ast_transforms) - end - - Core._setlowerer!(core_lowerer_hook) - Core._setparser!(core_parse_for_lowering_hook) - else - Core._setlowerer!(Base.fl_lower) - Core._setparser!(JuliaSyntax.core_parse_hook) - end -end diff --git a/src/vendored/JuliaLowering/src/kinds.jl b/src/vendored/JuliaLowering/src/kinds.jl index 09f33487..741307ba 100644 --- a/src/vendored/JuliaLowering/src/kinds.jl +++ b/src/vendored/JuliaLowering/src/kinds.jl @@ -19,6 +19,8 @@ function _register_kinds() "Value" # A (quoted) `Symbol` "Symbol" + # QuoteNode; not quasiquote + "inert" # Compiler metadata hints "meta" # TODO: Use `meta` for inbounds and loopinfo etc? @@ -94,7 +96,7 @@ function _register_kinds() "_opaque_closure" # The enclosed statements must be executed at top level "toplevel_butfirst" - "assign_const_if_global" + "assign_or_constdecl_if_global" "moved_local" "label" "trycatchelse" diff --git a/src/vendored/JuliaLowering/src/linear_ir.jl b/src/vendored/JuliaLowering/src/linear_ir.jl index ad06172f..d356d2f7 100644 --- a/src/vendored/JuliaLowering/src/linear_ir.jl +++ b/src/vendored/JuliaLowering/src/linear_ir.jl @@ -60,7 +60,7 @@ end Context for creating linear IR. One of these is created per lambda expression to flatten the body down to -a sequence of statements (linear IR). +a sequence of statements (linear IR), which eventually becomes one CodeInfo. """ struct LinearIRContext{GraphType} <: AbstractLoweringContext graph::GraphType @@ -332,7 +332,7 @@ function emit_assignment_or_setglobal(ctx, srcref, lhs, rhs, op=K"=") if binfo.kind == :global && op == K"=" emit(ctx, @ast ctx srcref [ K"call" - "setglobal!"::K"top" + "setglobal!"::K"core" binfo.mod::K"Value" binfo.name::K"Symbol" rhs @@ -884,13 +884,12 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos) if numchildren(ex) == 1 || is_identifier_like(ex[2]) emit(ctx, ex) else - rr = ssavar(ctx, ex[2]) - emit(ctx, @ast ctx ex [K"=" rr ex[2]]) + rr = emit_assign_tmp(ctx, ex[2]) emit(ctx, @ast ctx ex [K"globaldecl" ex[1] rr]) end ctx.is_toplevel_thunk && emit(ctx, makenode(ctx, ex, K"latestworld")) elseif k == K"latestworld" - emit(ctx, makeleaf(ctx, ex, K"latestworld")) + emit(ctx, ex) elseif k == K"latestworld_if_toplevel" ctx.is_toplevel_thunk && emit(ctx, makeleaf(ctx, ex, K"latestworld")) else diff --git a/src/vendored/JuliaLowering/src/precompile.jl b/src/vendored/JuliaLowering/src/precompile.jl new file mode 100644 index 00000000..0c07b546 --- /dev/null +++ b/src/vendored/JuliaLowering/src/precompile.jl @@ -0,0 +1,27 @@ +# exercise the whole lowering pipeline +if Base.get_bool_env("JULIA_LOWERING_PRECOMPILE", true) + thunks = String[ + """ + function foo(xxx, yyy) + @nospecialize xxx + return Pair{Any,Any}(typeof(xxx), typeof(yyy)) + end + """ + + """ + struct Foo + x::Int + Foo(x::Int) = new(x) + # Foo() = new() + end + """ + ] + for thunk in thunks + stream = JuliaSyntax.ParseStream(thunk) + JuliaSyntax.parse!(stream; rule=:all) + st0 = JuliaSyntax.build_tree(SyntaxTree, stream; filename=@__FILE__) + lwrst = lower(@__MODULE__, st0[1]) + lwr = to_lowered_expr(@__MODULE__, lwrst) + @assert Meta.isexpr(lwr, :thunk) && only(lwr.args) isa Core.CodeInfo + end +end diff --git a/src/vendored/JuliaLowering/src/scope_analysis.jl b/src/vendored/JuliaLowering/src/scope_analysis.jl index 59314149..38e8e23a 100644 --- a/src/vendored/JuliaLowering/src/scope_analysis.jl +++ b/src/vendored/JuliaLowering/src/scope_analysis.jl @@ -44,7 +44,7 @@ function _find_scope_vars!(ctx, assignments, locals, destructured_args, globals, end elseif k == K"global" _insert_if_not_present!(globals, NameKey(ex[1]), ex) - elseif k == K"assign_const_if_global" + elseif k == K"assign_or_constdecl_if_global" # like v = val, except that if `v` turns out global(either implicitly or # by explicit `global`), it gains an implicit `const` _insert_if_not_present!(assignments, NameKey(ex[1]), ex) @@ -565,15 +565,14 @@ function _resolve_scopes(ctx, ex::SyntaxTree) end end resolved - elseif k == K"assign_const_if_global" + elseif k == K"assign_or_constdecl_if_global" id = _resolve_scopes(ctx, ex[1]) bk = lookup_binding(ctx, id).kind - if bk == :local && numchildren(ex) != 1 - @ast ctx ex _resolve_scopes(ctx, [K"=" children(ex)...]) - elseif bk != :local # TODO: should this be == :global? - @ast ctx ex _resolve_scopes(ctx, [K"constdecl" children(ex)...]) + @assert numchildren(ex) === 2 + if bk == :global + @ast ctx ex _resolve_scopes(ctx, [K"constdecl" ex[1] ex[2]]) else - makeleaf(ctx, ex, K"TOMBSTONE") + @ast ctx ex _resolve_scopes(ctx, [K"=" ex[1] ex[2]]) end else mapchildren(e->_resolve_scopes(ctx, e), ctx, ex) diff --git a/src/vendored/JuliaLowering/src/syntax_graph.jl b/src/vendored/JuliaLowering/src/syntax_graph.jl index 82c91302..28af9cfb 100644 --- a/src/vendored/JuliaLowering/src/syntax_graph.jl +++ b/src/vendored/JuliaLowering/src/syntax_graph.jl @@ -6,7 +6,7 @@ one or several syntax trees. TODO: Global attributes! """ -struct SyntaxGraph{Attrs} +mutable struct SyntaxGraph{Attrs} edge_ranges::Vector{UnitRange{Int}} edges::Vector{NodeId} attributes::Attrs @@ -633,7 +633,7 @@ end #------------------------------------------------------------------------------- # Lightweight vector of nodes ids with associated pointer to graph stored separately. -struct SyntaxList{GraphType, NodeIdVecType} <: AbstractVector{SyntaxTree} +mutable struct SyntaxList{GraphType, NodeIdVecType} <: AbstractVector{SyntaxTree} graph::GraphType ids::NodeIdVecType end diff --git a/src/vendored/JuliaLowering/src/syntax_macros.jl b/src/vendored/JuliaLowering/src/syntax_macros.jl index 5a18059d..d9fac7be 100644 --- a/src/vendored/JuliaLowering/src/syntax_macros.jl +++ b/src/vendored/JuliaLowering/src/syntax_macros.jl @@ -2,14 +2,14 @@ # extensions": # # * They emit syntactic forms with special `Kind`s and semantics known to -# lowering +# lowering # * There is no other Julia surface syntax for these `Kind`s. # In order to implement these here without getting into bootstrapping problems, # we just write them as plain old macro-named functions and add the required # __context__ argument ourselves. # -# TODO: @inline, @noinline, @inbounds, @simd, @ccall, @isdefined, @assume_effects +# TODO: @inline, @noinline, @inbounds, @simd, @ccall, @assume_effects # # TODO: Eventually move these to proper `macro` definitions and use # `JuliaLowering.include()` or something. Then we'll be in the fun little world @@ -29,7 +29,8 @@ function _apply_nospecialize(ctx, ex) end end -function Base.var"@nospecialize"(__context__::MacroContext, ex) +function Base.var"@nospecialize"(__context__::MacroContext, ex, exs...) + # TODO support multi-arg version properly _apply_nospecialize(__context__, ex) end @@ -220,4 +221,3 @@ function var"@inert"(__context__::MacroContext, ex) @chk kind(ex) == K"quote" @ast __context__ __context__.macrocall [K"inert" ex] end - diff --git a/src/vendored/JuliaLowering/src/utils.jl b/src/vendored/JuliaLowering/src/utils.jl index ced8827e..b6f9e834 100644 --- a/src/vendored/JuliaLowering/src/utils.jl +++ b/src/vendored/JuliaLowering/src/utils.jl @@ -39,29 +39,34 @@ function _show_provtree(io::IO, prov, indent) printstyled(io, "@ $fn:$line\n", color=:light_black) end -function showprov(io::IO, exs::AbstractVector) +function showprov(io::IO, exs::AbstractVector; + note=nothing, include_location::Bool=true, highlight_kwargs...) for (i,ex) in enumerate(Iterators.reverse(exs)) sr = sourceref(ex) if i > 1 print(io, "\n\n") end k = kind(ex) - note = i > 1 && k == K"macrocall" ? "in macro expansion" : - i > 1 && k == K"$" ? "interpolated here" : - "in source" - highlight(io, sr, note=note) + if isnothing(note) + fallback_note = i > 1 && k == K"macrocall" ? "in macro expansion" : + i > 1 && k == K"$" ? "interpolated here" : + "in source" + end + highlight(io, sr; note=something(note, fallback_note), highlight_kwargs...) - line, _ = source_location(sr) - locstr = "$(filename(sr)):$line" - JuliaSyntax._printstyled(io, "\n# @ $locstr", fgcolor=:light_black) + if include_location + line, _ = source_location(sr) + locstr = "$(filename(sr)):$line" + JuliaSyntax._printstyled(io, "\n# @ $locstr", fgcolor=:light_black) + end end end -function showprov(io::IO, ex::SyntaxTree; tree=false) +function showprov(io::IO, ex::SyntaxTree; tree::Bool=false, showprov_kwargs...) if tree _show_provtree(io, ex, "") else - showprov(io, flattened_provenance(ex)) + showprov(io, flattened_provenance(ex); showprov_kwargs...) end end @@ -165,4 +170,3 @@ function _print_ir(io::IO, ex, indent) end end end - diff --git a/vendor/run.jl b/vendor/run.jl index 6631a360..d2543645 100644 --- a/vendor/run.jl +++ b/vendor/run.jl @@ -1,12 +1,23 @@ using PackageAnalyzer, UUIDs -deps = [find_package("JuliaSyntax"; version=v"1.0.2"), +function get_tree_hash(repo="JuliaLang/JuliaSyntax.jl", rev="46723f0") + return readchomp(`gh api "repos/$repo/commits/$rev" --jq '.commit.tree.sha'`) +end + +deps = [PackageAnalyzer.Added(; name="JuliaSyntax", + uuid=UUID("70703baa-626e-46a2-a12c-08ffd08c73b4"), + path="", + repo_url="https://github.com/JuliaLang/JuliaSyntax.jl", + # get_tree_hash("JuliaLang/JuliaSyntax.jl", "46723f0") + tree_hash="0d4b3dab95018bcf3925204475693d9f09dc45b8", + subdir=""), find_package("AbstractTrees"; version=v"0.4.5"), PackageAnalyzer.Added(; name="JuliaLowering", uuid=UUID("f3c80556-a63f-4383-b822-37d64f81a311"), path="", repo_url="https://github.com/mlechu/JuliaLowering.jl", - tree_hash="2d3dfe83e9be4318c056ed9df2d3788f5723bb9d", + # get_tree_hash("mlechu/JuliaLowering.jl", "fix-nightly") + tree_hash="24fb8b102582c4ecefc43a4fd61a9b4e578ca04d", subdir="")] for pkg in deps From 16e9fd7c487255fe29a0f88a34f77d2a3d60f5a2 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 31 Jul 2025 18:45:34 +0200 Subject: [PATCH 33/35] wip --- src/ExplicitImports.jl | 5 +++-- src/lower.jl | 7 ++++--- .../JuliaLowering/src/macro_expansion.jl | 15 +++++++++++---- test/wip4.jl | 17 +++++++++++++++++ vendor/run.jl | 5 ++--- 5 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 test/wip4.jl diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 4a174303..8c2370a7 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -1,4 +1,6 @@ module ExplicitImports +const HAS_PUBLIC2 = Base.eval(quote Base.VERSION >= v"1.11.0-DEV.469" end) + MUST_USE_JULIA_LOWERING::Bool = true @@ -31,8 +33,7 @@ using Pkg: Pkg # debug parsefile -# we'll borrow their `@_public` macro; if this goes away, we can get our own -JuliaSyntax.@_public ignore_submodules +public ignore_submodules export print_explicit_imports, explicit_imports, check_no_implicit_imports, explicit_imports_nonrecursive diff --git a/src/lower.jl b/src/lower.jl index ec064836..dda2601a 100644 --- a/src/lower.jl +++ b/src/lower.jl @@ -4,9 +4,9 @@ using .JuliaSyntax: parseall, numchildren using .JuliaLowering: SyntaxTree, MacroExpansionContext, DesugaringContext, ensure_attributes, expand_forms_1, expand_forms_2, resolve_scopes, reparent, ScopeLayer, - Bindings, IdTag, LayerId, LambdaBindings, eval_module, + IdTag, LayerId, LambdaBindings, eval_module, syntax_graph, makenode, is_leaf, @K_str, - Bindings + Bindings, CompileHints """ lower_tree(tree, into_mod=Main) -> scoped_toplevel @@ -21,7 +21,8 @@ function lower_tree(tree::SyntaxTree, into_mod::Module=Main) var_id=Int, scope_layer=Int, lambda_bindings=LambdaBindings, - bindings=Bindings) + bindings=Bindings, + meta=CompileHints) layers = [ScopeLayer(1, into_mod, false)] bindings = Bindings() diff --git a/src/vendored/JuliaLowering/src/macro_expansion.jl b/src/vendored/JuliaLowering/src/macro_expansion.jl index 1e4ac756..8feb07c0 100644 --- a/src/vendored/JuliaLowering/src/macro_expansion.jl +++ b/src/vendored/JuliaLowering/src/macro_expansion.jl @@ -145,8 +145,16 @@ function expand_macro(ctx, ex) end macro_invocation_world = Base.get_world_counter() expanded = try - # TODO: Allow invoking old-style macros for compat - invokelatest(macfunc, macro_args...) + if applicable(macfunc, macro_args...) + invokelatest(macfunc, macro_args...) + else + # try old-style macro + args = [Expr(x) for x in macro_args[2:end]] + line, _ = source_location(macname) + file = filename(macname) + line_number_node = Base.LineNumberNode(line, file) + invokelatest(macfunc, line_number_node, ctx.current_layer.mod, args...) + end catch exc if exc isa MacroExpansionError # Add context to the error. @@ -237,7 +245,7 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) @chk numchildren(ex) == 1 # TODO: Upstream should set a general flag for detecting parenthesized # expressions so we don't need to dig into `green_tree` here. Ugh! - plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) && + plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) && kind(ex[1]) == K"Identifier" && (sr = sourceref(ex); sr isa SourceRef && kind(sr.green_tree[2]) != K"parens") if plain_symbol @@ -337,4 +345,3 @@ function expand_forms_1(mod::Module, ex::SyntaxTree) ctx.current_layer) return ctx2, reparent(ctx2, ex2) end - diff --git a/test/wip4.jl b/test/wip4.jl new file mode 100644 index 00000000..ffbe6f38 --- /dev/null +++ b/test/wip4.jl @@ -0,0 +1,17 @@ +using ExplicitImports.Vendored.JuliaLowering, ExplicitImports.Vendored.JuliaSyntax + +using .JuliaLowering: JuliaLowering, SyntaxTree +using .JuliaSyntax: parsestmt + + +src = """ +function Base.:(==)(a::SyntaxNodeList, b::SyntaxNodeList) + return map(objectid, a.nodes) == map(objectid, b.nodes) +end +""" + +tree = parsestmt(SyntaxTree, src; filename="file.jl") +ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(Main, tree); +ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand); +ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar); +ex_scoped diff --git a/vendor/run.jl b/vendor/run.jl index d2543645..a0559fb8 100644 --- a/vendor/run.jl +++ b/vendor/run.jl @@ -15,9 +15,8 @@ deps = [PackageAnalyzer.Added(; name="JuliaSyntax", PackageAnalyzer.Added(; name="JuliaLowering", uuid=UUID("f3c80556-a63f-4383-b822-37d64f81a311"), path="", - repo_url="https://github.com/mlechu/JuliaLowering.jl", - # get_tree_hash("mlechu/JuliaLowering.jl", "fix-nightly") - tree_hash="24fb8b102582c4ecefc43a4fd61a9b4e578ca04d", + repo_url="https://github.com/ericphanson/JuliaLowering.jl", + tree_hash = get_tree_hash("mlechu/JuliaLowering.jl", "eph/trunk"), subdir="")] for pkg in deps From fa1f7d6461dcaea3fb00029471fdfb59e6aa82c3 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Thu, 31 Jul 2025 18:46:03 +0200 Subject: [PATCH 34/35] update --- src/vendored/JuliaLowering/src/desugaring.jl | 4 +- src/vendored/JuliaLowering/src/kinds.jl | 3 + .../JuliaLowering/src/macro_expansion.jl | 161 ++++++++++++------ .../JuliaLowering/src/syntax_graph.jl | 9 +- vendor/run.jl | 2 +- 5 files changed, 125 insertions(+), 54 deletions(-) diff --git a/src/vendored/JuliaLowering/src/desugaring.jl b/src/vendored/JuliaLowering/src/desugaring.jl index 4702bb5e..8251d2f7 100644 --- a/src/vendored/JuliaLowering/src/desugaring.jl +++ b/src/vendored/JuliaLowering/src/desugaring.jl @@ -15,7 +15,7 @@ function DesugaringContext(ctx) scope_type=Symbol, # :hard or :soft var_id=IdTag, is_toplevel_thunk=Bool) - DesugaringContext(graph, ctx.bindings, ctx.scope_layers, ctx.current_layer.mod) + DesugaringContext(graph, ctx.bindings, ctx.scope_layers, first(ctx.scope_layers).mod) end #------------------------------------------------------------------------------- @@ -3822,7 +3822,7 @@ function insert_struct_shim(ctx, fieldtypes, name) ex[2].name_val == name.name_val @ast ctx ex [K"call" "struct_name_shim"::K"core" ex[1] ex[2] ctx.mod::K"Value" name] elseif numchildren(ex) > 0 - @ast ctx ex [ex.kind map(replace_type, children(ex))...] + mapchildren(replace_type, ctx, ex) else ex end diff --git a/src/vendored/JuliaLowering/src/kinds.jl b/src/vendored/JuliaLowering/src/kinds.jl index 741307ba..6ae84c83 100644 --- a/src/vendored/JuliaLowering/src/kinds.jl +++ b/src/vendored/JuliaLowering/src/kinds.jl @@ -7,6 +7,9 @@ function _register_kinds() # expansion, and known to lowering. These are part of the AST API but # without having surface syntax. "BEGIN_EXTENSION_KINDS" + # Used for converting `esc()`'d expressions arising from old macro + # invocations during macro expansion + "escape" # atomic fields or accesses (see `@atomic`) "atomic" # Flag for @generated parts of a functon diff --git a/src/vendored/JuliaLowering/src/macro_expansion.jl b/src/vendored/JuliaLowering/src/macro_expansion.jl index 8feb07c0..59ec6c64 100644 --- a/src/vendored/JuliaLowering/src/macro_expansion.jl +++ b/src/vendored/JuliaLowering/src/macro_expansion.jl @@ -18,9 +18,12 @@ struct MacroExpansionContext{GraphType} <: AbstractLoweringContext graph::GraphType bindings::Bindings scope_layers::Vector{ScopeLayer} - current_layer::ScopeLayer + scope_layer_stack::Vector{LayerId} end +current_layer(ctx::MacroExpansionContext) = ctx.scope_layers[last(ctx.scope_layer_stack)] +current_layer_id(ctx::MacroExpansionContext) = last(ctx.scope_layer_stack) + #-------------------------------------------------- # Expansion of quoted expressions function collect_unquoted!(ctx, unquoted, ex, depth) @@ -122,66 +125,119 @@ function eval_macro_name(ctx, ex) ctx3, ex3 = resolve_scopes(ctx2, ex2) ctx4, ex4 = convert_closures(ctx3, ex3) ctx5, ex5 = linearize_ir(ctx4, ex4) - mod = ctx.current_layer.mod + mod = current_layer(ctx).mod expr_form = to_lowered_expr(mod, ex5) eval(mod, expr_form) end +# Record scope layer information for symbols passed to a macro by setting +# scope_layer for each expression and also processing any K"escape" arising +# from previous expansion of old-style macros. +# +# See also set_scope_layer() +function set_macro_arg_hygiene(ctx, ex, layer_ids, layer_idx) + k = kind(ex) + scope_layer = get(ex, :scope_layer, layer_ids[layer_idx]) + if k == K"module" || k == K"toplevel" || k == K"inert" + makenode(ctx, ex, ex, children(ex); + scope_layer=scope_layer) + elseif k == K"." + makenode(ctx, ex, ex, set_macro_arg_hygiene(ctx, ex[1], layer_ids, layer_idx), ex[2], + scope_layer=scope_layer) + elseif !is_leaf(ex) + inner_layer_idx = layer_idx + if k == K"escape" + inner_layer_idx = layer_idx - 1 + if inner_layer_idx < 1 + # If we encounter too many escape nodes, there's probably been + # an error in the previous macro expansion. + # todo: The error here isn't precise about that - maybe we + # should record that macro call expression with the scope layer + # if we want to report the error against the macro call? + throw(MacroExpansionError(ex, "`escape` node in outer context")) + end + end + mapchildren(e->set_macro_arg_hygiene(ctx, e, layer_ids, inner_layer_idx), + ctx, ex; scope_layer=scope_layer) + else + makeleaf(ctx, ex, ex; scope_layer=scope_layer) + end +end + function expand_macro(ctx, ex) @assert kind(ex) == K"macrocall" macname = ex[1] macfunc = eval_macro_name(ctx, macname) - # Macro call arguments may be either - # * Unprocessed by the macro expansion pass - # * Previously processed, but spliced into a further macro call emitted by - # a macro expansion. - # In either case, we need to set any unset scope layers before passing the - # arguments to the macro call. - mctx = MacroContext(ctx.graph, ex, ctx.current_layer) - macro_args = Any[mctx] - for i in 2:numchildren(ex) - push!(macro_args, set_scope_layer(ctx, ex[i], ctx.current_layer.id, false)) - end + mctx = MacroContext(ctx.graph, ex, current_layer(ctx)) + raw_args = ex[2:end] macro_invocation_world = Base.get_world_counter() - expanded = try - if applicable(macfunc, macro_args...) + if hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=Base.get_world_counter()) + macro_args = Any[mctx] + for arg in raw_args + # Add hygiene information to be carried along with macro arguments. + # + # Macro call arguments may be either + # * Unprocessed by the macro expansion pass + # * Previously processed, but spliced into a further macro call emitted by + # a macro expansion. + # In either case, we need to set scope layers before passing the + # arguments to the macro call. + push!(macro_args, set_macro_arg_hygiene(ctx, arg, ctx.scope_layer_stack, + length(ctx.scope_layer_stack))) + end + expanded = try invokelatest(macfunc, macro_args...) - else - # try old-style macro - args = [Expr(x) for x in macro_args[2:end]] - line, _ = source_location(macname) - file = filename(macname) - line_number_node = Base.LineNumberNode(line, file) - invokelatest(macfunc, line_number_node, ctx.current_layer.mod, args...) + catch exc + if exc isa MacroExpansionError + # Add context to the error. + # TODO: Using rethrow() is kinda ugh. Is there a way to avoid it? + rethrow(MacroExpansionError(mctx, exc.ex, exc.msg, exc.position)) + else + throw(MacroExpansionError(mctx, ex, "Error expanding macro", :all)) + end end - catch exc - if exc isa MacroExpansionError - # Add context to the error. - # TODO: Using rethrow() is kinda ugh. Is there a way to avoid it? - rethrow(MacroExpansionError(mctx, exc.ex, exc.msg, exc.position)) + if expanded isa SyntaxTree + if !is_compatible_graph(ctx, expanded) + # If the macro has produced syntax outside the macro context, + # copy it over. TODO: Do we expect this always to happen? What + # is the API for access to the macro expansion context? + expanded = copy_ast(ctx, expanded) + end else - throw(MacroExpansionError(mctx, ex, "Error expanding macro", :all)) + expanded = @ast ctx ex expanded::K"Value" end + else + # Compat: attempt to invoke an old-style macro if there's no applicable + # method for new-style macro arguments. + macro_args = Any[source_location(LineNumberNode, ex), current_layer(ctx).mod] + for arg in raw_args + # For hygiene in old-style macros, we omit any additional scope + # layer information from macro arguments. Old-style macros will + # handle that using manual escaping in the macro itself. + # + # Note that there's one somewhat-incompatibility here for + # identifiers interpolated into the `raw_args` from outer macro + # expansions of new-style macros which call old-style macros. + # Instead of seeing `Expr(:escape)` in such situations, old-style + # macros will now see `Expr(:scope_layer)` inside `macro_args`. + push!(macro_args, Expr(arg)) + end + expanded = invokelatest(macfunc, macro_args...) + expanded = expr_to_SyntaxTree(syntax_graph(ctx), expanded) end - if expanded isa SyntaxTree - if !is_compatible_graph(ctx, expanded) - # If the macro has produced syntax outside the macro context, copy it over. - # TODO: Do we expect this always to happen? What is the API for access - # to the macro expansion context? - expanded = copy_ast(ctx, expanded) - end + if kind(expanded) != K"Value" expanded = append_sourceref(ctx, expanded, ex) # Module scope for the returned AST is the module where this particular # method was defined (may be different from `parentmodule(macfunc)`) - mod_for_ast = lookup_method_instance(macfunc, macro_args, macro_invocation_world).def.module + mod_for_ast = lookup_method_instance(macfunc, macro_args, + macro_invocation_world).def.module new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod_for_ast, true) push!(ctx.scope_layers, new_layer) - inner_ctx = MacroExpansionContext(ctx.graph, ctx.bindings, ctx.scope_layers, new_layer) - expanded = expand_forms_1(inner_ctx, expanded) - else - expanded = @ast ctx ex expanded::K"Value" + push!(ctx.scope_layer_stack, new_layer.id) + expanded = expand_forms_1(ctx, expanded) + pop!(ctx.scope_layer_stack) end return expanded end @@ -223,18 +279,24 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) elseif is_ccall_or_cglobal(name_str) @ast ctx ex name_str::K"core" else - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) makeleaf(ctx, ex, ex, kind=K"Identifier", scope_layer=layerid) end elseif k == K"Identifier" || k == K"MacroName" || k == K"StringMacroName" - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) makeleaf(ctx, ex, ex, kind=K"Identifier", scope_layer=layerid) elseif k == K"var" || k == K"char" || k == K"parens" # Strip "container" nodes @chk numchildren(ex) == 1 expand_forms_1(ctx, ex[1]) + elseif k == K"escape" + # For processing of old-style macros + top_layer = pop!(ctx.scope_layer_stack) + escaped_ex = expand_forms_1(ctx, ex[1]) + push!(ctx.scope_layer_stack, top_layer) + escaped_ex elseif k == K"juxtapose" - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) @chk numchildren(ex) == 2 @ast ctx ex [K"call" "*"::K"Identifier"(scope_layer=layerid) @@ -245,7 +307,7 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) @chk numchildren(ex) == 1 # TODO: Upstream should set a general flag for detecting parenthesized # expressions so we don't need to dig into `green_tree` here. Ugh! - plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) && + plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) && kind(ex[1]) == K"Identifier" && (sr = sourceref(ex); sr isa SourceRef && kind(sr.green_tree[2]) != K"parens") if plain_symbol @@ -322,7 +384,7 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) elseif k == K"<:" || k == K">:" || k == K"-->" # TODO: Should every form get layerid systematically? Or only the ones # which expand_forms_2 needs? - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) mapchildren(e->expand_forms_1(ctx,e), ctx, ex; scope_layer=layerid) else mapchildren(e->expand_forms_1(ctx,e), ctx, ex) @@ -336,12 +398,13 @@ function expand_forms_1(mod::Module, ex::SyntaxTree) __macro_ctx__=Nothing, meta=CompileHints) layers = ScopeLayer[ScopeLayer(1, mod, false)] - ctx = MacroExpansionContext(graph, Bindings(), layers, layers[1]) + ctx = MacroExpansionContext(graph, Bindings(), layers, LayerId[1]) ex2 = expand_forms_1(ctx, reparent(ctx, ex)) graph2 = delete_attributes(graph, :__macro_ctx__) # TODO: Returning the context with pass-specific mutable data is a bad way - # to carry state into the next pass. - ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, - ctx.current_layer) + # to carry state into the next pass. We might fix this by attaching such + # data to the graph itself as global attributes? + ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[]) return ctx2, reparent(ctx2, ex2) end + diff --git a/src/vendored/JuliaLowering/src/syntax_graph.jl b/src/vendored/JuliaLowering/src/syntax_graph.jl index 28af9cfb..6a591300 100644 --- a/src/vendored/JuliaLowering/src/syntax_graph.jl +++ b/src/vendored/JuliaLowering/src/syntax_graph.jl @@ -422,7 +422,7 @@ attrsummary(name, value::Number) = "$name=$value" function _value_string(ex) k = kind(ex) str = k == K"Identifier" || k == K"MacroName" || is_operator(k) ? ex.name_val : - k == K"Placeholder" ? ex.name_val : + k == K"Placeholder" ? ex.name_val : k == K"SSAValue" ? "%" : k == K"BindingId" ? "#" : k == K"label" ? "label" : @@ -540,7 +540,12 @@ JuliaSyntax.byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) function JuliaSyntax._expr_leaf_val(ex::SyntaxTree) name = get(ex, :name_val, nothing) if !isnothing(name) - Symbol(name) + n = Symbol(name) + if hasattr(ex, :scope_layer) + Expr(:scope_layer, n, ex.scope_layer) + else + n + end else ex.value end diff --git a/vendor/run.jl b/vendor/run.jl index a0559fb8..ab293c82 100644 --- a/vendor/run.jl +++ b/vendor/run.jl @@ -16,7 +16,7 @@ deps = [PackageAnalyzer.Added(; name="JuliaSyntax", uuid=UUID("f3c80556-a63f-4383-b822-37d64f81a311"), path="", repo_url="https://github.com/ericphanson/JuliaLowering.jl", - tree_hash = get_tree_hash("mlechu/JuliaLowering.jl", "eph/trunk"), + tree_hash = get_tree_hash("ericphanson/JuliaLowering.jl", "eph/trunk"), subdir="")] for pkg in deps From 41d072a1b2d91396db06e5fef114b30da54cb0b5 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Sun, 3 Aug 2025 04:00:44 +0200 Subject: [PATCH 35/35] wip --- src/ExplicitImports.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ExplicitImports.jl b/src/ExplicitImports.jl index 8c2370a7..696bdee0 100644 --- a/src/ExplicitImports.jl +++ b/src/ExplicitImports.jl @@ -1,6 +1,4 @@ module ExplicitImports -const HAS_PUBLIC2 = Base.eval(quote Base.VERSION >= v"1.11.0-DEV.469" end) - MUST_USE_JULIA_LOWERING::Bool = true @@ -33,7 +31,8 @@ using Pkg: Pkg # debug parsefile -public ignore_submodules +# we'll borrow their `@_public` macro; if this goes away, we can get our own +JuliaSyntax.@_public public ignore_submodules export print_explicit_imports, explicit_imports, check_no_implicit_imports, explicit_imports_nonrecursive