From 45f69a43af457313b92d4f138120e982f76b8bfb Mon Sep 17 00:00:00 2001 From: ZacNugent Date: Wed, 26 Feb 2020 23:15:11 +0000 Subject: [PATCH 01/10] PoC --- src/imports.jl | 4 +++- src/server.jl | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/imports.jl b/src/imports.jl index aed6eacd..625f573b 100644 --- a/src/imports.jl +++ b/src/imports.jl @@ -93,7 +93,9 @@ end function _get_field(par, arg, state) if par isa Dict{String,SymbolServer.ModuleStore} # package store - if haskey(par, CSTParser.str_value(arg)) + if haskey(state.server.workspacepackages, CSTParser.str_value(arg)) + return state.server.workspacepackages[CSTParser.str_value(arg)] + elseif haskey(par, CSTParser.str_value(arg)) return par[CSTParser.str_value(arg)] end elseif par isa Scope diff --git a/src/server.jl b/src/server.jl index 946e9060..ea6d1ec9 100644 --- a/src/server.jl +++ b/src/server.jl @@ -13,6 +13,7 @@ mutable struct FileServer <: AbstractServer files::Dict{String,File} roots::Set{File} symbolserver::Dict{String,SymbolServer.ModuleStore} + workspacepackages::Dict{String,Binding} end # Interface spec. From 7f3db182226f7fecb7ce3a1bf7bbabd595791e45 Mon Sep 17 00:00:00 2001 From: ZacNugent Date: Sat, 14 Mar 2020 09:02:04 +0000 Subject: [PATCH 02/10] wip --- src/imports.jl | 12 ++++++++++-- src/server.jl | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/imports.jl b/src/imports.jl index 011b4d23..8f3b6b63 100644 --- a/src/imports.jl +++ b/src/imports.jl @@ -96,11 +96,19 @@ function _mark_import_arg(arg, par, state, u) end end +function has_workspace_package(server, name) + haskey(server.workspacepackages, name) && + hasscope(getcst(server.workspacepackages[name])) && + haskey(scopeof(getcst(server.workspacepackages[name])).names, name) && + scopeof(getcst(server.workspacepackages[name])).names[name] isa Binding && + scopeof(getcst(server.workspacepackages[name])).names[name].val isa EXPR && + (typof(scopeof(getcst(server.workspacepackages[name])).names[name].val) in (ModuleH,BareModule)) +end function _get_field(par, arg, state) if par isa Dict{String,SymbolServer.ModuleStore} # package store - if haskey(state.server.workspacepackages, CSTParser.str_value(arg)) - return state.server.workspacepackages[CSTParser.str_value(arg)] + if has_workspace_package(state.server, CSTParser.str_value(arg)) + return scopeof(getcst(state.server.workspacepackages[CSTParser.str_value(arg)])).names[CSTParser.str_value(arg)] elseif haskey(par, CSTParser.str_value(arg)) return par[CSTParser.str_value(arg)] end diff --git a/src/server.jl b/src/server.jl index ea6d1ec9..38968e5e 100644 --- a/src/server.jl +++ b/src/server.jl @@ -13,7 +13,7 @@ mutable struct FileServer <: AbstractServer files::Dict{String,File} roots::Set{File} symbolserver::Dict{String,SymbolServer.ModuleStore} - workspacepackages::Dict{String,Binding} + workspacepackages::Dict{String,File} # list of files that may represent within-workspace packages end # Interface spec. From 7e8d61fd78b1c7fd367afc5c9b1f9e9f332239d8 Mon Sep 17 00:00:00 2001 From: ZacNugent Date: Tue, 28 Apr 2020 08:59:52 +0100 Subject: [PATCH 03/10] fix --- src/imports.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imports.jl b/src/imports.jl index 0656c6c8..7f3fa805 100644 --- a/src/imports.jl +++ b/src/imports.jl @@ -106,9 +106,9 @@ end function _get_field(par, arg, state) arg_str_rep = CSTParser.str_value(arg) if par isa SymbolServer.EnvStore # package store - if has_workspace_package(state.server, CSTParser.str_value(arg)) - return scopeof(getcst(state.server.workspacepackages[CSTParser.str_value(arg)])).names[CSTParser.str_value(arg)] - elseif haskey(par, CSTParser.str_value(arg)) + if has_workspace_package(state.server, arg_str_rep) + return scopeof(getcst(state.server.workspacepackages[arg_str_rep])).names[arg_str_rep] + elseif haskey(par, Symbol(arg_str_rep)) return par[Symbol(arg_str_rep)] end elseif par isa Scope From 388c6e171bfeb2f1bff1e34a1bd20e64d55a222d Mon Sep 17 00:00:00 2001 From: ZacNugent Date: Sat, 2 May 2020 09:53:52 +0100 Subject: [PATCH 04/10] test --- src/server.jl | 2 +- test/runtests.jl | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/server.jl b/src/server.jl index 07c1306b..0a082d6a 100644 --- a/src/server.jl +++ b/src/server.jl @@ -16,7 +16,7 @@ mutable struct FileServer <: AbstractServer symbol_extends::Dict{SymbolServer.VarRef, Vector{SymbolServer.VarRef}} workspacepackages::Dict{String,File} # list of files that may represent within-workspace packages end -FileServer() = FileServer(Dict{String,File}(), Set{File}(), deepcopy(SymbolServer.stdlibs), SymbolServer.collect_extended_methods(SymbolServer.stdlibs)) +FileServer() = FileServer(Dict{String,File}(), Set{File}(), deepcopy(SymbolServer.stdlibs), SymbolServer.collect_extended_methods(SymbolServer.stdlibs), Dict{String,File}()) # Interface spec. # AbstractServer :-> (has/canload/load/set/get)file, getsymbolserver, getsymbolextends diff --git a/test/runtests.jl b/test/runtests.jl index a9c7642f..03f0cffb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -797,4 +797,31 @@ end @test StaticLint.hasref(cst[1][6]) end end + +@testset "test workspace packages" begin + empty!(server.files) + s1 = """ + module WorkspaceMod + inner_sym = 1 + exported_sym = 1 + export exported_sym + end""" + f1 = StaticLint.File("workspacemod.jl", s1, CSTParser.parse(s1, true), nothing, server) + StaticLint.setroot(f1, f1) + StaticLint.setfile(server, f1.path, f1) + StaticLint.scopepass(f1) + server.workspacepackages["WorkspaceMod"] = f1 + s2 = """ + using WorkspaceMod + exported_sym + WorkspaceMod.inner_sym + """ + f2 = StaticLint.File("someotherfile.jl", s2, CSTParser.parse(s2, true), nothing, server) + StaticLint.setroot(f2, f2) + StaticLint.setfile(server, f2.path, f2) + StaticLint.scopepass(f2) + @test StaticLint.hasref(StaticLint.getcst(f2)[1][2]) + @test StaticLint.hasref(StaticLint.getcst(f2)[2]) + @test StaticLint.hasref(StaticLint.getcst(f2)[3][3][1]) +end end From 100a49baa1b9241361c6f531246520b74e91476a Mon Sep 17 00:00:00 2001 From: ZacNugent Date: Sat, 2 May 2020 12:48:00 +0100 Subject: [PATCH 05/10] Fix symbol resolving for used modules that point to live code --- src/references.jl | 85 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/src/references.jl b/src/references.jl index 663519a5..5cb2bb87 100644 --- a/src/references.jl +++ b/src/references.jl @@ -42,16 +42,8 @@ end function resolve_ref(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool hasref(x) && return true - resolved = false - if (typof(scope.expr) === CSTParser.ModuleH || typof(scope.expr) === CSTParser.BareModule) && CSTParser.length(scope.expr) > 1 && CSTParser.typof(scope.expr[2]) === IDENTIFIER - s_m_name = scope.expr[2].val isa String ? scope.expr[2].val : "" - if s_m_name in visited_scopes - return resolved - else - push!(visited_scopes, s_m_name) - end - end + module_safety_trip(scope, visited_scopes) && return false if is_getfield(x) return resolve_getfield(x, scope, state) @@ -94,7 +86,7 @@ function resolve_ref(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool resolved = true elseif scope.modules isa Dict && length(scope.modules) > 0 for m in scope.modules - resolved = resolve_ref(x, m[2], state, visited_scopes) + resolved = resolve_ref_from_module(x, m[2], state, visited_scopes) resolved && return true end end @@ -105,7 +97,7 @@ function resolve_ref(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool end # Searches a module store for a binding/variable that matches the reference `x1`. -function resolve_ref(x1::EXPR, m::SymbolServer.ModuleStore, state::State, visited_scopes)::Bool +function resolve_ref_from_module(x1::EXPR, m::SymbolServer.ModuleStore, state::State, visited_scopes)::Bool hasref(x1) && return true if isidentifier(x1) x = x1 @@ -145,6 +137,57 @@ function resolve_ref(x1::EXPR, m::SymbolServer.ModuleStore, state::State, visite return false end +function resolve_ref_from_module(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool + hasref(x) && return true + resolved = false + module_safety_trip(scope, visited_scopes) && return false + + if isidentifier(x) + if typof(x) === IDENTIFIER + mn = valof(x) + x1 = x + else + # NONSTDIDENTIFIER, e.g. var"name" + mn = valof(x[2]) + x1 = x + end + elseif resolvable_macroname(x) + x1 = x[2] + mn = string("@", valof(x1)) + elseif typof(x) === x_Str + if typof(x[1]) === IDENTIFIER + x1 = x[1] + mn = string("@", valof(x1), "_str") + else + return false + end + else + return true # TODO: Should be false? + end + + if scope_exports(scope, mn) + setref!(x1, scope.names[mn]) + resolved = true + end + return resolved +end + +""" + scope_exports(scope::Scope, name::String) + +Does the scope export a variable called `name`? +""" +function scope_exports(scope::Scope, name::String) + if scopehasbinding(scope, name) && (b = scope.names[name]) isa Binding + for ref in b.refs + if ref isa EXPR && parentof(ref) isa EXPR && typof(parentof(ref)) === CSTParser.Export + return true + end + end + end + return false +end + # Fallback method function resolve_ref(x::EXPR, m, state::State, visited_scopes)::Bool return hasref(x)::Bool @@ -256,4 +299,22 @@ function _in_macro_def(x::EXPR) else return false end -end \ No newline at end of file +end + +""" + module_safety_trip(scope::Scope, visited_scopes) + +Checks whether the scope is a module and we've visited it before, +otherwise adds the module to the list. +""" +function module_safety_trip(scope::Scope, visited_scopes) + if CSTParser.defines_module(scope.expr) && CSTParser.length(scope.expr) > 1 && CSTParser.typof(scope.expr[2]) === IDENTIFIER + s_m_name = scope.expr[2].val isa String ? scope.expr[2].val : "" + if s_m_name in visited_scopes + return true + else + push!(visited_scopes, s_m_name) + end + end + return false +end From 53666be9c1a8f6f5e8cd65be7835712107ffc6e4 Mon Sep 17 00:00:00 2001 From: ZacNugent Date: Thu, 7 May 2020 12:38:38 +0100 Subject: [PATCH 06/10] revert changes moved to separate PR --- src/references.jl | 85 +++++++---------------------------------------- 1 file changed, 12 insertions(+), 73 deletions(-) diff --git a/src/references.jl b/src/references.jl index 5cb2bb87..663519a5 100644 --- a/src/references.jl +++ b/src/references.jl @@ -42,8 +42,16 @@ end function resolve_ref(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool hasref(x) && return true + resolved = false - module_safety_trip(scope, visited_scopes) && return false + if (typof(scope.expr) === CSTParser.ModuleH || typof(scope.expr) === CSTParser.BareModule) && CSTParser.length(scope.expr) > 1 && CSTParser.typof(scope.expr[2]) === IDENTIFIER + s_m_name = scope.expr[2].val isa String ? scope.expr[2].val : "" + if s_m_name in visited_scopes + return resolved + else + push!(visited_scopes, s_m_name) + end + end if is_getfield(x) return resolve_getfield(x, scope, state) @@ -86,7 +94,7 @@ function resolve_ref(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool resolved = true elseif scope.modules isa Dict && length(scope.modules) > 0 for m in scope.modules - resolved = resolve_ref_from_module(x, m[2], state, visited_scopes) + resolved = resolve_ref(x, m[2], state, visited_scopes) resolved && return true end end @@ -97,7 +105,7 @@ function resolve_ref(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool end # Searches a module store for a binding/variable that matches the reference `x1`. -function resolve_ref_from_module(x1::EXPR, m::SymbolServer.ModuleStore, state::State, visited_scopes)::Bool +function resolve_ref(x1::EXPR, m::SymbolServer.ModuleStore, state::State, visited_scopes)::Bool hasref(x1) && return true if isidentifier(x1) x = x1 @@ -137,57 +145,6 @@ function resolve_ref_from_module(x1::EXPR, m::SymbolServer.ModuleStore, state::S return false end -function resolve_ref_from_module(x::EXPR, scope::Scope, state::State, visited_scopes)::Bool - hasref(x) && return true - resolved = false - module_safety_trip(scope, visited_scopes) && return false - - if isidentifier(x) - if typof(x) === IDENTIFIER - mn = valof(x) - x1 = x - else - # NONSTDIDENTIFIER, e.g. var"name" - mn = valof(x[2]) - x1 = x - end - elseif resolvable_macroname(x) - x1 = x[2] - mn = string("@", valof(x1)) - elseif typof(x) === x_Str - if typof(x[1]) === IDENTIFIER - x1 = x[1] - mn = string("@", valof(x1), "_str") - else - return false - end - else - return true # TODO: Should be false? - end - - if scope_exports(scope, mn) - setref!(x1, scope.names[mn]) - resolved = true - end - return resolved -end - -""" - scope_exports(scope::Scope, name::String) - -Does the scope export a variable called `name`? -""" -function scope_exports(scope::Scope, name::String) - if scopehasbinding(scope, name) && (b = scope.names[name]) isa Binding - for ref in b.refs - if ref isa EXPR && parentof(ref) isa EXPR && typof(parentof(ref)) === CSTParser.Export - return true - end - end - end - return false -end - # Fallback method function resolve_ref(x::EXPR, m, state::State, visited_scopes)::Bool return hasref(x)::Bool @@ -299,22 +256,4 @@ function _in_macro_def(x::EXPR) else return false end -end - -""" - module_safety_trip(scope::Scope, visited_scopes) - -Checks whether the scope is a module and we've visited it before, -otherwise adds the module to the list. -""" -function module_safety_trip(scope::Scope, visited_scopes) - if CSTParser.defines_module(scope.expr) && CSTParser.length(scope.expr) > 1 && CSTParser.typof(scope.expr[2]) === IDENTIFIER - s_m_name = scope.expr[2].val isa String ? scope.expr[2].val : "" - if s_m_name in visited_scopes - return true - else - push!(visited_scopes, s_m_name) - end - end - return false -end +end \ No newline at end of file From 422787e8967305a7877d06e9e8622eb74e0317ad Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Sat, 24 Jul 2021 18:21:16 -0700 Subject: [PATCH 07/10] Update tests to use semantic_pass instead of scopepass --- test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index a5eb2ca9..c6e21379 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1692,7 +1692,7 @@ end f1 = StaticLint.File("workspacemod.jl", s1, CSTParser.parse(s1, true), nothing, server) StaticLint.setroot(f1, f1) StaticLint.setfile(server, f1.path, f1) - StaticLint.scopepass(f1) + StaticLint.semantic_pass(f1) server.workspacepackages["WorkspaceMod"] = f1 s2 = """ using WorkspaceMod @@ -1702,7 +1702,7 @@ end f2 = StaticLint.File("someotherfile.jl", s2, CSTParser.parse(s2, true), nothing, server) StaticLint.setroot(f2, f2) StaticLint.setfile(server, f2.path, f2) - StaticLint.scopepass(f2) + StaticLint.semantic_pass(f2) @test StaticLint.hasref(StaticLint.getcst(f2)[1][2]) @test StaticLint.hasref(StaticLint.getcst(f2)[2]) @test StaticLint.hasref(StaticLint.getcst(f2)[3][3][1]) From 3d39aee401ae387942f87445ca6a9c06f9ca89e9 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Sat, 24 Jul 2021 18:32:53 -0700 Subject: [PATCH 08/10] Fix a typo --- src/imports.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/imports.jl b/src/imports.jl index 0a8666f0..01712075 100644 --- a/src/imports.jl +++ b/src/imports.jl @@ -88,10 +88,10 @@ function has_workspace_package(server, name) haskey(server.workspacepackages, name) && hasscope(getcst(server.workspacepackages[name])) && haskey(scopeof(getcst(server.workspacepackages[name])).names, name) && - scopeof(getcst(server.workspacepackages[name])).names[name] isa Binding && - scopeof(getcst(server.workspacepackages[name])).names[name].val isa EXPR && - (typof(scopeof(getcst(server.workspacepackages[name])).names[name].val) in (ModuleH,BareModule)) -end + scopeof(getcst(server.workspacepackages[name])).names[name] isa Binding && + scopeof(getcst(server.workspacepackages[name])).names[name].val isa EXPR && + (typeof(scopeof(getcst(server.workspacepackages[name])).names[name].val) in (ModuleH, BareModule)) +end function add_to_imported_modules(scope::Scope, name::Symbol, val) if scope.modules isa Dict From 96d6f4c493ab36776c00814924fca0a2fb78cf33 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Sat, 24 Jul 2021 18:38:17 -0700 Subject: [PATCH 09/10] Fix a typo --- src/imports.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imports.jl b/src/imports.jl index 01712075..da0fbe45 100644 --- a/src/imports.jl +++ b/src/imports.jl @@ -90,7 +90,7 @@ function has_workspace_package(server, name) haskey(scopeof(getcst(server.workspacepackages[name])).names, name) && scopeof(getcst(server.workspacepackages[name])).names[name] isa Binding && scopeof(getcst(server.workspacepackages[name])).names[name].val isa EXPR && - (typeof(scopeof(getcst(server.workspacepackages[name])).names[name].val) in (ModuleH, BareModule)) + (typeof(scopeof(getcst(server.workspacepackages[name])).names[name].val) in (Module, BareModule)) end function add_to_imported_modules(scope::Scope, name::Symbol, val) From 01adc1e49d654b1f62a2e9d1fd5a58b71515b064 Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Mon, 26 Jul 2021 16:38:41 +0200 Subject: [PATCH 10/10] fixes --- src/imports.jl | 2 +- test/runtests.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imports.jl b/src/imports.jl index da0fbe45..9f5e9b52 100644 --- a/src/imports.jl +++ b/src/imports.jl @@ -90,7 +90,7 @@ function has_workspace_package(server, name) haskey(scopeof(getcst(server.workspacepackages[name])).names, name) && scopeof(getcst(server.workspacepackages[name])).names[name] isa Binding && scopeof(getcst(server.workspacepackages[name])).names[name].val isa EXPR && - (typeof(scopeof(getcst(server.workspacepackages[name])).names[name].val) in (Module, BareModule)) + CSTParser.defines_module(scopeof(getcst(server.workspacepackages[name])).names[name].val) end function add_to_imported_modules(scope::Scope, name::Symbol, val) diff --git a/test/runtests.jl b/test/runtests.jl index c6e21379..50cfbacc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1703,7 +1703,7 @@ end StaticLint.setroot(f2, f2) StaticLint.setfile(server, f2.path, f2) StaticLint.semantic_pass(f2) - @test StaticLint.hasref(StaticLint.getcst(f2)[1][2]) + @test StaticLint.hasref(StaticLint.getcst(f2)[1][2][1]) @test StaticLint.hasref(StaticLint.getcst(f2)[2]) @test StaticLint.hasref(StaticLint.getcst(f2)[3][3][1]) end