Merge branch 'master' into mb/mapreducedim_empty

Author: mbauman

Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl

@@ -47,7 +47,7 @@ end
 
 function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector)
     opt = result.src::Compiler.OptimizationState
-    ir = opt.optresult.ir::Compiler.IRCode
+    ir = opt.result.ir::Compiler.IRCode
     override = with_new_compiler
     for inst in ir.stmts
         stmt = inst[:stmt]

Compiler/src/Compiler.jl

@@ -65,7 +65,7 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout,
     uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal,
     _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized,
-    partialstruct_init_undefs, fieldcount_noerror
+    partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using
 using Base.Order
 
 import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, get!,
@@ -120,7 +120,7 @@ function is_return_type(Core.@nospecialize(f))
     return false
 end
 
-include("profiling.jl")
+include("timing.jl")
 include("sort.jl")
 
 # We don't include some.jl, but this definition is still useful.

Compiler/src/abstractinterpretation.jl

@@ -2437,14 +2437,15 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s
 end
 
 @nospecs function abstract_eval_get_binding_type(interp::AbstractInterpreter, sv::AbsIntState, M, s)
+    @nospecialize M s
     ⊑ = partialorder(typeinf_lattice(interp))
     if isa(M, Const) && isa(s, Const)
         (M, s) = (M.val, s.val)
         if !isa(M, Module) || !isa(s, Symbol)
             return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo())
         end
         gr = GlobalRef(M, s)
-        (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition
+        (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition
             local rt
             kind = binding_kind(partition)
             if is_some_guard(kind) || kind == PARTITION_KIND_DECLARED
@@ -2574,13 +2575,14 @@ function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntSta
             M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo())
             s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo())
             gr = GlobalRef(M, s)
-            (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, binding, partition
+            v′ = RefValue{Any}(v)
+            (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition
                 partition_T = nothing
                 partition_rte = abstract_eval_partition_load(interp, binding, partition)
                 if binding_kind(partition) == PARTITION_KIND_GLOBAL
                     partition_T = partition_restriction(partition)
                 end
-                partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v)[2]}
+                partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v′[])[2]}
                 partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects)
                 Pair{RTEffects, Any}(partition_rte, partition_T)
             end
@@ -3558,7 +3560,7 @@ function abstract_eval_binding_partition!(interp::AbstractInterpreter, g::Global
     return partition
 end
 
-function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, binding::Core.Binding, partition::Core.BindingPartition)
+function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing}, binding::Core.Binding, partition::Core.BindingPartition)
     kind = binding_kind(partition)
     isdepwarn = (partition.kind & PARTITION_FLAG_DEPWARN) != 0
     local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE)
@@ -3607,7 +3609,8 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing
     return RTEffects(rt, exct, effects)
 end
 
-function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp, g::GlobalRef, wwr::WorldWithRange)
+function scan_specified_partitions(query::F1, walk_binding_partition::F2,
+    interp::Union{AbstractInterpreter,Nothing}, g::GlobalRef, wwr::WorldWithRange) where {F1,F2}
     local total_validity, rte, binding_partition
     binding = convert(Core.Binding, g)
     lookup_world = max_world(wwr.valid_worlds)
@@ -3640,19 +3643,25 @@ function scan_specified_partitions(query::Function, walk_binding_partition::Func
     return Pair{WorldRange, typeof(rte)}(total_validity, rte)
 end
 
-scan_leaf_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) =
+scan_leaf_partitions(query::F, ::Nothing, g::GlobalRef, wwr::WorldWithRange) where F =
+    scan_specified_partitions(query, walk_binding_partition, nothing, g, wwr)
+scan_leaf_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F =
     scan_specified_partitions(query, walk_binding_partition, interp, g, wwr)
 
-scan_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) =
-    scan_specified_partitions(query,
-        (b::Core.Binding, bpart::Core.BindingPartition, world::UInt)->
-            Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(WorldRange(bpart.min_world, bpart.max_world), b=>bpart),
-        interp, g, wwr)
+function scan_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F
+    walk_binding_partition = function (b::Core.Binding, partition::Core.BindingPartition, world::UInt)
+        Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(
+            WorldRange(partition.min_world, partition.max_world), b=>partition)
+    end
+    return scan_specified_partitions(query, walk_binding_partition, interp, g, wwr)
+end
 
-abstract_load_all_consistent_leaf_partitions(interp, g::GlobalRef, wwr::WorldWithRange) =
+abstract_load_all_consistent_leaf_partitions(interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) =
     scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr)
+abstract_load_all_consistent_leaf_partitions(::Nothing, g::GlobalRef, wwr::WorldWithRange) =
+    scan_leaf_partitions(abstract_eval_partition_load, nothing, g, wwr)
 
-function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState)
+function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState)
     if saw_latestworld
         return RTEffects(Any, Any, generic_getglobal_effects)
     end
@@ -3668,7 +3677,10 @@ function global_assignment_rt_exct(interp::AbstractInterpreter, sv::AbsIntState,
     if saw_latestworld
         return Pair{Any,Any}(newty, ErrorException)
     end
-    (valid_worlds, ret) = scan_partitions((interp, _, partition)->global_assignment_binding_rt_exct(interp, partition, newty), interp, g, sv.world)
+    newty′ = RefValue{Any}(newty)
+    (valid_worlds, ret) = scan_partitions(interp, g, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition
+        global_assignment_binding_rt_exct(interp, partition, newty′[])
+    end
     update_valid_age!(sv, valid_worlds)
     return ret
 end
@@ -3716,8 +3728,9 @@ struct AbstractEvalBasicStatementResult
     end
 end
 
-function abstract_eval_basic_statement(interp::AbstractInterpreter, @nospecialize(stmt), sstate::StatementState, frame::InferenceState,
-                                       result::Union{Nothing,Future{RTEffects}}=nothing)
+@inline function abstract_eval_basic_statement(
+    interp::AbstractInterpreter, @nospecialize(stmt), sstate::StatementState, frame::InferenceState,
+    result::Union{Nothing,Future{RTEffects}}=nothing)
     rt = nothing
     exct = Bottom
     changes = nothing

Compiler/src/optimize.jl

@@ -116,14 +116,11 @@ function inline_cost_clamp(x::Int)
     return convert(InlineCostType, x)
 end
 
-const SRC_FLAG_DECLARED_INLINE = 0x1
-const SRC_FLAG_DECLARED_NOINLINE = 0x2
-
 is_declared_inline(@nospecialize src::MaybeCompressed) =
-    ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_INLINE
+    ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 1
 
 is_declared_noinline(@nospecialize src::MaybeCompressed) =
-    ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_NOINLINE
+    ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 2
 
 #####################
 # OptimizationState #
@@ -160,7 +157,6 @@ code_cache(state::InliningState) = WorldView(code_cache(state.interp), state.wor
 
 mutable struct OptimizationResult
     ir::IRCode
-    inline_flag::UInt8
     simplified::Bool # indicates whether the IR was processed with `cfg_simplify!`
 end
 
@@ -172,7 +168,7 @@ end
 mutable struct OptimizationState{Interp<:AbstractInterpreter}
     linfo::MethodInstance
     src::CodeInfo
-    optresult::Union{Nothing, OptimizationResult}
+    result::Union{Nothing, OptimizationResult}
     stmt_info::Vector{CallInfo}
     mod::Module
     sptypes::Vector{VarState}
@@ -240,29 +236,13 @@ include("ssair/EscapeAnalysis.jl")
 include("ssair/passes.jl")
 include("ssair/irinterp.jl")
 
-function ir_to_codeinf!(opt::OptimizationState, frame::InferenceState, edges::SimpleVector)
-    ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp, frame.result, opt.optresult))
-end
-
-function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector, inlining_cost::InlineCostType)
-    src = ir_to_codeinf!(opt, edges)
-    src.inlining_cost = inlining_cost
-    src
-end
-
-function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector)
-    src = ir_to_codeinf!(opt)
-    src.edges = edges
-    src
-end
-
 function ir_to_codeinf!(opt::OptimizationState)
-    (; linfo, src, optresult) = opt
-    if optresult === nothing
+    (; linfo, src, result) = opt
+    if result === nothing
         return src
     end
-    src = ir_to_codeinf!(src, optresult.ir)
-    opt.optresult = nothing
+    src = ir_to_codeinf!(src, result.ir)
+    opt.result = nothing
     opt.src = src
     maybe_validate_code(linfo, src, "optimized")
     return src
@@ -505,12 +485,63 @@ end
 abstract_eval_ssavalue(s::SSAValue, src::Union{IRCode,IncrementalCompact}) = types(src)[s]
 
 """
-    finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode)
+    finish(interp::AbstractInterpreter, opt::OptimizationState,
+           ir::IRCode, caller::InferenceResult)
 
-Called at the end of optimization to store the resulting IR back into the OptimizationState.
+Post-process information derived by Julia-level optimizations for later use.
+In particular, this function determines the inlineability of the optimized code.
 """
-function finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode)
-    opt.optresult = OptimizationResult(ir, ccall(:jl_ir_flag_inlining, UInt8, (Any,), opt.src), false)
+function finish(interp::AbstractInterpreter, opt::OptimizationState,
+                ir::IRCode, caller::InferenceResult)
+    (; src, linfo) = opt
+    (; def, specTypes) = linfo
+
+    force_noinline = is_declared_noinline(src)
+
+    # compute inlining and other related optimizations
+    result = caller.result
+    @assert !(result isa LimitedAccuracy)
+    result = widenslotwrapper(result)
+
+    opt.result = OptimizationResult(ir, false)
+
+    # determine and cache inlineability
+    if !force_noinline
+        sig = unwrap_unionall(specTypes)
+        if !(isa(sig, DataType) && sig.name === Tuple.name)
+            force_noinline = true
+        end
+        if !is_declared_inline(src) && result === Bottom
+            force_noinline = true
+        end
+    end
+    if force_noinline
+        set_inlineable!(src, false)
+    elseif isa(def, Method)
+        if is_declared_inline(src) && isdispatchtuple(specTypes)
+            # obey @inline declaration if a dispatch barrier would not help
+            set_inlineable!(src, true)
+        else
+            # compute the cost (size) of inlining this code
+            params = OptimizationParams(interp)
+            cost_threshold = default = params.inline_cost_threshold
+            if ⊑(optimizer_lattice(interp), result, Tuple) && !isconcretetype(widenconst(result))
+                cost_threshold += params.inline_tupleret_bonus
+            end
+            # if the method is declared as `@inline`, increase the cost threshold 20x
+            if is_declared_inline(src)
+                cost_threshold += 19*default
+            end
+            # a few functions get special treatment
+            if def.module === _topmod(def.module)
+                name = def.name
+                if name === :iterate || name === :unsafe_convert || name === :cconvert
+                    cost_threshold += 4*default
+                end
+            end
+            src.inlining_cost = inline_cost(ir, params, cost_threshold)
+        end
+    end
     return nothing
 end
 
@@ -984,8 +1015,7 @@ end
 function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult)
     @zone "CC: OPTIMIZER" ir = run_passes_ipo_safe(opt.src, opt)
     ipo_dataflow_analysis!(interp, opt, ir, caller)
-    finishopt!(interp, opt, ir)
-    return nothing
+    return finish(interp, opt, ir, caller)
 end
 
 const ALL_PASS_NAMES = String[]
@@ -1436,7 +1466,7 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod
     return thiscost
 end
 
-function inline_cost_model(ir::IRCode, params::OptimizationParams, cost_threshold::Int)
+function inline_cost(ir::IRCode, params::OptimizationParams, cost_threshold::Int)
     bodycost = 0
     for i = 1:length(ir.stmts)
         stmt = ir[SSAValue(i)][:stmt]