argmin and argmax return missing if all values are missing

Author: sl-solution

src/byrow/byrow.jl

@@ -61,10 +61,10 @@ byrow(ds::AbstractDataset, ::typeof(maximum), col::ColumnIndex; by = identity, t
 byrow(ds::AbstractDataset, ::typeof(minimum), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, threads = nrow(ds)>1000) = threads ? hp_row_minimum(ds, by, cols) : row_minimum(ds, by, cols)
 byrow(ds::AbstractDataset, ::typeof(minimum), col::ColumnIndex; by = identity, threads = nrow(ds)>1000) = byrow(ds, minimum, [col]; by = by, threads = threads)
 
-byrow(ds::AbstractDataset, ::typeof(argmin), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, threads = nrow(ds)>1000) = threads ? hp_row_argmin(ds, by, cols) : row_argmin(ds, by, cols)
+byrow(ds::AbstractDataset, ::typeof(argmin), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, threads = nrow(ds)>1000) = row_argmin(ds, by, cols, threads = threads)
 byrow(ds::AbstractDataset, ::typeof(argmin), col::ColumnIndex; by = identity, threads = nrow(ds)>1000) = byrow(ds, argmin, [col]; by = by, threads = threads)
 
-byrow(ds::AbstractDataset, ::typeof(argmax), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, threads = nrow(ds)>1000) = threads ? hp_row_argmax(ds, by, cols) : row_argmax(ds, by, cols)
+byrow(ds::AbstractDataset, ::typeof(argmax), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, threads = nrow(ds)>1000) = row_argmax(ds, by, cols, threads = threads)
 byrow(ds::AbstractDataset, ::typeof(argmax), col::ColumnIndex; by = identity, threads = nrow(ds)>1000) = byrow(ds, argmax, [col]; by = by, threads = threads)
 
 byrow(ds::AbstractDataset, ::typeof(var), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, dof = true, threads = nrow(ds)>1000) = threads ? hp_row_var(ds, by, cols; dof = dof) : row_var(ds, by, cols; dof = dof)

src/byrow/hp_row_functions.jl

@@ -154,40 +154,6 @@ end
 hp_row_maximum(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = hp_row_maximum(ds, identity, cols)
 
 
-function hp_op_for_argminmax!(x, y, f, vals, idx)
-    idx[] += 1
-    Threads.@threads for i in 1:length(x)
-        if isequal(vals[i], f(y[i])) && ismissing(x[i])
-            x[i] = idx[]
-        end
-    end
-    x
-end
-
-function hp_row_argmin(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}))
-    colsidx = index(ds)[cols]
-    minvals = hp_row_minimum(ds, f, cols)
-    colnames_pa = PooledArray(names(ds, colsidx))
-    idx = Ref{Int}(0)
-    res = mapreduce(identity, (x,y)->hp_op_for_argminmax!(x,y,f, minvals, idx), view(_columns(ds),colsidx), init = missings(eltype(colnames_pa.refs), nrow(ds)))
-    colnames_pa.refs = res
-    colnames_pa
-end
-hp_row_argmin(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = hp_row_argmin(ds, identity, cols)
-
-function hp_row_argmax(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}))
-    colsidx = index(ds)[cols]
-    maxvals = hp_row_maximum(ds, f, cols)
-    colnames_pa = PooledArray(names(ds, colsidx))
-    idx = Ref{Int}(0)
-    res = mapreduce(identity, (x,y)->hp_op_for_argminmax!(x,y,f, maxvals, idx), view(_columns(ds),colsidx), init = missings(eltype(colnames_pa.refs), nrow(ds)))
-    colnames_pa.refs = res
-    colnames_pa
-end
-hp_row_argmax(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = hp_row_argmax(ds, identity, cols)
-
-
-
 function hp_row_var(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); dof = true)
     colsidx = index(ds)[cols]
     CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))

src/byrow/row_functions.jl

@@ -205,37 +205,62 @@ function row_maximum(ds::AbstractDataset, f::Function, cols = names(ds, Union{Mi
 end
 row_maximum(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = row_maximum(ds, identity, cols)
 
-function _op_for_argminmax!(x, y, f, vals, idx)
+function _op_for_argminmax!(x, y, f, vals, idx, missref)
     idx[] += 1
     for i in 1:length(x)
-        if isequal(vals[i], f(y[i])) && ismissing(x[i])
+        if !ismissing(vals[i]) && isequal(vals[i], f(y[i])) && isequal(x[i], missref)
             x[i] = idx[]
         end
     end
     x
 end
+function hp_op_for_argminmax!(x, y, f, vals, idx, missref)
+    idx[] += 1
+    Threads.@threads for i in 1:length(x)
+        if !ismissing(vals[i]) && isequal(vals[i], f(y[i])) && isequal(x[i], missref)
+            x[i] = idx[]
+        end
+    end
+    x
+end
+
 
-function row_argmin(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}))
+function row_argmin(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
     colsidx = index(ds)[cols]
     minvals = row_minimum(ds, f, cols)
-    colnames_pa = PooledArray(names(ds, colsidx))
+    colnames_pa = allowmissing(PooledArray(names(ds, colsidx)))
+    push!(colnames_pa, missing)
+    missref = get(colnames_pa.invpool, missing, missing)
+    init0 = fill(missref, nrow(ds))
     idx = Ref{Int}(0)
-    res = mapreduce(identity, (x,y)->_op_for_argminmax!(x,y, f, minvals, idx), view(_columns(ds),colsidx), init = missings(eltype(colnames_pa.refs), nrow(ds)))
+    if threads
+        res = mapreduce(identity, (x,y)->hp_op_for_argminmax!(x,y, f, minvals, idx, missref), view(_columns(ds),colsidx), init = init0)
+
+    else
+        res = mapreduce(identity, (x,y)->_op_for_argminmax!(x,y, f, minvals, idx, missref), view(_columns(ds),colsidx), init = init0)
+    end
     colnames_pa.refs = res
     colnames_pa
 end
-row_argmin(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = row_argmin(ds, identity, cols)
+row_argmin(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_argmin(ds, identity, cols, threads = threads)
 
-function row_argmax(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}))
+function row_argmax(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
     colsidx = index(ds)[cols]
     maxvals = row_maximum(ds, f, cols)
-    colnames_pa = PooledArray(names(ds, colsidx))
+    colnames_pa = allowmissing(PooledArray(names(ds, colsidx)))
+    push!(colnames_pa, missing)
+    missref = get(colnames_pa.invpool, missing, missing)
+    init0 = fill(missref, nrow(ds))
     idx = Ref{Int}(0)
-    res = mapreduce(identity, (x,y)->_op_for_argminmax!(x,y,f, maxvals, idx), view(_columns(ds),colsidx), init = missings(eltype(colnames_pa.refs), nrow(ds)))
+    if threads
+        res = mapreduce(identity, (x,y)->hp_op_for_argminmax!(x,y,f, maxvals, idx, missref), view(_columns(ds),colsidx), init = init0)
+    else
+        res = mapreduce(identity, (x,y)->_op_for_argminmax!(x,y,f, maxvals, idx, missref), view(_columns(ds),colsidx), init = init0)
+    end
     colnames_pa.refs = res
     colnames_pa
 end
-row_argmax(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = row_argmax(ds, identity, cols)
+row_argmax(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_argmax(ds, identity, cols, threads = threads)
 
 
 # TODO better function for the first component of operator

src/stat/non_hp_stat.jl

@@ -67,6 +67,7 @@ end
 function stat_findmax(f, x::AbstractArray{T,1}) where T
     isempty(x) && throw(ArgumentError("input vector cannot be empty"))
     maxval = stat_maximum(f, x)
+    ismissing(maxval) && return (missing, missing)
     (maxval, _arg_minmax_barrier(x, maxval, f))
 end
 stat_findmax(x::AbstractArray{T,1}) where T = stat_findmax(identity, x)

test/byrow.jl

@@ -20,8 +20,8 @@
     @test isequal(byrow(ds, minimum, r"float", threads = true) , [1.2, missing, -1.4,2.3,-100.0])
 
     @test byrow(ds, argmax, :) == ["x2_int", "x2_int", "x2_float", "x2_int", "x1_float"]
-    @test byrow(ds, argmin, r"float") == ["x1_float", "x1_float", "x3_float", "x1_float", "x3_float"]
-    @test byrow(ds, argmin, r"float", by = abs) == ["x1_float", "x1_float", "x1_float", "x1_float", "x1_float"]
+    @test isequal(byrow(ds, argmin, r"float"), ["x1_float", missing, "x3_float", "x1_float", "x3_float"])
+    @test isequal(byrow(ds, argmin, r"float", by = abs) , ["x1_float", missing, "x1_float", "x1_float", "x1_float"])
     @test byrow(ds, coalesce, ["x2_float", "x1_float", "x1_int"]) == [1.2,0,3.0,2.3,10]
     @test isequal(byrow(ds, var, r"float"), [missing, missing, 5.92, 0.24499999999999922, 6050.0])
     @test isequal(byrow(ds, var, r"float", dof = false), [0.0, missing, 3.9466666666666663, 0.12249999999999961, 3025.0])
@@ -31,6 +31,9 @@
     @test byrow(sds, sum, [:g, :x2_float]) == [2,2.0,2,4,4,4,1,1,1,2,2,2,2,2]
     @test byrow(sds, argmax, [3,2,4,1]) == ["x1_float","x1_float","x1_float","x2_float","x2_float","x2_float","g", "g", "g", "x1_float","x1_float","x1_float","x1_float","x1_float"]
     @test byrow(sds, argmax, [1,2,4,3], by = ismissing) == ["x2_float","x2_float","x2_float", "x1_float","x1_float","x1_float", "x1_float","x1_float","x1_float", "x2_float", "x2_float", "x2_float", "x2_float", "x2_float"]
+    @test isequal(byrow(sds, argmax, [3,4]), ["x1_int","x1_int","x1_int","x2_float","x2_float","x2_float","x1_int","x1_int","x1_int",missing, missing, missing, missing, missing])
+    @test isequal(byrow(sds, argmin, [3,4], threads = true), ["x1_int","x1_int","x1_int","x1_int","x1_int","x1_int","x1_int","x1_int","x1_int",missing, missing, missing, missing, missing])
+
     @test byrow(sds, any, :, by = ismissing) == [true, true, true, false, false, false, true, true, true, true, true, true, true, true]
 
     ds = Dataset(x1 = [1,2,3,4,missing], x2 = [3,2,4,5, missing])