bug fix - byrow(nunique)

Author: sl-solution

src/byrow/byrow.jl

@@ -167,7 +167,14 @@ byrow(ds::AbstractDataset, ::typeof(var), col::ColumnIndex; by = identity, dof =
 byrow(ds::AbstractDataset, ::typeof(std), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, dof = true, threads = nrow(ds) > Threads.nthreads()*10) = row_std(ds, by, cols; dof = dof, threads = threads)
 byrow(ds::AbstractDataset, ::typeof(std), col::ColumnIndex; by = identity, dof = true, threads = nrow(ds) > Threads.nthreads()*10) = byrow(ds, std, [col]; by = by, dof = dof, threads = threads)
 
-byrow(ds::AbstractDataset, ::typeof(nunique), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, count_missing = true) = row_nunique(ds, by, cols; count_missing = count_missing)
+function byrow(ds::AbstractDataset, ::typeof(nunique), cols::MultiColumnIndex = names(ds, Union{Missing, Number}); by = identity, count_missing = true, threads=nrow(ds)>1000) 
+	res = byrow(ds, x->length(Set(Base.Generator(by, x))), cols, threads=threads)
+	if count_missing
+		return res
+	else
+		return res .- row_any(ds, ismissing, cols)
+	end
+end
 byrow(ds::AbstractDataset, ::typeof(nunique), col::ColumnIndex; by = identity, count_missing = true) = byrow(ds, nunique, [col]; by = by, count_missing = count_missing)
 
 

src/byrow/row_functions.jl

@@ -1077,25 +1077,26 @@ function _fill_dict_and_add!(init0, dict, prehashed, n, p)
     end
 end
 
-function row_nunique(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); count_missing = true)
-    colsidx = multiple_getindex(index(ds), cols)
-    prehashed = Matrix{_Prehashed}(undef, size(ds,1), length(colsidx))
-    allcols = view(_columns(ds),colsidx)
-
-    for j in 1:size(prehashed,2)
-        _fill_prehashed!(prehashed, allcols[j], f, size(ds,1), j)
-    end
-
-    init0 = zeros(Int32, size(ds,1))
-    dict = Dict{_Prehashed, Nothing}()
-    _fill_dict_and_add!(init0, dict, prehashed, size(ds,1), length(colsidx))
-    if count_missing
-        return init0
-    else
-        return init0 .- row_any(ds, ismissing, cols)
-    end
-end
-row_nunique(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); count_missing = true) = row_nunique(ds, identity, cols; count_missing = count_missing)
+# This is not working - because we only the hash values and in many cases like 2.1 and 4611911198408756429 the hash is the same
+# function row_nunique(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); count_missing = true)
+#     colsidx = multiple_getindex(index(ds), cols)
+#     prehashed = Matrix{_Prehashed}(undef, size(ds,1), length(colsidx))
+#     allcols = view(_columns(ds),colsidx)
+
+#     for j in 1:size(prehashed,2)
+#         _fill_prehashed!(prehashed, allcols[j], f, size(ds,1), j)
+#     end
+
+#     init0 = zeros(Int32, size(ds,1))
+#     dict = Dict{_Prehashed, Nothing}()
+#     _fill_dict_and_add!(init0, dict, prehashed, size(ds,1), length(colsidx))
+#     if count_missing
+#         return init0
+#     else
+#         return init0 .- row_any(ds, ismissing, cols)
+#     end
+# end
+# row_nunique(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); count_missing = true) = row_nunique(ds, identity, cols; count_missing = count_missing)
 
 Base.@propagate_inbounds function _op_for_hash!(x, y, f, lo, hi)
     @simd for i in lo:hi

test/byrow.jl

@@ -414,4 +414,12 @@ end
     @test byrow(ds, fun123, (1,2,3)) == [1,-1.0,-9,2.5]
     fun123_2(x,y) = x == 1 && y < 0 ? true : false
     @test byrow(ds, fun123_2, (:x1, :x2)) == [false, false, true, false]
+end
+
+@testset "byrow - nunique" begin
+    ds = Dataset(x=2.1, y=4611911198408756429, z=missing, k=-2.1)
+    @test byrow(ds, nunique, :)[1] == 4
+    @test byrow(ds, nunique, :, count_missing = false)[1] == 3
+    @test byrow(ds, nunique, :, by = abs)[1] == 3
+    @test byrow(ds, nunique, :, by = abs, count_missing=false)[1] == 2
 end