improved support for struct columns with missing values

Author: baumgold

Project.toml

@@ -17,7 +17,7 @@
 name = "Arrow"
 uuid = "69666777-d1a9-59fb-9406-91d4454c9d45"
 authors = ["quinnj <quinn.jacobd@gmail.com>"]
-version = "2.7.1"
+version = "2.7.2"
 
 [deps]
 ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"

src/arraytypes/struct.jl

@@ -77,16 +77,19 @@ struct ToStruct{T,i,A} <: AbstractVector{T}
     data::A # eltype is NamedTuple or some struct
 end
 
-ToStruct(x::A, j::Integer) where {A} =
-    ToStruct{fieldtype(Base.nonmissingtype(eltype(A)), j),j,A}(x)
+function ToStruct(x::A, j::Integer, hasmissing::Bool=false) where {A}
+    AT = fieldtype(eltype(A), j)
+    T = hasmissing ? Union{Missing,AT} : AT
+    ToStruct{T,j,A}(x)
+end
 
 Base.IndexStyle(::Type{<:ToStruct}) = Base.IndexLinear()
 Base.size(x::ToStruct) = (length(x.data),)
 
 Base.@propagate_inbounds function Base.getindex(A::ToStruct{T,j}, i::Integer) where {T,j}
     @boundscheck checkbounds(A, i)
     @inbounds x = A.data[i]
-    return x === missing ? ArrowTypes.default(T) : getfield(x, j)
+    ismissing(x) ? missing : getfield(x, j)
 end
 
 arrowvector(::StructKind, x::Struct, i, nl, fi, de, ded, meta; kw...) = x
@@ -102,9 +105,10 @@ function arrowvector(::StructKind, x, i, nl, fi, de, ded, meta; kw...)
     len = length(x)
     validity = ValidityBitmap(x)
     T = Base.nonmissingtype(eltype(x))
+    hasmissing = Missing <: eltype(x)
     data = Tuple(
-        arrowvector(ToStruct(x, j), i, nl + 1, j, de, ded, nothing; kw...) for
-        j = 1:fieldcount(T)
+        arrowvector(ToStruct(x, j, hasmissing), i, nl + 1, j, de, ded, nothing; kw...)
+        for j = 1:fieldcount(T)
     )
     NT = namedtupletype(T, data)
     return Struct{withmissing(eltype(x), NT),typeof(data),fieldnames(NT)}(