From 69725cda0c581f46e0987933d54c14bbcfef2b41 Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Fri, 24 Oct 2025 09:58:12 +0200 Subject: [PATCH 1/2] Move AbstractChunkTiledArray into separate file --- src/DiskArrays.jl | 1 + src/cached.jl | 26 -------------------------- src/chunktiledarray.jl | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 src/chunktiledarray.jl diff --git a/src/DiskArrays.jl b/src/DiskArrays.jl index 9f0bba6..28689df 100644 --- a/src/DiskArrays.jl +++ b/src/DiskArrays.jl @@ -34,6 +34,7 @@ include("cat.jl") include("generator.jl") include("zip.jl") include("show.jl") +include("chunktiledarray.jl") include("cached.jl") include("pad.jl") diff --git a/src/cached.jl b/src/cached.jl index e428b77..8fd17fd 100644 --- a/src/cached.jl +++ b/src/cached.jl @@ -1,31 +1,5 @@ import Mmap -""" - ChunkTiledDiskArray <: AbstractDiskArray - -And abstract supertype for disk arrays that have fast indexing -of tiled chunks already stored as separate arrays, such as [`CachedDiskArray`](@ref). -""" -abstract type ChunkTiledDiskArray{T,N} <: AbstractDiskArray{T,N} end - -Base.size(a::ChunkTiledDiskArray) = arraysize_from_chunksize.(eachchunk(a).chunks) - -function readblock!(A::ChunkTiledDiskArray{T,N}, data, I...) where {T,N} - chunks = eachchunk(A) - chunk_indices = findchunk.(chunks.chunks, I) - data_offset = OffsetArray(data, map(i -> first(i) - 1, I)...) - foreach(CartesianIndices(chunk_indices)) do ci - chunkindex = ChunkIndex(ci; offset=true) - chunk = A[chunkindex] - # Find the overlapping indices - inner_indices = map(axes(chunk), axes(data_offset)) do ax1, ax2 - max(first(ax1), first(ax2)):min(last(ax1), last(ax2)) - end - for ii in CartesianIndices(inner_indices) - data_offset[ii] = chunk[ii] - end - end -end """ CachedDiskArray <: ChunkTiledDiskArray diff --git a/src/chunktiledarray.jl b/src/chunktiledarray.jl new file mode 100644 index 0000000..56fb502 --- /dev/null +++ b/src/chunktiledarray.jl @@ -0,0 +1,37 @@ +""" + AbstractChunkTiledDiskArray <: AbstractDiskArray + +An abstract supertype for disk arrays that have fast indexing +of tiled chunks already stored as separate arrays, such as [`CachedDiskArray`](@ref). +""" +abstract type AbstractChunkTiledDiskArray{T,N} <: AbstractDiskArray{T,N} end + +Base.size(a::AbstractChunkTiledDiskArray) = arraysize_from_chunksize.(eachchunk(a).chunks) + +function readblock!(A::AbstractAbstractChunkTiledDiskArray{T,N}, data, I...) where {T,N} + chunks = eachchunk(A) + chunk_indices = findchunk.(chunks.chunks, I) + data_offset = OffsetArray(data, map(i -> first(i) - 1, I)...) + foreach(CartesianIndices(chunk_indices)) do ci + chunkindex = ChunkIndex(ci; offset=true) + chunk = A[chunkindex] + # Find the overlapping indices + inner_indices = map(axes(chunk), axes(data_offset)) do ax1, ax2 + max(first(ax1), first(ax2)):min(last(ax1), last(ax2)) + end + for ii in CartesianIndices(inner_indices) + data_offset[ii] = chunk[ii] + end + end +end + +""" + TiledDiskArray <: AbstractChunkTiledDiskArray + +Construct an array from a collection of tiles. +This needs a function to find the tile given a tile position and the overall size of the array. +""" +struct TiledDiskArray{T,N} <: AbstractChunkTiledDiskArray{T,N} + tilefunction + size::NTuple{N, Int} +end From ced4c474608507be64ecf4b3fe9282508591f192 Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Fri, 24 Oct 2025 14:48:28 +0200 Subject: [PATCH 2/2] Initial implementation of TiledDiskArray --- src/cached.jl | 5 ++--- src/chunktiledarray.jl | 27 +++++++++++++++++++++++++-- test/runtests.jl | 6 ++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/cached.jl b/src/cached.jl index 8fd17fd..2f4eb89 100644 --- a/src/cached.jl +++ b/src/cached.jl @@ -1,8 +1,7 @@ import Mmap - """ - CachedDiskArray <: ChunkTiledDiskArray + CachedDiskArray <: AbstractChunkTiledDiskArray CachedDiskArray(A::AbstractArray; maxsize=1000, mmap=false) @@ -14,7 +13,7 @@ to temproray files. Can also be called with `cache`, which can be extended for wrapper array types. """ -struct CachedDiskArray{T,N,A<:AbstractArray{T,N},C} <: ChunkTiledDiskArray{T,N} +struct CachedDiskArray{T,N,A<:AbstractArray{T,N},C} <: AbstractChunkTiledDiskArray{T,N} parent::A cache::C mmap::Bool diff --git a/src/chunktiledarray.jl b/src/chunktiledarray.jl index 56fb502..b4cfaa5 100644 --- a/src/chunktiledarray.jl +++ b/src/chunktiledarray.jl @@ -8,12 +8,14 @@ abstract type AbstractChunkTiledDiskArray{T,N} <: AbstractDiskArray{T,N} end Base.size(a::AbstractChunkTiledDiskArray) = arraysize_from_chunksize.(eachchunk(a).chunks) -function readblock!(A::AbstractAbstractChunkTiledDiskArray{T,N}, data, I...) where {T,N} +function readblock!(A::AbstractChunkTiledDiskArray{T,N}, data, I...) where {T,N} chunks = eachchunk(A) chunk_indices = findchunk.(chunks.chunks, I) data_offset = OffsetArray(data, map(i -> first(i) - 1, I)...) foreach(CartesianIndices(chunk_indices)) do ci + @show ci chunkindex = ChunkIndex(ci; offset=true) + @show chunkindex chunk = A[chunkindex] # Find the overlapping indices inner_indices = map(axes(chunk), axes(data_offset)) do ax1, ax2 @@ -33,5 +35,26 @@ This needs a function to find the tile given a tile position and the overall siz """ struct TiledDiskArray{T,N} <: AbstractChunkTiledDiskArray{T,N} tilefunction - size::NTuple{N, Int} + tilenum + tilesizes +end + + +Base.size(A::TiledDiskArray) = A.tilenum .* A.tilesizes +eachchunk(A::TiledDiskArray) = DiskArrays.GridChunks(size(A), A.tilesizes) +haschunks(A::TiledDiskArray) = Chunked() + +function Base.getindex(A::TiledDiskArray, i::ChunkIndex{N,OffsetChunks}) where {N} + tile = _getchunk(A,i) + inds = eachchunk(A)[i.I] + wrapchunk(tile, inds) +end + +Base.getindex(A::TiledDiskArray, i::ChunkIndex{N,OneBasedChunks}) where {N} = + _getchunk(A, i) + + + +function _getchunk(A::TiledDiskArray, i::ChunkIndex) + A.tilefunction(i.I.I...) end diff --git a/test/runtests.jl b/test/runtests.jl index ef0aee1..da28271 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1157,3 +1157,9 @@ end @test DiskArrays.haschunks(a_chunked_2) isa DiskArrays.Chunked @test size(DiskArrays.eachchunk(a_chunked_2)) == (5,5) end + +@testset "TiledChunkArray" begin + tiles = [fill(10x+y, 10,10) for x in 1:9, y in 1:9] + a = DiskArrays.TiledDiskArray((x,y) -> tiles[x,y], (90,90)) + @test a[1,1] == 11 +end