Skip to content

Commit 763d3cf

Browse files
JohanMabilleJohanMabille
authored
Merge pull request #104 from JohanMabille/assign
Refactored chunked file array assignment
2 parents 9365b6c + 9672039 commit 763d3cf

File tree

5 files changed

+250
-180
lines changed

5 files changed

+250
-180
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ install:
3535
# Install mamba
3636
- conda install mamba -c conda-forge
3737
# Install host dependencies
38-
- mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 xtensor=0.21.8 -c conda-forge
38+
- mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 xtensor=0.21.8 cpp-filesystem -c conda-forge
3939
- mamba install ffmpeg=4.1.3 -c conda-forge # openimageio is missing ffmpedg as a runtime requirement
4040
# Install build dependencies
4141
- mamba install cmake -c conda-forge

‎include/xtensor-io/xchunk_store_manager.hpp‎

Lines changed: 142 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,16 @@
44
#include <vector>
55
#include <array>
66

7+
#include "ghc/filesystem.hpp"
8+
79
#include "xtensor/xarray.hpp"
810
#include "xtensor/xchunked_array.hpp"
911
#include "xfile_array.hpp"
1012

1113
namespace xt
1214
{
15+
template <class EC, class IP>
16+
class xchunk_store_manager;
1317

1418
/***************************
1519
* xindex_path declaration *
@@ -18,21 +22,37 @@ namespace xt
1822
class xindex_path
1923
{
2024
public:
25+
26+
std::string get_directory() const;
2127
void set_directory(const std::string& directory);
28+
2229
template <class I>
23-
void index_to_path(I, I, std::string&);
30+
void index_to_path(I, I, std::string&) const;
2431

2532
private:
33+
2634
std::string m_directory;
2735
};
2836

37+
/*********************************
38+
* xchunked_assigner declaration *
39+
*********************************/
40+
41+
template <class T, class EC, class IP>
42+
class xchunked_assigner<T, xchunk_store_manager<EC, IP>>
43+
{
44+
public:
45+
46+
using temporary_type = T;
47+
48+
template <class E, class DST>
49+
void build_and_assign_temporary(const xexpression<E>& e, DST& dst);
50+
};
51+
2952
/************************************
3053
* xchunk_store_manager declaration *
3154
************************************/
3255

33-
template <class EC, class IP>
34-
class xchunk_store_manager;
35-
3656
template <class EC, class IP>
3757
struct xcontainer_inner_types<xchunk_store_manager<EC, IP>>
3858
{
@@ -73,7 +93,11 @@ namespace xt
7393
using shape_type = typename iterable_base::inner_shape_type;
7494

7595
template <class S>
76-
xchunk_store_manager(S&& shape, S&& chunk_shape, const std::string& directory, std::size_t pool_size, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
96+
xchunk_store_manager(S&& shape,
97+
S&& chunk_shape,
98+
const std::string& directory,
99+
std::size_t pool_size,
100+
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
77101
~xchunk_store_manager() = default;
78102

79103
xchunk_store_manager(const xchunk_store_manager&) = default;
@@ -109,7 +133,9 @@ namespace xt
109133
template <class S>
110134
void resize(S&& shape);
111135

112-
std::size_t size();
136+
size_type size() const;
137+
const std::string& get_directory() const;
138+
bool get_pool_size() const;
113139

114140
IP& get_index_path();
115141
void flush();
@@ -123,6 +149,9 @@ namespace xt
123149
template <class I>
124150
const_reference map_file_array(I first, I last) const;
125151

152+
std::string get_temporary_directory() const;
153+
void reset_to_directory(const std::string& directory);
154+
126155
private:
127156

128157
template <class... Idxs>
@@ -140,47 +169,36 @@ namespace xt
140169

141170
template <class T, class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class EXT = empty_extension, class S>
142171
xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>, EXT>
143-
chunked_file_array(S&& shape, S&& chunk_shape, const std::string& path, std::size_t pool_size = 1, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
172+
chunked_file_array(S&& shape,
173+
S&& chunk_shape,
174+
const std::string& path,
175+
std::size_t pool_size = 1,
176+
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
144177

145178
template <class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class EXT = empty_extension, class E, class S>
146179
xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
147-
chunked_file_array(const xexpression<E>& e, S&& chunk_shape, const std::string& path, std::size_t pool_size = 1, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
180+
chunked_file_array(const xexpression<E>& e,
181+
S&& chunk_shape,
182+
const std::string& path,
183+
std::size_t pool_size = 1,
184+
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
148185

149186
template <class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class EXT = empty_extension, class E>
150187
xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
151-
chunked_file_array(const xexpression<E>& e, const std::string& path, std::size_t pool_size = 1, layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
152-
153-
template <class T, class IOH, layout_type L, class IP, class EXT, class S>
154-
inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>, EXT>
155-
chunked_file_array(S&& shape, S&& chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout)
156-
{
157-
using chunk_storage = xchunk_store_manager<xfile_array<T, IOH, L>, IP>;
158-
chunk_storage chunks(shape, chunk_shape, path, pool_size, chunk_memory_layout);
159-
return xchunked_array<chunk_storage, EXT>(std::move(chunks), std::forward<S>(shape), std::forward<S>(chunk_shape));
160-
}
161-
162-
template <class IOH, layout_type L, class IP, class EXT, class E, class S>
163-
inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
164-
chunked_file_array(const xexpression<E>& e, S&& chunk_shape, const std::string& path, std::size_t pool_size, char chunk_memory_layout)
165-
{
166-
using chunk_storage = xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>;
167-
chunk_storage chunks(e.derived_cast().shape(), chunk_shape, path, pool_size, chunk_memory_layout);
168-
return xchunked_array<chunk_storage, EXT>(e, chunk_storage(), std::forward<S>(chunk_shape));
169-
}
170-
171-
template <class IOH, layout_type L, class IP, class EXT, class E>
172-
inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
173-
chunked_file_array(const xexpression<E>& e, const std::string& path, std::size_t pool_size, char chunk_memory_layout)
174-
{
175-
using chunk_storage = xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>;
176-
chunk_storage chunks(e.derived_cast().shape(), detail::chunk_helper<E>::chunk_shape(e), path, pool_size, chunk_memory_layout);
177-
return xchunked_array<chunk_storage, EXT>(e, chunk_storage());
178-
}
188+
chunked_file_array(const xexpression<E>& e,
189+
const std::string& path,
190+
std::size_t pool_size = 1,
191+
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
179192

180193
/******************************
181194
* xindex_path implementation *
182195
******************************/
183196

197+
inline std::string xindex_path::get_directory() const
198+
{
199+
return m_directory;
200+
}
201+
184202
inline void xindex_path::set_directory(const std::string& directory)
185203
{
186204
m_directory = directory;
@@ -191,7 +209,7 @@ namespace xt
191209
}
192210

193211
template <class I>
194-
void xindex_path::index_to_path(I first, I last, std::string& path)
212+
void xindex_path::index_to_path(I first, I last, std::string& path) const
195213
{
196214
std::string fname;
197215
for (auto it = first; it != last; ++it)
@@ -205,6 +223,53 @@ namespace xt
205223
path = m_directory + fname;
206224
}
207225

226+
/************************************
227+
* xchunked_assigner implementation *
228+
************************************/
229+
230+
template <class T, class EC, class IP>
231+
template <class E, class DST>
232+
inline void xchunked_assigner<T, xchunk_store_manager<EC, IP>>::build_and_assign_temporary(const xexpression<E>& e,
233+
DST& dst)
234+
{
235+
using store_type = xchunk_store_manager<EC, IP>;
236+
store_type store(e.derived_cast().shape(), dst.chunk_shape(), dst.chunks().get_temporary_directory(), dst.chunks().get_pool_size());
237+
temporary_type tmp(e, std::move(store), dst.chunk_shape());
238+
tmp.chunks().flush();
239+
dst.chunks().reset_to_directory(tmp.chunks().get_directory());
240+
}
241+
242+
/*******************************************
243+
* xchunke_store_manager factory functions *
244+
*******************************************/
245+
246+
template <class T, class IOH, layout_type L, class IP, class EXT, class S>
247+
inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>, EXT>
248+
chunked_file_array(S&& shape, S&& chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout)
249+
{
250+
using chunk_storage = xchunk_store_manager<xfile_array<T, IOH, L>, IP>;
251+
chunk_storage chunks(shape, chunk_shape, path, pool_size, chunk_memory_layout);
252+
return xchunked_array<chunk_storage, EXT>(std::move(chunks), std::forward<S>(shape), std::forward<S>(chunk_shape));
253+
}
254+
255+
template <class IOH, layout_type L, class IP, class EXT, class E, class S>
256+
inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
257+
chunked_file_array(const xexpression<E>& e, S&& chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout)
258+
{
259+
using chunk_storage = xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>;
260+
chunk_storage chunks(e.derived_cast().shape(), chunk_shape, path, pool_size, chunk_memory_layout);
261+
return xchunked_array<chunk_storage, EXT>(e, chunk_storage(), std::forward<S>(chunk_shape));
262+
}
263+
264+
template <class IOH, layout_type L, class IP, class EXT, class E>
265+
inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
266+
chunked_file_array(const xexpression<E>& e, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout)
267+
{
268+
using chunk_storage = xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>;
269+
chunk_storage chunks(e.derived_cast().shape(), detail::chunk_helper<E>::chunk_shape(e), path, pool_size, chunk_memory_layout);
270+
return xchunked_array<chunk_storage, EXT>(e, chunk_storage());
271+
}
272+
208273
/***************************************
209274
* xchunk_store_manager implementation *
210275
***************************************/
@@ -219,7 +284,7 @@ namespace xt
219284
: m_shape(shape)
220285
, m_unload_index(0u)
221286
{
222-
if (pool_size == -1)
287+
if (pool_size == SIZE_MAX)
223288
{
224289
// as many "physical" chunks in the pool as there are "logical" chunks
225290
pool_size = size();
@@ -313,11 +378,23 @@ namespace xt
313378
}
314379

315380
template <class EC, class IP>
316-
inline std::size_t xchunk_store_manager<EC, IP>::size()
381+
inline auto xchunk_store_manager<EC, IP>::size() const -> size_type
317382
{
318383
return compute_size(m_shape);
319384
}
320385

386+
template <class EC, class IP>
387+
inline const std::string& xchunk_store_manager<EC, IP>::get_directory() const
388+
{
389+
return m_index_path.get_directory();
390+
}
391+
392+
template <class EC, class IP>
393+
inline bool xchunk_store_manager<EC, IP>::get_pool_size() const
394+
{
395+
return m_chunk_pool.size();
396+
}
397+
321398
template <class EC, class IP>
322399
inline void xchunk_store_manager<EC, IP>::flush()
323400
{
@@ -393,6 +470,32 @@ namespace xt
393470
return const_cast<xchunk_store_manager<EC, IP>*>(this)->map_file_array(first, last);
394471
}
395472

473+
template <class EC, class IP>
474+
inline std::string xchunk_store_manager<EC, IP>::get_temporary_directory() const
475+
{
476+
namespace fs = ghc::filesystem;
477+
fs::path tmp_dir = fs::temp_directory_path();
478+
std::size_t count = 0;
479+
while(fs::exists(tmp_dir / std::to_string(count)))
480+
{
481+
++count;
482+
}
483+
return tmp_dir / std::to_string(count);
484+
}
485+
486+
template <class EC, class IP>
487+
inline void xchunk_store_manager<EC, IP>::reset_to_directory(const std::string& directory)
488+
{
489+
namespace fs = ghc::filesystem;
490+
fs::remove_all(get_directory());
491+
fs::rename(directory, get_directory());
492+
m_unload_index = 0u;
493+
for (auto& chunk: m_chunk_pool)
494+
{
495+
chunk.ignore_empty_path(true);
496+
}
497+
}
498+
396499
template <class EC, class IP>
397500
template <class... Idxs>
398501
inline std::array<std::size_t, sizeof...(Idxs)>

‎test/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ set(XTENSOR_IO_TESTS
8989
set(XTENSOR_IO_HO_TESTS
9090
main.cpp
9191
test_xchunk_store_manager.cpp
92+
test_xfile_array.cpp
9293
)
9394

9495
# Add files for tests

0 commit comments

Comments
 (0)