Skip to content

Commit 8a5e5a5

Browse files
davidbrochartdavidbrochart
authored
Allow xfile_array initialization with value (#107)
Allow xfile_array initialization with value
1 parent 33f50e7 commit 8a5e5a5

File tree

3 files changed

+128
-9
lines changed

3 files changed

+128
-9
lines changed

‎include/xtensor-io/xchunk_store_manager.hpp‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,15 @@ namespace xt
9898
const std::string& directory,
9999
std::size_t pool_size,
100100
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
101+
102+
template <class S, class T>
103+
xchunk_store_manager(S&& shape,
104+
S&& chunk_shape,
105+
const std::string& directory,
106+
std::size_t pool_size,
107+
const T& init_value,
108+
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
109+
101110
~xchunk_store_manager() = default;
102111

103112
xchunk_store_manager(const xchunk_store_manager&) = default;
@@ -157,6 +166,15 @@ namespace xt
157166
template <class... Idxs>
158167
std::array<std::size_t, sizeof...(Idxs)> get_indexes(Idxs... idxs) const;
159168

169+
template <class S, class T>
170+
void initialize(S&& shape,
171+
S&& chunk_shape,
172+
const std::string& directory,
173+
bool init,
174+
const T& init_value,
175+
std::size_t pool_size,
176+
layout_type chunk_memory_layout);
177+
160178
using chunk_pool_type = std::vector<EC>;
161179
using index_pool_type = std::vector<shape_type>;
162180

@@ -175,6 +193,15 @@ namespace xt
175193
std::size_t pool_size = 1,
176194
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
177195

196+
template <class T, class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class EXT = empty_extension, class S>
197+
xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>, EXT>
198+
chunked_file_array(S&& shape,
199+
S&& chunk_shape,
200+
const std::string& path,
201+
const T& init_value,
202+
std::size_t pool_size = 1,
203+
layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT);
204+
178205
template <class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class EXT = empty_extension, class E, class S>
179206
xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
180207
chunked_file_array(const xexpression<E>& e,
@@ -252,6 +279,15 @@ namespace xt
252279
return xchunked_array<chunk_storage, EXT>(std::move(chunks), std::forward<S>(shape), std::forward<S>(chunk_shape));
253280
}
254281

282+
template <class T, class IOH, layout_type L, class IP, class EXT, class S>
283+
inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>, EXT>
284+
chunked_file_array(S&& shape, S&& chunk_shape, const std::string& path, const T& init_value,std::size_t pool_size, layout_type chunk_memory_layout)
285+
{
286+
using chunk_storage = xchunk_store_manager<xfile_array<T, IOH, L>, IP>;
287+
chunk_storage chunks(shape, chunk_shape, path, pool_size, init_value, chunk_memory_layout);
288+
return xchunked_array<chunk_storage, EXT>(std::move(chunks), std::forward<S>(shape), std::forward<S>(chunk_shape));
289+
}
290+
255291
template <class IOH, layout_type L, class IP, class EXT, class E, class S>
256292
inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>, EXT>
257293
chunked_file_array(const xexpression<E>& e, S&& chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout)
@@ -283,13 +319,47 @@ namespace xt
283319
layout_type chunk_memory_layout)
284320
: m_shape(shape)
285321
, m_unload_index(0u)
322+
{
323+
initialize(shape, chunk_shape, directory, false, 0, pool_size, chunk_memory_layout);
324+
}
325+
326+
template <class EC, class IP>
327+
template <class S, class T>
328+
inline xchunk_store_manager<EC, IP>::xchunk_store_manager(S&& shape,
329+
S&& chunk_shape,
330+
const std::string& directory,
331+
std::size_t pool_size,
332+
const T& init_value,
333+
layout_type chunk_memory_layout)
334+
: m_shape(shape)
335+
, m_unload_index(0u)
336+
{
337+
initialize(shape, chunk_shape, directory, true, init_value, pool_size, chunk_memory_layout);
338+
}
339+
340+
template <class EC, class IP>
341+
template <class S, class T>
342+
inline void xchunk_store_manager<EC, IP>::initialize(S&& shape,
343+
S&& chunk_shape,
344+
const std::string& directory,
345+
bool init,
346+
const T& init_value,
347+
std::size_t pool_size,
348+
layout_type chunk_memory_layout)
286349
{
287350
if (pool_size == SIZE_MAX)
288351
{
289352
// as many "physical" chunks in the pool as there are "logical" chunks
290353
pool_size = size();
291354
}
292-
m_chunk_pool.resize(pool_size, EC("", xfile_mode::init_on_fail));
355+
if (init)
356+
{
357+
m_chunk_pool.resize(pool_size, EC("", xfile_mode::init_on_fail, init_value));
358+
}
359+
else
360+
{
361+
m_chunk_pool.resize(pool_size, EC("", xfile_mode::init_on_fail));
362+
}
293363
m_index_pool.resize(pool_size);
294364
// resize the pool chunks
295365
for (auto& chunk: m_chunk_pool)

‎include/xtensor-io/xfile_array.hpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,9 @@ namespace xt
132132
static constexpr bool contiguous_layout = true;
133133

134134
xfile_array_container(const std::string& path, xfile_mode mode=xfile_mode::load);
135+
136+
xfile_array_container(const std::string& path, xfile_mode mode, const value_type& init_value);
137+
135138
~xfile_array_container();
136139

137140
xfile_array_container(const self_type&) = default;
@@ -226,6 +229,8 @@ namespace xt
226229
IOH m_io_handler;
227230
std::string m_path;
228231
xfile_mode m_file_mode;
232+
value_type m_init_value;
233+
bool m_init;
229234
};
230235

231236
template <class T,
@@ -360,6 +365,19 @@ namespace xt
360365
, m_dirty(false)
361366
, m_io_handler()
362367
, m_file_mode(file_mode)
368+
, m_init(false)
369+
{
370+
set_path(path);
371+
}
372+
373+
template <class E, class IOH>
374+
inline xfile_array_container<E, IOH>::xfile_array_container(const std::string& path, xfile_mode file_mode, const value_type& init_value)
375+
: m_storage()
376+
, m_dirty(false)
377+
, m_io_handler()
378+
, m_file_mode(file_mode)
379+
, m_init_value(init_value)
380+
, m_init(true)
363381
{
364382
set_path(path);
365383
}
@@ -378,6 +396,7 @@ namespace xt
378396
, m_io_handler()
379397
, m_path(detail::file_helper<E>::path(e))
380398
, m_file_mode(xfile_mode::init)
399+
, m_init(false)
381400
{
382401
}
383402

@@ -389,6 +408,7 @@ namespace xt
389408
, m_io_handler()
390409
, m_path(path)
391410
, m_file_mode(xfile_mode::init)
411+
, m_init(false)
392412
{
393413
}
394414

@@ -613,6 +633,11 @@ namespace xt
613633
{
614634
throw e;
615635
}
636+
// m_file_mode == xfile_mode::init_on_fail
637+
if (m_init)
638+
{
639+
std::fill(m_storage.begin(), m_storage.end(), m_init_value);
640+
}
616641
}
617642
}
618643
}

‎test/test_xchunk_store_manager.cpp‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,28 @@ namespace xt
2323
inline auto make_test_chunked_array(const S& shape,
2424
const S& chunk_shape,
2525
const std::string& chunk_dir,
26-
size_t pool_size)
26+
size_t pool_size,
27+
bool init=false,
28+
double init_value=0)
2729
{
28-
return chunked_file_array<double, xio_disk_handler<xio_binary_config>>(shape, chunk_shape, chunk_dir, pool_size);
30+
if (init)
31+
{
32+
return chunked_file_array<double, xio_disk_handler<xio_binary_config>>(shape, chunk_shape, chunk_dir, init_value, pool_size);
33+
}
34+
else
35+
{
36+
return chunked_file_array<double, xio_disk_handler<xio_binary_config>>(shape, chunk_shape, chunk_dir, pool_size);
37+
}
2938
}
3039

3140
TEST(xchunked_array, disk_array)
3241
{
3342
std::vector<size_t> shape = {4, 4};
3443
std::vector<size_t> chunk_shape = {2, 2};
35-
std::string chunk_dir0 = "files0";
36-
fs::create_directory(chunk_dir0);
44+
std::string chunk_dir = "files0";
45+
fs::create_directory(chunk_dir);
3746
std::size_t pool_size = 2;
38-
auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir0, pool_size);
47+
auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir, pool_size);
3948
std::vector<size_t> idx = {1, 2};
4049
double v1 = 3.4;
4150
double v2 = 5.6;
@@ -49,18 +58,18 @@ namespace xt
4958

5059
std::ifstream in_file;
5160
xt::xarray<double> data;
52-
in_file.open(chunk_dir0 + "/1.0");
61+
in_file.open(chunk_dir + "/1.0");
5362
data = xt::load_bin<double>(in_file);
5463
EXPECT_EQ(data(1), v1);
5564
in_file.close();
5665

5766
a1.chunks().flush();
58-
in_file.open(chunk_dir0 + "/0.1");
67+
in_file.open(chunk_dir + "/0.1");
5968
data = xt::load_bin<double>(in_file);
6069
EXPECT_EQ(data(2), v2);
6170
in_file.close();
6271

63-
in_file.open(chunk_dir0 + "/0.0");
72+
in_file.open(chunk_dir + "/0.0");
6473
data = xt::load_bin<double>(in_file);
6574
EXPECT_EQ(data(0), v3);
6675
in_file.close();
@@ -97,4 +106,19 @@ namespace xt
97106
EXPECT_EQ(a2.begin()[i], v1[i]);
98107
}
99108
}
109+
110+
TEST(xchunked_array, init_value)
111+
{
112+
std::vector<size_t> shape = {4, 4};
113+
std::vector<size_t> chunk_shape = {2, 2};
114+
std::string chunk_dir = "files3";
115+
fs::create_directory(chunk_dir);
116+
std::size_t pool_size = 1;
117+
double init_value = 5.5;
118+
auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir, pool_size, true, init_value);
119+
for (auto v: a1)
120+
{
121+
EXPECT_EQ(v, init_value);
122+
}
123+
}
100124
}

0 commit comments

Comments
 (0)