Skip to content

Commit f97c1b5

Browse files
davidbrochartdavidbrochart
authored
Add xfile_mode (#105)
Add xfile_mode
1 parent 763d3cf commit f97c1b5

File tree

6 files changed

+63
-79
lines changed

6 files changed

+63
-79
lines changed

‎include/xtensor-io/xchunk_store_manager.hpp‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ namespace xt
2222
class xindex_path
2323
{
2424
public:
25-
25+
2626
std::string get_directory() const;
2727
void set_directory(const std::string& directory);
2828

@@ -239,9 +239,9 @@ namespace xt
239239
dst.chunks().reset_to_directory(tmp.chunks().get_directory());
240240
}
241241

242-
/*******************************************
243-
* xchunke_store_manager factory functions *
244-
*******************************************/
242+
/******************************************
243+
* xchunk_store_manager factory functions *
244+
******************************************/
245245

246246
template <class T, class IOH, layout_type L, class IP, class EXT, class S>
247247
inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>, EXT>
@@ -289,13 +289,12 @@ namespace xt
289289
// as many "physical" chunks in the pool as there are "logical" chunks
290290
pool_size = size();
291291
}
292-
m_chunk_pool.resize(pool_size);
292+
m_chunk_pool.resize(pool_size, EC("", xfile_mode::init_on_fail));
293293
m_index_pool.resize(pool_size);
294294
// resize the pool chunks
295295
for (auto& chunk: m_chunk_pool)
296296
{
297297
chunk.resize(chunk_shape, chunk_memory_layout);
298-
chunk.ignore_empty_path(true);
299298
}
300299
m_index_path.set_directory(directory);
301300
}
@@ -490,10 +489,6 @@ namespace xt
490489
fs::remove_all(get_directory());
491490
fs::rename(directory, get_directory());
492491
m_unload_index = 0u;
493-
for (auto& chunk: m_chunk_pool)
494-
{
495-
chunk.ignore_empty_path(true);
496-
}
497492
}
498493

499494
template <class EC, class IP>

‎include/xtensor-io/xfile_array.hpp‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
namespace xt
1212
{
13+
enum class xfile_mode { load, init, init_on_fail };
1314

1415
template <class T>
1516
class xfile_value_reference
@@ -113,7 +114,7 @@ namespace xt
113114
static constexpr layout_type static_layout = layout_type::dynamic;
114115
static constexpr bool contiguous_layout = true;
115116

116-
xfile_array_container() = default;
117+
xfile_array_container(const std::string& path, xfile_mode mode=xfile_mode::load);
117118
~xfile_array_container();
118119

119120
xfile_array_container(const self_type&) = default;
@@ -194,7 +195,6 @@ namespace xt
194195
load_simd(size_type i) const;
195196

196197
const std::string& path() const noexcept;
197-
void ignore_empty_path(bool ignore);
198198
void set_path(const std::string& path);
199199

200200
template <class C>
@@ -204,13 +204,11 @@ namespace xt
204204

205205
private:
206206

207-
bool enable_io(const std::string& path) const;
208-
209207
E m_storage;
210208
bool m_dirty;
211209
IOH m_io_handler;
212210
std::string m_path;
213-
bool m_ignore_empty_path;
211+
xfile_mode m_file_mode;
214212
};
215213

216214
template <class T,
@@ -339,6 +337,16 @@ namespace xt
339337
return return_type::value;
340338
}
341339

340+
template <class E, class IOH>
341+
inline xfile_array_container<E, IOH>::xfile_array_container(const std::string& path, xfile_mode file_mode)
342+
: m_storage()
343+
, m_dirty(false)
344+
, m_io_handler()
345+
, m_file_mode(file_mode)
346+
{
347+
set_path(path);
348+
}
349+
342350
template <class E, class IOH>
343351
inline xfile_array_container<E, IOH>::~xfile_array_container()
344352
{
@@ -352,7 +360,7 @@ namespace xt
352360
, m_dirty(true)
353361
, m_io_handler()
354362
, m_path(detail::file_helper<E>::path(e))
355-
, m_ignore_empty_path(false)
363+
, m_file_mode(xfile_mode::init)
356364
{
357365
}
358366

@@ -363,7 +371,7 @@ namespace xt
363371
, m_dirty(true)
364372
, m_io_handler()
365373
, m_path(path)
366-
, m_ignore_empty_path(false)
374+
, m_file_mode(xfile_mode::init)
367375
{
368376
}
369377

@@ -562,18 +570,6 @@ namespace xt
562570
m_io_handler.configure_format(config);
563571
}
564572

565-
template <class E, class IOH>
566-
inline void xfile_array_container<E, IOH>::ignore_empty_path(bool ignore)
567-
{
568-
m_ignore_empty_path = ignore;
569-
}
570-
571-
template <class E, class IOH>
572-
inline bool xfile_array_container<E, IOH>::enable_io(const std::string& path) const
573-
{
574-
return !path.empty() || !m_ignore_empty_path;
575-
}
576-
577573
template <class E, class IOH>
578574
inline void xfile_array_container<E, IOH>::set_path(const std::string& path)
579575
{
@@ -582,10 +578,20 @@ namespace xt
582578
// maybe write to old file
583579
flush();
584580
m_path = path;
585-
// read new file
586-
if (enable_io(path))
581+
if (m_file_mode != xfile_mode::init)
587582
{
588-
m_io_handler.read(m_storage, path);
583+
// read new file
584+
try
585+
{
586+
m_io_handler.read(m_storage, path);
587+
}
588+
catch (const std::runtime_error& e)
589+
{
590+
if (m_file_mode == xfile_mode::load)
591+
{
592+
throw e;
593+
}
594+
}
589595
}
590596
}
591597
}
@@ -595,10 +601,7 @@ namespace xt
595601
{
596602
if (m_dirty)
597603
{
598-
if (enable_io(m_path))
599-
{
600-
m_io_handler.write(m_storage, m_path);
601-
}
604+
m_io_handler.write(m_storage, m_path);
602605
m_dirty = false;
603606
}
604607
}

‎include/xtensor-io/xio_disk_handler.hpp‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ namespace xt
1515
void write(const xexpression<E>& expression, const std::string& path) const;
1616

1717
template <class ET>
18-
void read(ET& array, const std::string& path, bool throw_on_fail = false) const;
18+
void read(ET& array, const std::string& path) const;
1919

2020
void configure_format(const C& format_config);
2121

@@ -41,7 +41,7 @@ namespace xt
4141

4242
template <class C>
4343
template <class ET>
44-
inline void xio_disk_handler<C>::read(ET& array, const std::string& path, bool throw_on_fail) const
44+
inline void xio_disk_handler<C>::read(ET& array, const std::string& path) const
4545
{
4646
std::ifstream in_file(path, std::ifstream::binary);
4747
if (in_file.is_open())
@@ -50,15 +50,7 @@ namespace xt
5050
}
5151
else
5252
{
53-
if (throw_on_fail)
54-
{
55-
XTENSOR_THROW(std::runtime_error, "read: failed to open file " + path);
56-
}
57-
else
58-
{
59-
auto shape = array.shape();
60-
array = zeros<typename ET::value_type>(shape);
61-
}
53+
XTENSOR_THROW(std::runtime_error, "read: failed to open file " + path);
6254
}
6355
}
6456

‎include/xtensor-io/xio_gcs_handler.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ namespace xt
2020
void write(const xexpression<E>& expression, const std::string& path) const;
2121

2222
template <class ET>
23-
void read(ET& array, const std::string& path, bool throw_on_fail = false) const;
23+
void read(ET& array, const std::string& path) const;
2424

2525
void configure_format(const C& format_config);
2626
void split_bucket_path(const std::string& path, std::string& bucket_name, std::string& file_path) const;
@@ -49,7 +49,7 @@ namespace xt
4949

5050
template <class C>
5151
template <class ET>
52-
inline void xio_gcs_handler<C>::read(ET& array, const std::string& path, bool throw_on_fail) const
52+
inline void xio_gcs_handler<C>::read(ET& array, const std::string& path) const
5353
{
5454
std::string bucket_name;
5555
std::string file_path;

‎test/test_xchunk_store_manager.cpp‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
#include "gtest/gtest.h"
1111

1212
#include <xtensor/xbroadcast.hpp>
13-
#include <xtensor/xcsv.hpp>
1413
#include "xtensor-io/xchunk_store_manager.hpp"
1514
#include "xtensor-io/xfile_array.hpp"
15+
#include <xtensor-io/xio_binary.hpp>
1616
#include "xtensor-io/xio_disk_handler.hpp"
1717

1818
namespace xt
@@ -25,16 +25,17 @@ namespace xt
2525
const std::string& chunk_dir,
2626
size_t pool_size)
2727
{
28-
return chunked_file_array<double, xio_disk_handler<xcsv_config>>(shape, chunk_shape, chunk_dir, pool_size);
28+
return chunked_file_array<double, xio_disk_handler<xio_binary_config>>(shape, chunk_shape, chunk_dir, pool_size);
2929
}
3030

3131
TEST(xchunked_array, disk_array)
3232
{
3333
std::vector<size_t> shape = {4, 4};
3434
std::vector<size_t> chunk_shape = {2, 2};
35-
std::string chunk_dir = "files";
35+
std::string chunk_dir0 = "files0";
36+
fs::create_directory(chunk_dir0);
3637
std::size_t pool_size = 2;
37-
auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir, pool_size);
38+
auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir0, pool_size);
3839
std::vector<size_t> idx = {1, 2};
3940
double v1 = 3.4;
4041
double v2 = 5.6;
@@ -47,25 +48,21 @@ namespace xt
4748
EXPECT_EQ(a1(0, 0), v3);
4849

4950
std::ifstream in_file;
50-
xt::xarray<double> ref;
5151
xt::xarray<double> data;
52-
in_file.open(chunk_dir + "/1.0");
53-
data = xt::load_csv<double>(in_file);
54-
ref = {{0, v1}, {0, 0}};
55-
EXPECT_EQ(data, ref);
52+
in_file.open(chunk_dir0 + "/1.0");
53+
data = xt::load_bin<double>(in_file);
54+
EXPECT_EQ(data(1), v1);
5655
in_file.close();
5756

5857
a1.chunks().flush();
59-
in_file.open(chunk_dir + "/0.1");
60-
data = xt::load_csv<double>(in_file);
61-
ref = {{0, 0}, {v2, 0}};
62-
EXPECT_EQ(data, ref);
58+
in_file.open(chunk_dir0 + "/0.1");
59+
data = xt::load_bin<double>(in_file);
60+
EXPECT_EQ(data(2), v2);
6361
in_file.close();
6462

65-
in_file.open(chunk_dir + "/0.0");
66-
data = xt::load_csv<double>(in_file);
67-
ref = {{v3, 0}, {0, 0}};
68-
EXPECT_EQ(data, ref);
63+
in_file.open(chunk_dir0 + "/0.0");
64+
data = xt::load_bin<double>(in_file);
65+
EXPECT_EQ(data(0), v3);
6966
in_file.close();
7067
}
7168

‎test/test_xfile_array.cpp‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,16 @@
1010
#include "gtest/gtest.h"
1111

1212
#include <xtensor/xbroadcast.hpp>
13-
#include <xtensor/xcsv.hpp>
1413
#include "xtensor-io/xfile_array.hpp"
14+
#include <xtensor-io/xio_binary.hpp>
1515
#include "xtensor-io/xio_disk_handler.hpp"
1616

1717
namespace xt
1818
{
1919
TEST(xfile_array, indexed_access)
2020
{
2121
std::vector<size_t> shape = {2, 2, 2};
22-
xfile_array<double, xio_disk_handler<xcsv_config>> a;
23-
a.ignore_empty_path(true);
22+
xfile_array<double, xio_disk_handler<xio_binary_config>> a("a", xfile_mode::init);
2423
a.resize(shape);
2524
double val = 3.;
2625
for (auto it: a)
@@ -32,16 +31,14 @@ namespace xt
3231
TEST(xfile_array, assign_expression)
3332
{
3433
double v1 = 3.;
35-
auto a1 = xfile_array<double, xio_disk_handler<xcsv_config>>(broadcast(v1, {2, 2}), "a1");
36-
a1.ignore_empty_path(true);
34+
auto a1 = xfile_array<double, xio_disk_handler<xio_binary_config>>(broadcast(v1, {2, 2}), "a1");
3735
for (const auto& v: a1)
3836
{
3937
EXPECT_EQ(v, v1);
4038
}
4139

4240
double v2 = 2. * v1;
43-
auto a2 = xfile_array<double, xio_disk_handler<xcsv_config>>(a1 + a1, "a2");
44-
a2.ignore_empty_path(true);
41+
auto a2 = xfile_array<double, xio_disk_handler<xio_binary_config>>(a1 + a1, "a2");
4542
for (const auto& v: a2)
4643
{
4744
EXPECT_EQ(v, v2);
@@ -52,15 +49,15 @@ namespace xt
5249

5350
std::ifstream in_file;
5451
in_file.open("a1");
55-
auto data = load_csv<double>(in_file);
56-
xarray<double> ref = {{v1, v1}, {v1, v1}};
57-
EXPECT_EQ(data, ref);
52+
auto data = load_bin<double>(in_file);
53+
xarray<double> ref = {v1, v1, v1, v1};
54+
EXPECT_TRUE(xt::all(xt::equal(data, ref)));
5855
in_file.close();
5956

6057
in_file.open("a2");
61-
data = load_csv<double>(in_file);
62-
ref = {{v2, v2}, {v2, v2}};
63-
EXPECT_EQ(data, ref);
58+
data = load_bin<double>(in_file);
59+
ref = {v2, v2, v2, v2};
60+
EXPECT_TRUE(xt::all(xt::equal(data, ref)));
6461
in_file.close();
6562
}
6663
}

0 commit comments

Comments
 (0)