Skip to content

Commit 33f50e7

Browse files
JohanMabilleJohanMabille
authored
Merge pull request #106 from davidbrochart/flush_ioh
Move flushing responsibility to format handler
2 parents f97c1b5 + 655fe6e commit 33f50e7

File tree

7 files changed

+95
-27
lines changed

7 files changed

+95
-27
lines changed

‎include/xtensor-io/xfile_array.hpp‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,23 @@ namespace xt
1212
{
1313
enum class xfile_mode { load, init, init_on_fail };
1414

15+
struct xfile_dirty
16+
{
17+
bool data_dirty;
18+
bool shape_dirty;
19+
20+
xfile_dirty(bool is_dirty=false)
21+
{
22+
data_dirty = is_dirty;
23+
shape_dirty = is_dirty;
24+
}
25+
26+
operator bool() const
27+
{
28+
return data_dirty || shape_dirty;
29+
}
30+
};
31+
1532
template <class T>
1633
class xfile_value_reference
1734
{
@@ -20,7 +37,7 @@ namespace xt
2037
using self_type = xfile_value_reference<T>;
2138
using const_reference = const T&;
2239

23-
xfile_value_reference(T& value, bool& dirty);
40+
xfile_value_reference(T& value, xfile_dirty& dirty);
2441
~xfile_value_reference() = default;
2542

2643
xfile_value_reference(const xfile_value_reference&) = default;
@@ -49,7 +66,7 @@ namespace xt
4966
private:
5067

5168
T& m_value;
52-
bool& m_dirty;
69+
xfile_dirty& m_dirty;
5370
};
5471
}
5572

@@ -205,7 +222,7 @@ namespace xt
205222
private:
206223

207224
E m_storage;
208-
bool m_dirty;
225+
xfile_dirty m_dirty;
209226
IOH m_io_handler;
210227
std::string m_path;
211228
xfile_mode m_file_mode;
@@ -223,7 +240,7 @@ namespace xt
223240
****************************************/
224241

225242
template <class T>
226-
inline xfile_value_reference<T>::xfile_value_reference(T& value, bool& dirty)
243+
inline xfile_value_reference<T>::xfile_value_reference(T& value, xfile_dirty& dirty)
227244
: m_value(value), m_dirty(dirty)
228245
{
229246
}
@@ -235,7 +252,7 @@ namespace xt
235252
if (v != m_value)
236253
{
237254
m_value = v;
238-
m_dirty = true;
255+
m_dirty.data_dirty = true;
239256
}
240257
return *this;
241258
}
@@ -247,7 +264,7 @@ namespace xt
247264
if (v != T(0))
248265
{
249266
m_value += v;
250-
m_dirty = true;
267+
m_dirty.data_dirty = true;
251268
}
252269
return *this;
253270
}
@@ -259,7 +276,7 @@ namespace xt
259276
if (v != T(0))
260277
{
261278
m_value -= v;
262-
m_dirty = true;
279+
m_dirty.data_dirty = true;
263280
}
264281
return *this;
265282
}
@@ -271,7 +288,7 @@ namespace xt
271288
if (v != T(1))
272289
{
273290
m_value *= v;
274-
m_dirty = true;
291+
m_dirty.data_dirty = true;
275292
}
276293
return *this;
277294
}
@@ -283,7 +300,7 @@ namespace xt
283300
if (v != T(1))
284301
{
285302
m_value /= v;
286-
m_dirty = true;
303+
m_dirty.data_dirty = true;
287304
}
288305
return *this;
289306
}
@@ -411,27 +428,31 @@ namespace xt
411428
inline void xfile_array_container<E, IOH>::resize(S&& shape, bool force)
412429
{
413430
m_storage.resize(std::forward<S>(shape), force);
431+
m_dirty.shape_dirty = true;
414432
}
415433

416434
template <class E, class IOH>
417435
template <class S>
418436
inline void xfile_array_container<E, IOH>::resize(S&& shape, layout_type l)
419437
{
420438
m_storage.resize(std::forward<S>(shape), l);
439+
m_dirty.shape_dirty = true;
421440
}
422441

423442
template <class E, class IOH>
424443
template <class S>
425444
inline void xfile_array_container<E, IOH>::resize(S&& shape, const strides_type& strides)
426445
{
427446
m_storage.resize(std::forward<S>(shape), strides);
447+
m_dirty.shape_dirty = true;
428448
}
429449

430450
template <class E, class IOH>
431451
template <class S>
432452
inline auto xfile_array_container<E, IOH>::reshape(S&& shape, layout_type layout) & -> self_type&
433453
{
434454
m_storage.reshape(std::forward<S>(shape), layout);
455+
m_dirty.shape_dirty = true;
435456
return *this;
436457
}
437458

@@ -440,6 +461,7 @@ namespace xt
440461
inline auto xfile_array_container<E, IOH>::reshape(std::initializer_list<T> shape, layout_type layout) & -> self_type&
441462
{
442463
m_storage.reshape(shape, layout);
464+
m_dirty.shape_dirty = true;
443465
return *this;
444466
}
445467

@@ -601,7 +623,7 @@ namespace xt
601623
{
602624
if (m_dirty)
603625
{
604-
m_io_handler.write(m_storage, m_path);
626+
m_io_handler.write(m_storage, m_path, m_dirty);
605627
m_dirty = false;
606628
}
607629
}

‎include/xtensor-io/xio_binary.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include "xtensor/xadapt.hpp"
1616
#include "xtensor-io.hpp"
17+
#include "xfile_array.hpp"
1718

1819
namespace xt
1920
{
@@ -162,6 +163,11 @@ namespace xt
162163
void read_from(T& j)
163164
{
164165
}
166+
167+
bool will_dump(xfile_dirty dirty)
168+
{
169+
return dirty.data_dirty;
170+
}
165171
};
166172

167173
template <class E>

‎include/xtensor-io/xio_blosc.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include "xtensor/xadapt.hpp"
1616
#include "xtensor-io.hpp"
17+
#include "xfile_array.hpp"
1718
#include "blosc.h"
1819

1920
namespace xt
@@ -222,6 +223,11 @@ namespace xt
222223
cname = j["cname"];
223224
blocksize = j["blocksize"];
224225
}
226+
227+
bool will_dump(xfile_dirty dirty)
228+
{
229+
return dirty.data_dirty;
230+
}
225231
};
226232

227233
template <class E>

‎include/xtensor-io/xio_disk_handler.hpp‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ namespace xt
1212
public:
1313

1414
template <class E>
15-
void write(const xexpression<E>& expression, const std::string& path) const;
15+
void write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty);
1616

1717
template <class ET>
1818
void read(ET& array, const std::string& path) const;
@@ -26,16 +26,19 @@ namespace xt
2626

2727
template <class C>
2828
template <class E>
29-
inline void xio_disk_handler<C>::write(const xexpression<E>& expression, const std::string& path) const
29+
inline void xio_disk_handler<C>::write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty)
3030
{
31-
std::ofstream out_file(path, std::ofstream::binary);
32-
if (out_file.is_open())
31+
if (m_format_config.will_dump(dirty))
3332
{
34-
dump_file(out_file, expression, m_format_config);
35-
}
36-
else
37-
{
38-
XTENSOR_THROW(std::runtime_error, "write: failed to open file " + path);
33+
std::ofstream out_file(path, std::ofstream::binary);
34+
if (out_file.is_open())
35+
{
36+
dump_file(out_file, expression, m_format_config);
37+
}
38+
else
39+
{
40+
XTENSOR_THROW(std::runtime_error, "write: failed to open file " + path);
41+
}
3942
}
4043
}
4144

‎include/xtensor-io/xio_gcs_handler.hpp‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ namespace xt
1717
xio_gcs_handler();
1818

1919
template <class E>
20-
void write(const xexpression<E>& expression, const std::string& path) const;
20+
void write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty);
2121

2222
template <class ET>
2323
void read(ET& array, const std::string& path) const;
@@ -37,14 +37,17 @@ namespace xt
3737

3838
template <class C>
3939
template <class E>
40-
inline void xio_gcs_handler<C>::write(const xexpression<E>& expression, const std::string& path) const
40+
inline void xio_gcs_handler<C>::write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty)
4141
{
42-
std::string bucket_name;
43-
std::string file_path;
44-
split_bucket_path(path, bucket_name, file_path);
45-
gcs::Client client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials())));
46-
auto writer = client.WriteObject(bucket_name, file_path);
47-
dump_file(writer, expression, m_format_config);
42+
if (m_format_config.will_dump(dirty))
43+
{
44+
std::string bucket_name;
45+
std::string file_path;
46+
split_bucket_path(path, bucket_name, file_path);
47+
gcs::Client client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials())));
48+
auto writer = client.WriteObject(bucket_name, file_path);
49+
dump_file(writer, expression, m_format_config);
50+
}
4851
}
4952

5053
template <class C>

‎include/xtensor-io/xio_gzip.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#include "xtensor/xadapt.hpp"
1818
#include "xtensor-io.hpp"
19+
#include "xfile_array.hpp"
1920

2021
#define GZIP_CHUNK 0x4000
2122
#define GZIP_WINDOWBITS 15
@@ -237,6 +238,11 @@ namespace xt
237238
{
238239
level = j["level"];
239240
}
241+
242+
bool will_dump(xfile_dirty dirty)
243+
{
244+
return dirty.data_dirty;
245+
}
240246
};
241247

242248
template <class E>

‎test/test_xfile_array.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,26 @@ namespace xt
6060
EXPECT_TRUE(xt::all(xt::equal(data, ref)));
6161
in_file.close();
6262
}
63+
64+
TEST(xfile_array, flush)
65+
{
66+
std::vector<std::size_t> shape = {2, 2};
67+
auto a1 = xfile_array<double, xio_disk_handler<xio_binary_config>>(broadcast(5., shape), "a1");
68+
a1.flush();
69+
auto a2 = xfile_array<double, xio_disk_handler<xio_binary_config>>("");
70+
// point a2 to a1's file
71+
a2.set_path("a1");
72+
// they should have the same size
73+
EXPECT_EQ(a2.size(), compute_size(shape));
74+
// resize a2
75+
std::vector<std::size_t> new_shape = {3, 3};
76+
a2.resize(new_shape);
77+
// should not be flushed, as binary format only stores data (not shape)
78+
a2.flush();
79+
auto a3 = xfile_array<double, xio_disk_handler<xio_binary_config>>("");
80+
// point a3 to a1's file (which is also a2's file)
81+
a3.set_path("a1");
82+
// shape should not have been changed
83+
EXPECT_EQ(a3.size(), compute_size(shape));
84+
}
6385
}

0 commit comments

Comments
 (0)