Skip to content

Commit b0fd657

Browse files
davidbrochartdavidbrochart
committed
Refactor, add file and stream wrapper classes
1 parent 86ec45e commit b0fd657

18 files changed

+406
-240
lines changed

‎CMakeLists.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,9 @@ set(XTENSOR_IO_HEADERS
6464
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gcs_handler.hpp
6565
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gdal_handler.hpp
6666
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_gzip.hpp
67-
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file.hpp
68-
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile.hpp
67+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_file_wrapper.hpp
68+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_vsilfile_wrapper.hpp
69+
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xio_stream_wrapper.hpp
6970
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xnpz.hpp
7071
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor-io.hpp
7172
${XTENSOR_IO_INCLUDE_DIR}/xtensor-io/xtensor_io_config.hpp

‎include/xtensor-io/xio_aws_handler.hpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "xtensor/xarray.hpp"
55
#include "xtensor/xexpression.hpp"
66
#include "xfile_array.hpp"
7+
#include "xio_stream_wrapper.hpp"
78
#include <aws/core/Aws.h>
89
#include <aws/s3/S3Client.h>
910
#include <aws/s3/model/GetObjectRequest.h>
@@ -70,7 +71,8 @@ namespace xt
7071
request.SetKey(path2);
7172

7273
std::shared_ptr<Aws::IOStream> writer = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", path, std::ios_base::in | std::ios_base::binary);
73-
dump_file(*writer, expression, m_format_config);
74+
auto s = xostream_wrapper(*writer);
75+
dump_file(s, expression, m_format_config);
7476

7577
request.SetBody(writer);
7678

@@ -109,7 +111,8 @@ namespace xt
109111
}
110112

111113
auto& reader = outcome.GetResultWithOwnership().GetBody();
112-
load_file<ET>(reader, array, m_format_config);
114+
auto s = xistream_wrapper(reader);
115+
load_file<ET>(s, array, m_format_config);
113116
}
114117

115118
template <class C>

‎include/xtensor-io/xio_binary.hpp‎

Lines changed: 33 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,17 @@
1515
#include "xtensor/xadapt.hpp"
1616
#include "xtensor-io.hpp"
1717
#include "xfile_array.hpp"
18-
#include "xio_file.hpp"
18+
#include "xio_stream_wrapper.hpp"
1919

2020
namespace xt
2121
{
2222
namespace detail
2323
{
24-
// load_bin "overload" for file-like objects
25-
// we check that `fclose` can be called on them!
26-
template<typename T, class I>
27-
auto load_bin_imp(I& file, std::string& buffer)
28-
-> decltype(file.ftell(), void())
29-
{
30-
file.fseek(0, SEEK_END);
31-
std::size_t size = file.ftell();
32-
buffer.resize(size);
33-
file.rewind();
34-
file.fread(&buffer[0], 1, size);
35-
}
36-
37-
// load_bin "overload" for stream-like objects
38-
// we check that they have a `tellg` method!
39-
template<typename T, class I>
40-
auto load_bin_imp(I& stream, std::string& buffer)
41-
-> decltype(stream.tellg(), void())
42-
{
43-
buffer = {std::istreambuf_iterator<char>{stream}, {}};
44-
}
45-
4624
template <typename T, class I>
4725
inline xt::svector<T> load_bin(I& stream, bool as_big_endian)
4826
{
4927
std::string buffer;
50-
load_bin_imp<T>(stream, buffer);
28+
stream.read_all(buffer);
5129
std::size_t uncompressed_size = buffer.size() / sizeof(T);
5230
xt::svector<T> uncompressed_buffer(uncompressed_size);
5331
std::copy((const T*)(buffer.data()), (const T*)(buffer.data()) + uncompressed_size, uncompressed_buffer.begin());
@@ -58,26 +36,6 @@ namespace xt
5836
return uncompressed_buffer;
5937
}
6038

61-
// dump_bin "overload" for file-like objects
62-
// we check that `fclose` can be called on them!
63-
template <class O>
64-
auto dump_bin_imp(O& file, const char* uncompressed_buffer, std::size_t uncompressed_size)
65-
-> decltype(file.ftell(), void())
66-
{
67-
file.fwrite(uncompressed_buffer, 1, uncompressed_size);
68-
file.fflush();
69-
}
70-
71-
// dump_bin "overload" for stream-like objects
72-
// we check that they have a `tellp` method!
73-
template <class O>
74-
auto dump_bin_imp(O& stream, const char* uncompressed_buffer, std::size_t uncompressed_size)
75-
-> decltype(stream.tellp(), void())
76-
{
77-
stream.write(uncompressed_buffer, std::streamsize(uncompressed_size));
78-
stream.flush();
79-
}
80-
8139
template <class O, class E>
8240
inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian)
8341
{
@@ -100,26 +58,28 @@ namespace xt
10058
{
10159
uncompressed_buffer = reinterpret_cast<const char*>(eval_ex.data());
10260
}
103-
dump_bin_imp(stream, uncompressed_buffer, uncompressed_size);
61+
stream.write(uncompressed_buffer, uncompressed_size);
62+
stream.flush();
10463
}
10564
} // namespace detail
10665

107-
template <typename E, class O>
108-
inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
109-
{
110-
detail::dump_bin(stream, e, as_big_endian);
111-
}
112-
11366
/**
11467
* Save xexpression to binary format
11568
*
11669
* @param stream An output stream to which to dump the data
11770
* @param e the xexpression
11871
*/
72+
template <typename E, class O>
73+
inline void dump_bin(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
74+
{
75+
detail::dump_bin(stream, e, as_big_endian);
76+
}
77+
11978
template <typename E>
12079
inline void dump_bin(std::ofstream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian())
12180
{
122-
detail::dump_bin(stream, e, as_big_endian);
81+
auto s = xostream_wrapper(stream);
82+
detail::dump_bin(s, e, as_big_endian);
12383
}
12484

12585
/**
@@ -129,14 +89,21 @@ namespace xt
12989
* @param e the xexpression
13090
*/
13191
template <typename E>
132-
inline void dump_bin(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
92+
inline void dump_bin(const char* filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
13393
{
13494
std::ofstream stream(filename, std::ofstream::binary);
13595
if (!stream.is_open())
13696
{
13797
std::runtime_error("IO Error: failed to open file");
13898
}
139-
detail::dump_bin(stream, e, as_big_endian);
99+
auto s = xostream_wrapper(stream);
100+
detail::dump_bin(s, e, as_big_endian);
101+
}
102+
103+
template <typename E>
104+
inline void dump_bin(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian())
105+
{
106+
dump_bin<E>(filename.c_str(), e, as_big_endian);
140107
}
141108

142109
/**
@@ -148,7 +115,8 @@ namespace xt
148115
inline std::string dump_bin(const xexpression<E>& e, bool as_big_endian=is_big_endian())
149116
{
150117
std::stringstream stream;
151-
detail::dump_bin(stream, e, as_big_endian);
118+
auto s = xostream_wrapper(stream);
119+
detail::dump_bin(s, e, as_big_endian);
152120
return stream.str();
153121
}
154122

@@ -180,14 +148,21 @@ namespace xt
180148
* @return xarray with contents from binary file
181149
*/
182150
template <typename T, layout_type L = layout_type::dynamic>
183-
inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian())
151+
inline auto load_bin(const char* filename, bool as_big_endian=is_big_endian())
184152
{
185153
std::ifstream stream(filename, std::ifstream::binary);
186154
if (!stream.is_open())
187155
{
188-
std::runtime_error("load_bin: failed to open file " + filename);
156+
std::runtime_error(std::string("load_bin: failed to open file ") + filename);
189157
}
190-
return load_bin<T, L>(stream, as_big_endian);
158+
auto s = xistream_wrapper(stream);
159+
return load_bin<T, L>(s, as_big_endian);
160+
}
161+
162+
template <typename T, layout_type L = layout_type::dynamic>
163+
inline auto load_bin(const std::string& filename, bool as_big_endian=is_big_endian())
164+
{
165+
return load_bin<T, L>(filename.c_str(), as_big_endian);
191166
}
192167

193168
struct xio_binary_config

‎include/xtensor-io/xio_blosc.hpp‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "xtensor-io.hpp"
1717
#include "xfile_array.hpp"
1818
#include "blosc.h"
19+
#include "xio_stream_wrapper.hpp"
1920

2021
namespace xt
2122
{
@@ -31,11 +32,12 @@ namespace xt
3132
}
3233
}
3334

34-
template <typename T>
35-
inline xt::svector<T> load_blosc_file(std::istream& stream, bool as_big_endian)
35+
template <typename T, class I>
36+
inline xt::svector<T> load_blosc(I& stream, bool as_big_endian)
3637
{
3738
init_blosc();
38-
std::string compressed_buffer{std::istreambuf_iterator<char>{stream}, {}};
39+
std::string compressed_buffer;
40+
stream.read_all(compressed_buffer);
3941
auto compressed_size = compressed_buffer.size();
4042
std::size_t uncompressed_size = 0;
4143
int res = blosc_cbuffer_validate(compressed_buffer.data(), compressed_size, &uncompressed_size);
@@ -60,7 +62,7 @@ namespace xt
6062
}
6163

6264
template <class O, class E>
63-
inline void dump_blosc_stream(O& stream, const xexpression<E>& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize)
65+
inline void dump_blosc(O& stream, const xexpression<E>& e, bool as_big_endian, int clevel, int shuffle, const char* cname, std::size_t blocksize)
6466
{
6567
init_blosc();
6668
using value_type = typename E::value_type;
@@ -99,8 +101,8 @@ namespace xt
99101
{
100102
XTENSOR_THROW(std::runtime_error, "Blosc: compression error");
101103
}
102-
stream.write(compressed_buffer,
103-
std::streamsize(true_compressed_size));
104+
stream.write(compressed_buffer, std::streamsize(true_compressed_size));
105+
stream.flush();
104106
char_allocator.deallocate(compressed_buffer, max_compressed_size);
105107
}
106108
} // namespace detail
@@ -111,10 +113,17 @@ namespace xt
111113
* @param stream An output stream to which to dump the data
112114
* @param e the xexpression
113115
*/
116+
template <typename E, class O>
117+
inline void dump_blosc(O& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
118+
{
119+
detail::dump_blosc(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
120+
}
121+
114122
template <typename E>
115123
inline void dump_blosc(std::ostream& stream, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
116124
{
117-
detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
125+
auto s = xostream_wrapper(stream);
126+
detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
118127
}
119128

120129
/**
@@ -124,14 +133,21 @@ namespace xt
124133
* @param e the xexpression
125134
*/
126135
template <typename E>
127-
inline void dump_blosc(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
136+
inline void dump_blosc(const char* filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
128137
{
129138
std::ofstream stream(filename, std::ofstream::binary);
130139
if (!stream.is_open())
131140
{
132-
XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename);
141+
XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename);
133142
}
134-
detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
143+
auto s = xostream_wrapper(stream);
144+
detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
145+
}
146+
147+
template <typename E>
148+
inline void dump_blosc(const std::string& filename, const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
149+
{
150+
dump_blosc<E>(filename.c_str(), e, as_big_endian, clevel, shuffle, cname, blocksize);
135151
}
136152

137153
/**
@@ -143,7 +159,8 @@ namespace xt
143159
inline std::string dump_blosc(const xexpression<E>& e, bool as_big_endian=is_big_endian(), int clevel=5, int shuffle=1, const char* cname="blosclz", std::size_t blocksize=0)
144160
{
145161
std::stringstream stream;
146-
detail::dump_blosc_stream(stream, e, as_big_endian, clevel, shuffle, cname, blocksize);
162+
auto s = xostream_wrapper(stream);
163+
detail::dump_blosc(s, e, as_big_endian, clevel, shuffle, cname, blocksize);
147164
return stream.str();
148165
}
149166

@@ -156,10 +173,10 @@ namespace xt
156173
* Fortran format
157174
* @return xarray with contents from blosc file
158175
*/
159-
template <typename T, layout_type L = layout_type::dynamic>
160-
inline auto load_blosc(std::istream& stream, bool as_big_endian=is_big_endian())
176+
template <typename T, layout_type L = layout_type::dynamic, class I>
177+
inline auto load_blosc(I& stream, bool as_big_endian=is_big_endian())
161178
{
162-
xt::svector<T> uncompressed_buffer = detail::load_blosc_file<T>(stream, as_big_endian);
179+
xt::svector<T> uncompressed_buffer = detail::load_blosc<T>(stream, as_big_endian);
163180
std::vector<std::size_t> shape = {uncompressed_buffer.size()};
164181
auto array = adapt(std::move(uncompressed_buffer), shape);
165182
return array;
@@ -175,14 +192,21 @@ namespace xt
175192
* @return xarray with contents from blosc file
176193
*/
177194
template <typename T, layout_type L = layout_type::dynamic>
178-
inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian())
195+
inline auto load_blosc(const char* filename, bool as_big_endian=is_big_endian())
179196
{
180197
std::ifstream stream(filename, std::ifstream::binary);
181198
if (!stream.is_open())
182199
{
183-
XTENSOR_THROW(std::runtime_error, "Blosc: failed to open file " + filename);
200+
XTENSOR_THROW(std::runtime_error, std::string("Blosc: failed to open file ") + filename);
184201
}
185-
return load_blosc<T, L>(stream, as_big_endian);
202+
auto s = xistream_wrapper(stream);;
203+
return load_blosc<T, L>(s, as_big_endian);
204+
}
205+
206+
template <typename T, layout_type L = layout_type::dynamic>
207+
inline auto load_blosc(const std::string& filename, bool as_big_endian=is_big_endian())
208+
{
209+
return load_blosc<T, L>(filename.c_str(), as_big_endian);
186210
}
187211

188212
struct xio_blosc_config
@@ -230,8 +254,8 @@ namespace xt
230254
}
231255
};
232256

233-
template <class E>
234-
void load_file(std::istream& stream, xexpression<E>& e, const xio_blosc_config& config)
257+
template <class E, class I>
258+
void load_file(I& stream, xexpression<E>& e, const xio_blosc_config& config)
235259
{
236260
E& ex = e.derived_cast();
237261
auto shape = ex.shape();
@@ -246,8 +270,8 @@ namespace xt
246270
}
247271
}
248272

249-
template <class E>
250-
void dump_file(std::ostream& stream, const xexpression<E> &e, const xio_blosc_config& config)
273+
template <class E, class O>
274+
void dump_file(O& stream, const xexpression<E> &e, const xio_blosc_config& config)
251275
{
252276
dump_blosc(stream, e, config.big_endian, config.clevel, config.shuffle, config.cname.c_str(), config.blocksize);
253277
}

‎include/xtensor-io/xio_disk_handler.hpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#include <xtensor/xarray.hpp>
55
#include <xtensor/xexpression.hpp>
6+
#include <xtensor-io/xio_stream_wrapper.hpp>
67

78
namespace xt
89
{
@@ -39,7 +40,8 @@ namespace xt
3940
std::ofstream out_file(path, std::ofstream::binary);
4041
if (out_file.is_open())
4142
{
42-
dump_file(out_file, expression, m_format_config);
43+
auto s = xostream_wrapper(out_file);
44+
dump_file(s, expression, m_format_config);
4345
}
4446
else
4547
{
@@ -55,7 +57,8 @@ namespace xt
5557
std::ifstream in_file(path, std::ifstream::binary);
5658
if (in_file.is_open())
5759
{
58-
load_file<ET>(in_file, array, m_format_config);
60+
auto s = xistream_wrapper(in_file);
61+
load_file<ET>(s, array, m_format_config);
5962
}
6063
else
6164
{

0 commit comments

Comments
 (0)