Skip to content

Commit 49feebd

Browse files
JohanMabilleJohanMabille
authored
Merge pull request #109 from davidbrochart/io_config
Add IO configuration
2 parents 8a5e5a5 + db1bf2a commit 49feebd

File tree

6 files changed

+70
-35
lines changed

6 files changed

+70
-35
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ install:
3535
# Install mamba
3636
- conda install mamba -c conda-forge
3737
# Install host dependencies
38-
- mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 xtensor=0.21.8 cpp-filesystem -c conda-forge
38+
- mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 xtensor=0.21.9 cpp-filesystem -c conda-forge
3939
- mamba install ffmpeg=4.1.3 -c conda-forge # openimageio is missing ffmpedg as a runtime requirement
4040
# Install build dependencies
4141
- mamba install cmake -c conda-forge

‎include/xtensor-io/xchunk_store_manager.hpp‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ namespace xt
116116
xchunk_store_manager& operator=(xchunk_store_manager&&) = default;
117117

118118
const shape_type& shape() const noexcept;
119+
const shape_type& chunk_shape() const noexcept;
119120

120121
template <class... Idxs>
121122
reference operator()(Idxs... idxs);
@@ -149,8 +150,8 @@ namespace xt
149150
IP& get_index_path();
150151
void flush();
151152

152-
template <class C>
153-
void configure_format(C& config);
153+
template <class FC, class IOC>
154+
void configure(FC& format_config, IOC& io_config);
154155

155156
template <class I>
156157
reference map_file_array(I first, I last);
@@ -179,6 +180,7 @@ namespace xt
179180
using index_pool_type = std::vector<shape_type>;
180181

181182
shape_type m_shape;
183+
shape_type m_chunk_shape;
182184
chunk_pool_type m_chunk_pool;
183185
index_pool_type m_index_pool;
184186
std::size_t m_unload_index;
@@ -318,6 +320,7 @@ namespace xt
318320
std::size_t pool_size,
319321
layout_type chunk_memory_layout)
320322
: m_shape(shape)
323+
, m_chunk_shape(chunk_shape)
321324
, m_unload_index(0u)
322325
{
323326
initialize(shape, chunk_shape, directory, false, 0, pool_size, chunk_memory_layout);
@@ -332,6 +335,7 @@ namespace xt
332335
const T& init_value,
333336
layout_type chunk_memory_layout)
334337
: m_shape(shape)
338+
, m_chunk_shape(chunk_shape)
335339
, m_unload_index(0u)
336340
{
337341
initialize(shape, chunk_shape, directory, true, init_value, pool_size, chunk_memory_layout);
@@ -375,6 +379,12 @@ namespace xt
375379
return m_shape;
376380
}
377381

382+
template <class EC, class IP>
383+
inline auto xchunk_store_manager<EC, IP>::chunk_shape() const noexcept -> const shape_type&
384+
{
385+
return m_chunk_shape;
386+
}
387+
378388
template <class EC, class IP>
379389
template <class... Idxs>
380390
inline auto xchunk_store_manager<EC, IP>::operator()(Idxs... idxs) -> reference
@@ -474,12 +484,12 @@ namespace xt
474484
}
475485

476486
template <class EC, class IP>
477-
template <class C>
478-
void xchunk_store_manager<EC, IP>::configure_format(C& config)
487+
template <class FC, class IOC>
488+
void xchunk_store_manager<EC, IP>::configure(FC& format_config, IOC& io_config)
479489
{
480490
for (auto& chunk: m_chunk_pool)
481491
{
482-
chunk.configure_format(config);
492+
chunk.configure(format_config, io_config);
483493
}
484494
}
485495

‎include/xtensor-io/xfile_array.hpp‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,11 @@ namespace xt
217217
const std::string& path() const noexcept;
218218
void set_path(const std::string& path);
219219

220-
template <class C>
221-
void configure_format(C& config);
220+
template <class FC, class IOC>
221+
void configure(FC& format_config, IOC& io_config);
222+
223+
template <class IOC>
224+
void configure_io(IOC& io_config);
222225

223226
void flush();
224227

@@ -606,10 +609,17 @@ namespace xt
606609
}
607610

608611
template <class E, class IOH>
609-
template <class C>
610-
inline void xfile_array_container<E, IOH>::configure_format(C& config)
612+
template <class FC, class IOC>
613+
inline void xfile_array_container<E, IOH>::configure(FC& format_config, IOC& io_config)
614+
{
615+
m_io_handler.configure(format_config, io_config);
616+
}
617+
618+
template <class E, class IOH>
619+
template <class IOC>
620+
inline void xfile_array_container<E, IOH>::configure_io(IOC& io_config)
611621
{
612-
m_io_handler.configure_format(config);
622+
m_io_handler.configure_io(io_config);
613623
}
614624

615625
template <class E, class IOH>

‎include/xtensor-io/xio_disk_handler.hpp‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,24 @@
66

77
namespace xt
88
{
9+
struct xio_disk_config
10+
{
11+
};
12+
913
template <class C>
1014
class xio_disk_handler
1115
{
1216
public:
17+
using io_config = xio_disk_config;
1318

1419
template <class E>
1520
void write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty);
1621

1722
template <class ET>
18-
void read(ET& array, const std::string& path) const;
23+
void read(ET& array, const std::string& path);
1924

20-
void configure_format(const C& format_config);
25+
void configure(const C& format_config, const xio_disk_config& io_config);
26+
void configure_io(const xio_disk_config& io_config);
2127

2228
private:
2329

@@ -44,7 +50,7 @@ namespace xt
4450

4551
template <class C>
4652
template <class ET>
47-
inline void xio_disk_handler<C>::read(ET& array, const std::string& path) const
53+
inline void xio_disk_handler<C>::read(ET& array, const std::string& path)
4854
{
4955
std::ifstream in_file(path, std::ifstream::binary);
5056
if (in_file.is_open())
@@ -58,11 +64,15 @@ namespace xt
5864
}
5965

6066
template <class C>
61-
inline void xio_disk_handler<C>::configure_format(const C& format_config)
67+
inline void xio_disk_handler<C>::configure(const C& format_config, const xio_disk_config& io_config)
6268
{
6369
m_format_config = format_config;
6470
}
6571

72+
template <class C>
73+
inline void xio_disk_handler<C>::configure_io(const xio_disk_config& io_config)
74+
{
75+
}
6676

6777
}
6878

‎include/xtensor-io/xio_gcs_handler.hpp‎

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,29 +9,39 @@ namespace gcs = google::cloud::storage;
99

1010
namespace xt
1111
{
12+
struct xio_gcs_config
13+
{
14+
gcs::Client client;
15+
std::string bucket;
16+
};
17+
1218
template <class C>
1319
class xio_gcs_handler
1420
{
1521
public:
22+
using io_config = xio_gcs_config;
1623

1724
xio_gcs_handler();
1825

1926
template <class E>
2027
void write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty);
2128

2229
template <class ET>
23-
void read(ET& array, const std::string& path) const;
30+
void read(ET& array, const std::string& path);
2431

25-
void configure_format(const C& format_config);
26-
void split_bucket_path(const std::string& path, std::string& bucket_name, std::string& file_path) const;
32+
void configure(const C& format_config, const xio_gcs_config& io_config);
33+
void configure_io(const xio_gcs_config& io_config);
2734

2835
private:
2936

3037
C m_format_config;
38+
gcs::Client m_client;
39+
std::string m_bucket;
3140
};
3241

3342
template <class C>
3443
xio_gcs_handler<C>::xio_gcs_handler()
44+
: m_client(gcs::ClientOptions((gcs::oauth2::CreateAnonymousCredentials())))
3545
{
3646
}
3747

@@ -41,39 +51,32 @@ namespace xt
4151
{
4252
if (m_format_config.will_dump(dirty))
4353
{
44-
std::string bucket_name;
45-
std::string file_path;
46-
split_bucket_path(path, bucket_name, file_path);
47-
gcs::Client client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials())));
48-
auto writer = client.WriteObject(bucket_name, file_path);
54+
auto writer = m_client.WriteObject(m_bucket, path);
4955
dump_file(writer, expression, m_format_config);
5056
}
5157
}
5258

5359
template <class C>
5460
template <class ET>
55-
inline void xio_gcs_handler<C>::read(ET& array, const std::string& path) const
61+
inline void xio_gcs_handler<C>::read(ET& array, const std::string& path)
5662
{
57-
std::string bucket_name;
58-
std::string file_path;
59-
split_bucket_path(path, bucket_name, file_path);
60-
gcs::Client client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials())));
61-
auto reader = client.ReadObject(bucket_name, file_path);
63+
auto reader = m_client.ReadObject(m_bucket, path);
6264
load_file<ET>(reader, array, m_format_config);
6365
}
6466

6567
template <class C>
66-
inline void xio_gcs_handler<C>::configure_format(const C& format_config)
68+
inline void xio_gcs_handler<C>::configure(const C& format_config, const xio_gcs_config& io_config)
6769
{
6870
m_format_config = format_config;
71+
m_client = io_config.client;
72+
m_bucket = io_config.bucket;
6973
}
7074

7175
template <class C>
72-
inline void xio_gcs_handler<C>::split_bucket_path(const std::string& path, std::string& bucket_name, std::string& file_path) const
76+
inline void xio_gcs_handler<C>::configure_io(const xio_gcs_config& io_config)
7377
{
74-
std::size_t i = path.find('/');
75-
bucket_name = path.substr(0, i);
76-
file_path = path.substr(i + 1);
78+
m_client = io_config.client;
79+
m_bucket = io_config.bucket;
7780
}
7881

7982
}

‎test/test_xio_gcs_handler.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@ namespace xt
2020
TEST(xio_gcs_handler, read)
2121
{
2222
xio_gcs_handler<xio_gzip_config> h;
23+
xio_gcs_config c = {gcs::Client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials()))), "zarr-demo"};
24+
h.configure_io(c);
2325
xarray<int32_t> a0;
24-
h.read(a0, "zarr-demo/v3/test.zr3/data/root/arthur/dent/c0/0");
26+
h.read(a0, "v3/test.zr3/data/root/arthur/dent/c0/0");
2527
xarray<int32_t> a1 = {0, 1, 2, 3, 4, 10, 11, 12, 13, 14};
2628
EXPECT_TRUE(xt::all(xt::equal(a0, a1)));
2729
}

0 commit comments

Comments
 (0)