Skip to content

Commit cfcf419

Browse files
JohanMabilleJohanMabille
authored
Merge pull request #116 from davidbrochart/aws
Add AWS S3 IO handler
2 parents f2d6097 + 769d926 commit cfcf419

File tree

6 files changed

+228
-1
lines changed

6 files changed

+228
-1
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ install:
3535
# Install mamba
3636
- conda install mamba -c conda-forge
3737
# Install host dependencies
38-
- mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 xtensor=0.21.9 cpp-filesystem -c conda-forge
38+
- mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 aws-sdk-cpp xtensor=0.21.9 cpp-filesystem -c conda-forge
3939
- mamba install ffmpeg=4.1.3 -c conda-forge # openimageio is missing ffmpedg as a runtime requirement
4040
# Install build dependencies
4141
- mamba install cmake -c conda-forge

‎CMakeLists.txt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,28 @@ else()
212212
message(WARNING "Google Cloud Storage not found - install google-cloud-cpp for GCS IO handler support")
213213
endif()
214214

215+
if (NOT DEFINED BUILD_SHARED_LIBS)
216+
set (BUILD_SHARED_LIBS ON)
217+
find_package(AWSSDK REQUIRED COMPONENTS s3)
218+
unset (BUILD_SHARED_LIBS)
219+
else ()
220+
find_package(AWSSDK REQUIRED COMPONENTS s3)
221+
endif ()
222+
message(STATUS "Trying to find AWS SDK for AWS S3 IO handler support")
223+
if(${AWSSDK_FOUND})
224+
message(STATUS "AWSSDK ${AWSSDK_VERSION} found, AWS S3 IO handler support enabled")
225+
target_include_directories(xtensor-io
226+
INTERFACE
227+
${AWSSDK_INCLUDE_DIRS}
228+
)
229+
target_link_libraries(xtensor-io
230+
INTERFACE
231+
${AWSSDK_LINK_LIBRARIES}
232+
)
233+
else()
234+
message(WARNING "AWSSDK not found - install aws-sdk-cpp for AWS S3 IO handler support")
235+
endif()
236+
215237
if(DOWNLOAD_GTEST OR GTEST_SRC_DIR)
216238
set(BUILD_TESTS ON)
217239
endif()

‎include/xtensor-io/xfile_array.hpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ namespace xt
145145

146146
xfile_array_container(const std::string& path, xfile_mode mode=xfile_mode::load);
147147

148+
template <class IOC>
149+
xfile_array_container(const std::string& path, IOC& io_config, xfile_mode mode=xfile_mode::load);
150+
148151
xfile_array_container(const std::string& path, xfile_mode mode, const value_type& init_value);
149152

150153
~xfile_array_container();
@@ -385,6 +388,19 @@ namespace xt
385388
set_path(path);
386389
}
387390

391+
template <class E, class IOH>
392+
template <class IOC>
393+
inline xfile_array_container<E, IOH>::xfile_array_container(const std::string& path, IOC& io_config, xfile_mode file_mode)
394+
: m_storage()
395+
, m_dirty(false)
396+
, m_io_handler()
397+
, m_file_mode(file_mode)
398+
, m_init(false)
399+
{
400+
m_io_handler.configure_io(io_config);
401+
set_path(path);
402+
}
403+
388404
template <class E, class IOH>
389405
inline xfile_array_container<E, IOH>::xfile_array_container(const std::string& path, xfile_mode file_mode, const value_type& init_value)
390406
: m_storage()

‎include/xtensor-io/xio_aws_handler.hpp‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#ifndef XTENSOR_IO_AWS_HANDLER_HPP
2+
#define XTENSOR_IO_AWS_HANDLER_HPP
3+
4+
#include "xtensor/xarray.hpp"
5+
#include "xtensor/xexpression.hpp"
6+
#include "xfile_array.hpp"
7+
#include <aws/core/Aws.h>
8+
#include <aws/s3/S3Client.h>
9+
#include <aws/s3/model/GetObjectRequest.h>
10+
#include <aws/s3/model/PutObjectRequest.h>
11+
12+
namespace xt
13+
{
14+
struct xio_aws_config
15+
{
16+
Aws::S3::S3Client client;
17+
Aws::String bucket;
18+
};
19+
20+
template <class C>
21+
class xio_aws_handler
22+
{
23+
public:
24+
using io_config = xio_aws_config;
25+
26+
xio_aws_handler();
27+
28+
template <class E>
29+
void write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty);
30+
31+
template <class ET>
32+
void read(ET& array, const std::string& path);
33+
34+
void configure(const C& format_config, const xio_aws_config& io_config);
35+
void configure_io(const xio_aws_config& io_config);
36+
37+
private:
38+
template <class E>
39+
void write(const xexpression<E>& expression, const char* path, xfile_dirty dirty);
40+
41+
template <class ET>
42+
void read(ET& array, const char* path);
43+
44+
C m_format_config;
45+
Aws::S3::S3Client m_client;
46+
Aws::String m_bucket;
47+
};
48+
49+
template <class C>
50+
xio_aws_handler<C>::xio_aws_handler()
51+
{
52+
}
53+
54+
template <class C>
55+
template <class E>
56+
inline void xio_aws_handler<C>::write(const xexpression<E>& expression, const std::string& path, xfile_dirty dirty)
57+
{
58+
write(expression, path.c_str(), dirty);
59+
}
60+
61+
template <class C>
62+
template <class E>
63+
inline void xio_aws_handler<C>::write(const xexpression<E>& expression, const char* path, xfile_dirty dirty)
64+
{
65+
if (m_format_config.will_dump(dirty))
66+
{
67+
Aws::String path2 = path;
68+
Aws::S3::Model::PutObjectRequest request;
69+
request.SetBucket(m_bucket);
70+
request.SetKey(path2);
71+
72+
std::shared_ptr<Aws::IOStream> writer = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", path, std::ios_base::in | std::ios_base::binary);
73+
dump_file(*writer, expression, m_format_config);
74+
75+
request.SetBody(writer);
76+
77+
Aws::S3::Model::PutObjectOutcome outcome = m_client.PutObject(request);
78+
79+
if (!outcome.IsSuccess())
80+
{
81+
auto err = outcome.GetError();
82+
XTENSOR_THROW(std::runtime_error, std::string("Error: PutObject: ") + err.GetExceptionName().c_str() + ": " + err.GetMessage().c_str());
83+
}
84+
}
85+
}
86+
87+
template <class C>
88+
template <class ET>
89+
inline void xio_aws_handler<C>::read(ET& array, const std::string& path)
90+
{
91+
read(array, path.c_str());
92+
}
93+
94+
template <class C>
95+
template <class ET>
96+
inline void xio_aws_handler<C>::read(ET& array, const char* path)
97+
{
98+
Aws::String path2 = path;
99+
Aws::S3::Model::GetObjectRequest request;
100+
request.SetBucket(m_bucket);
101+
request.SetKey(path2);
102+
103+
Aws::S3::Model::GetObjectOutcome outcome = m_client.GetObject(request);
104+
105+
if (!outcome.IsSuccess())
106+
{
107+
auto err = outcome.GetError();
108+
XTENSOR_THROW(std::runtime_error, std::string("Error: GetObject: ") + err.GetExceptionName().c_str() + ": " + err.GetMessage().c_str());
109+
}
110+
111+
auto& reader = outcome.GetResultWithOwnership().GetBody();
112+
load_file<ET>(reader, array, m_format_config);
113+
}
114+
115+
template <class C>
116+
inline void xio_aws_handler<C>::configure(const C& format_config, const xio_aws_config& io_config)
117+
{
118+
m_format_config = format_config;
119+
m_client = io_config.client;
120+
m_bucket = io_config.bucket;
121+
}
122+
123+
template <class C>
124+
inline void xio_aws_handler<C>::configure_io(const xio_aws_config& io_config)
125+
{
126+
m_client = io_config.client;
127+
m_bucket = io_config.bucket;
128+
}
129+
130+
}
131+
132+
#endif

‎test/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ set(XTENSOR_IO_TESTS
8484
test_xio_gzip.cpp
8585
test_xio_binary.cpp
8686
test_xio_gcs_handler.cpp
87+
test_xio_aws_handler.cpp
8788
)
8889

8990
set(XTENSOR_IO_HO_TESTS

‎test/test_xio_aws_handler.cpp‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/***************************************************************************
2+
* Copyright (c) Wolf Vollprecht, Sylvain Corlay and Johan Mabille *
3+
* Copyright (c) QuantStack *
4+
* *
5+
* Distributed under the terms of the BSD 3-Clause License. *
6+
* *
7+
* The full license is in the file LICENSE, distributed with this software. *
8+
****************************************************************************/
9+
10+
#include <cstdint>
11+
#include <sstream>
12+
#include <exception>
13+
14+
#include "gtest/gtest.h"
15+
#include "xtensor/xview.hpp"
16+
#include "xtensor-io/xfile_array.hpp"
17+
#include "xtensor-io/xio_binary.hpp"
18+
#include "xtensor-io/xio_aws_handler.hpp"
19+
20+
namespace xt
21+
{
22+
TEST(xio_aws_handler, read)
23+
{
24+
Aws::SDKOptions options;
25+
Aws::InitAPI(options);
26+
27+
xio_aws_handler<xio_binary_config> h;
28+
29+
Aws::S3::S3Client client;
30+
31+
xio_aws_config c = {client, "elevation-tiles-prod"};
32+
h.configure_io(c);
33+
xarray<char> a0;
34+
std::string path = "main.js";
35+
h.read(a0, path);
36+
xarray<char> a1 = {'/', '*', 'j', 's', 'l', 'i', 'n', 't', ' ', 'b', 'r' ,'o', 'w', 's', 'e', 'r', ':', ' ', 't', 'r', 'u', 'e', '*', '/'};
37+
EXPECT_TRUE(xt::all(xt::equal(xt::view(a0, xt::range(0, 24)), a1)));
38+
39+
Aws::ShutdownAPI(options);
40+
}
41+
42+
TEST(xio_aws_handler, xfile_array)
43+
{
44+
Aws::SDKOptions options;
45+
Aws::InitAPI(options);
46+
47+
Aws::S3::S3Client client;
48+
xio_aws_config c = {client, "elevation-tiles-prod"};
49+
xfile_array<char, xio_aws_handler<xio_binary_config>> a0("main.js", c);
50+
51+
xarray<char> a1 = {'/', '*', 'j', 's', 'l', 'i', 'n', 't', ' ', 'b', 'r' ,'o', 'w', 's', 'e', 'r', ':', ' ', 't', 'r', 'u', 'e', '*', '/'};
52+
EXPECT_TRUE(xt::all(xt::equal(xt::view(a0, xt::range(0, 24)), a1)));
53+
54+
Aws::ShutdownAPI(options);
55+
}
56+
}

0 commit comments

Comments
 (0)