Add AWS S3 IO handler

Author: davidbrochart

.travis.yml

@@ -35,7 +35,7 @@ install:
     # Install mamba
     - conda install mamba -c conda-forge
     # Install host dependencies
-    - mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 xtensor=0.21.9 cpp-filesystem -c conda-forge
+    - mamba install openimageio=2.1.10 libsndfile=1.0.28 zlib=1.2.11 highfive=2.1.1 blosc gdal nlohmann_json google-cloud-cpp=1.19.0 aws-sdk-cpp xtensor=0.21.9 cpp-filesystem -c conda-forge
     - mamba install ffmpeg=4.1.3 -c conda-forge  # openimageio is missing ffmpedg as a runtime requirement
     # Install build dependencies
     - mamba install cmake -c conda-forge

CMakeLists.txt

@@ -212,6 +212,28 @@ else()
   message(WARNING "Google Cloud Storage not found - install google-cloud-cpp for GCS IO handler support")
 endif()
 
+if (NOT DEFINED BUILD_SHARED_LIBS)
+  set (BUILD_SHARED_LIBS ON)
+  find_package(AWSSDK REQUIRED COMPONENTS s3)
+  unset (BUILD_SHARED_LIBS)
+else ()
+  find_package(AWSSDK REQUIRED COMPONENTS s3)
+endif ()
+message(STATUS "Trying to find AWS SDK for AWS S3 IO handler support")
+if(${AWSSDK_FOUND})
+  message(STATUS "AWSSDK ${AWSSDK_VERSION} found, AWS S3 IO handler support enabled")
+  target_include_directories(xtensor-io
+      INTERFACE
+      ${AWSSDK_INCLUDE_DIRS}
+  )
+  target_link_libraries(xtensor-io
+      INTERFACE
+      ${AWSSDK_LINK_LIBRARIES}
+  )
+else()
+  message(WARNING "AWSSDK not found - install aws-sdk-cpp for AWS S3 IO handler support")
+endif()
+
 if(DOWNLOAD_GTEST OR GTEST_SRC_DIR)
     set(BUILD_TESTS ON)
 endif()

include/xtensor-io/xio_aws_handler.hpp

@@ -0,0 +1,111 @@
+#ifndef XTENSOR_IO_AWS_HANDLER_HPP
+#define XTENSOR_IO_AWS_HANDLER_HPP
+
+#include "xtensor/xarray.hpp"
+#include "xtensor/xexpression.hpp"
+#include "xfile_array.hpp"
+#include <aws/core/Aws.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
+
+namespace xt
+{
+    struct xio_aws_config
+    {
+        Aws::S3::S3Client client;
+        Aws::String bucket;
+    };
+
+    template <class C>
+    class xio_aws_handler
+    {
+    public:
+        using io_config = xio_aws_config;
+
+        xio_aws_handler();
+
+        template <class E>
+        void write(const xexpression<E>& expression, const Aws::String& path, xfile_dirty dirty);
+
+        template <class ET>
+        void read(ET& array, const Aws::String& path);
+
+        void configure(const C& format_config, const xio_aws_config& io_config);
+        void configure_io(const xio_aws_config& io_config);
+
+    private:
+
+        C m_format_config;
+        Aws::S3::S3Client m_client;
+        Aws::String m_bucket;
+    };
+
+    template <class C>
+    xio_aws_handler<C>::xio_aws_handler()
+    {
+    }
+
+    template <class C>
+    template <class E>
+    inline void xio_aws_handler<C>::write(const xexpression<E>& expression, const Aws::String& path, xfile_dirty dirty)
+    {
+        if (m_format_config.will_dump(dirty))
+        {
+            Aws::S3::Model::PutObjectRequest request;
+            request.SetBucket(m_bucket);
+            request.SetKey(path);
+
+            std::shared_ptr<Aws::IOStream> writer = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", path.c_str(), std::ios_base::in | std::ios_base::binary);
+            dump_file(writer, expression, m_format_config);
+
+            request.SetBody(writer);
+
+            Aws::S3::Model::PutObjectOutcome outcome = m_client.PutObject(request);
+
+            if (!outcome.IsSuccess())
+            {
+                auto err = outcome.GetError();
+                XTENSOR_THROW(std::runtime_error, std::string("Error: PutObject: ") + err.GetExceptionName().c_str() + ": " + err.GetMessage().c_str());
+            }
+        }
+    }
+
+    template <class C>
+    template <class ET>
+    inline void xio_aws_handler<C>::read(ET& array, const Aws::String& path)
+    {
+        Aws::S3::Model::GetObjectRequest request;
+        request.SetBucket(m_bucket);
+        request.SetKey(path);
+
+        Aws::S3::Model::GetObjectOutcome outcome = m_client.GetObject(request);
+
+        if (!outcome.IsSuccess())
+        {
+            auto err = outcome.GetError();
+            XTENSOR_THROW(std::runtime_error, std::string("Error: GetObject: ") + err.GetExceptionName().c_str() + ": " + err.GetMessage().c_str());
+        }
+
+        auto& reader = outcome.GetResultWithOwnership().GetBody();
+        load_file<ET>(reader, array, m_format_config);
+    }
+
+    template <class C>
+    inline void xio_aws_handler<C>::configure(const C& format_config, const xio_aws_config& io_config)
+    {
+        m_format_config = format_config;
+        m_client = io_config.client;
+        m_bucket = io_config.bucket;
+    }
+
+    template <class C>
+    inline void xio_aws_handler<C>::configure_io(const xio_aws_config& io_config)
+    {
+        m_client = io_config.client;
+        m_bucket = io_config.bucket;
+    }
+
+}
+
+#endif

test/CMakeLists.txt

@@ -84,6 +84,7 @@ set(XTENSOR_IO_TESTS
     test_xio_gzip.cpp
     test_xio_binary.cpp
     test_xio_gcs_handler.cpp
+    test_xio_aws_handler.cpp
 )
 
 set(XTENSOR_IO_HO_TESTS

test/test_xio_aws_handler.cpp

@@ -0,0 +1,39 @@
+/***************************************************************************
+* Copyright (c) Wolf Vollprecht, Sylvain Corlay and Johan Mabille          *
+* Copyright (c) QuantStack                                                 *
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#include <cstdint>
+#include <sstream>
+#include <exception>
+
+#include "gtest/gtest.h"
+#include "xtensor/xview.hpp"
+#include "xtensor-io/xio_binary.hpp"
+#include "xtensor-io/xio_aws_handler.hpp"
+
+namespace xt
+{
+    TEST(xio_aws_handler, read)
+    {
+        Aws::SDKOptions options;
+        Aws::InitAPI(options);
+
+        xio_aws_handler<xio_binary_config> h;
+
+        Aws::S3::S3Client client;
+
+        xio_aws_config c = {client, "elevation-tiles-prod"};
+        h.configure_io(c);
+        xarray<char> a0;
+        h.read(a0, "main.js");
+        xarray<char> a1 = {'/', '*', 'j', 's', 'l', 'i', 'n', 't', ' ', 'b', 'r' ,'o', 'w', 's', 'e', 'r', ':', ' ', 't', 'r', 'u', 'e', '*', '/'};
+        EXPECT_TRUE(xt::all(xt::equal(xt::view(a0, xt::range(0, 24)), a1)));
+
+        Aws::ShutdownAPI(options);
+    }
+}