Skip to content

Commit 4d6e910

Browse files
maciejwalkowiakmaciejwalkowiak
authored
Add S3Template (#314)
Adds S3Template with ability to: - create & delete buckets - store/read & serialize Java objects - upload/download from input stream/resource. - better handling for content types. Fixes #286 Fixes #131
1 parent d90287f commit 4d6e910

File tree

28 files changed

+3028
-680
lines changed

28 files changed

+3028
-680
lines changed

‎docs/src/main/asciidoc/s3.adoc‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,45 @@ If `DiskBufferingS3OutputStream` behavior does not fit your needs, you can imple
148148

149149
Possible alternative implementations can use multi-part upload (for example with https://github.com/CI-CMG/aws-s3-outputstream[aws-s3-outputstream library)] or https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/transfer/s3/S3TransferManager.html[S3TransferManager].
150150

151+
=== Using S3Template
152+
153+
Spring Cloud AWS provides a higher abstraction on the top of `S3Client` providing methods for the most common use cases when working with S3.
154+
155+
On the top of self-explanatory methods for creating and deleting buckets, `S3Template` provides a simple methods for uploading and downloading files:
156+
157+
[source,java]
158+
----
159+
@Autowired
160+
private S3Template s3Template;
161+
162+
InputStream is = ...
163+
// uploading file without metadata
164+
s3Template.upload(BUCKET, "file.txt", is);
165+
166+
// uploading file with metadata
167+
s3Template.upload(BUCKET, "file.txt", is, ObjectMetadata.builder().contentType("text/plain").build());
168+
----
169+
170+
`S3Template` also allows storing & retrieving Java objects.
171+
172+
[source,java]
173+
----
174+
Person p = new Person("John", "Doe");
175+
s3Template.store(BUCKET, "person.json", p);
176+
177+
Person loadedPerson = s3Template.read(BUCKET, "person.json", Person.class);
178+
----
179+
180+
By default, if Jackson is on the classpath, `S3Template` uses `ObjectMapper` based `Jackson2JsonS3ObjectConverter` to convert from S3 object to Java object and vice versa.
181+
This behavior can be overwritten by providing custom bean of type `S3ObjectConverter`.
182+
183+
=== Determining S3 Objects Content Type
184+
185+
All S3 objects stored in S3 through `S3Template`, `S3Resource` or `S3OutputStream` automatically get set a `contentType` property on the S3 object metadata, based on the S3 object key (file name).
186+
187+
By default, `PropertiesS3ObjectContentTypeResolver` - a component supporting over 800 file extensions is responsible for content type resolution.
188+
If this content type resolution does not meet your needs, you can provide a custom bean of type `S3ObjectContentTypeResolver` which will be automatically used in all components responsible for uploading files.
189+
151190
=== Configuration
152191

153192
The Spring Boot Starter for S3 provides the following configuration options:

‎spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfiguration.java‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,22 @@
1515
*/
1616
package io.awspring.cloud.autoconfigure.s3;
1717

18+
import com.fasterxml.jackson.databind.ObjectMapper;
1819
import io.awspring.cloud.autoconfigure.core.AwsClientBuilderConfigurer;
1920
import io.awspring.cloud.autoconfigure.core.AwsProperties;
2021
import io.awspring.cloud.s3.DiskBufferingS3OutputStreamProvider;
22+
import io.awspring.cloud.s3.Jackson2JsonS3ObjectConverter;
23+
import io.awspring.cloud.s3.PropertiesS3ObjectContentTypeResolver;
24+
import io.awspring.cloud.s3.S3ObjectContentTypeResolver;
25+
import io.awspring.cloud.s3.S3ObjectConverter;
26+
import io.awspring.cloud.s3.S3Operations;
2127
import io.awspring.cloud.s3.S3OutputStreamProvider;
2228
import io.awspring.cloud.s3.S3ProtocolResolver;
29+
import io.awspring.cloud.s3.S3Template;
2330
import io.awspring.cloud.s3.crossregion.CrossRegionS3Client;
31+
import java.util.Optional;
2432
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
33+
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
2534
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
2635
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
2736
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass;
@@ -64,8 +73,18 @@ S3ClientBuilder s3ClientBuilder(AwsClientBuilderConfigurer awsClientBuilderConfi
6473

6574
@Bean
6675
@ConditionalOnMissingBean
67-
S3OutputStreamProvider s3OutputStreamProvider(S3Client s3Client) {
68-
return new DiskBufferingS3OutputStreamProvider(s3Client);
76+
S3OutputStreamProvider s3OutputStreamProvider(S3Client s3Client,
77+
Optional<S3ObjectContentTypeResolver> contentTypeResolver) {
78+
return new DiskBufferingS3OutputStreamProvider(s3Client,
79+
contentTypeResolver.orElseGet(PropertiesS3ObjectContentTypeResolver::new));
80+
}
81+
82+
@Bean
83+
@ConditionalOnMissingBean(S3Operations.class)
84+
@ConditionalOnBean(S3ObjectConverter.class)
85+
S3Template s3Template(S3Client s3Client, S3OutputStreamProvider s3OutputStreamProvider,
86+
S3ObjectConverter s3ObjectConverter) {
87+
return new S3Template(s3Client, s3OutputStreamProvider, s3ObjectConverter);
6988
}
7089

7190
private S3Configuration s3ServiceConfiguration() {
@@ -105,4 +124,15 @@ S3Client s3Client(S3ClientBuilder s3ClientBuilder) {
105124

106125
}
107126

127+
@Configuration
128+
@ConditionalOnClass(ObjectMapper.class)
129+
static class Jackson2JsonS3ObjectConverterConfiguration {
130+
131+
@ConditionalOnMissingBean
132+
@Bean
133+
S3ObjectConverter s3ObjectConverter(Optional<ObjectMapper> objectMapper) {
134+
return new Jackson2JsonS3ObjectConverter(objectMapper.orElseGet(ObjectMapper::new));
135+
}
136+
}
137+
108138
}

‎spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfigurationTests.java‎

Lines changed: 128 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,29 @@
1818
import static org.assertj.core.api.Assertions.assertThat;
1919
import static org.mockito.Mockito.mock;
2020

21+
import com.fasterxml.jackson.databind.ObjectMapper;
2122
import io.awspring.cloud.autoconfigure.ConfiguredAwsClient;
2223
import io.awspring.cloud.autoconfigure.core.AwsAutoConfiguration;
2324
import io.awspring.cloud.autoconfigure.core.CredentialsProviderAutoConfiguration;
2425
import io.awspring.cloud.autoconfigure.core.RegionProviderAutoConfiguration;
2526
import io.awspring.cloud.s3.DiskBufferingS3OutputStreamProvider;
2627
import io.awspring.cloud.s3.ObjectMetadata;
28+
import io.awspring.cloud.s3.S3ObjectConverter;
2729
import io.awspring.cloud.s3.S3OutputStream;
2830
import io.awspring.cloud.s3.S3OutputStreamProvider;
31+
import io.awspring.cloud.s3.S3Template;
2932
import io.awspring.cloud.s3.crossregion.CrossRegionS3Client;
3033
import java.io.IOException;
3134
import java.net.URI;
35+
import org.junit.jupiter.api.Nested;
3236
import org.junit.jupiter.api.Test;
3337
import org.springframework.boot.autoconfigure.AutoConfigurations;
3438
import org.springframework.boot.test.context.FilteredClassLoader;
3539
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
3640
import org.springframework.context.annotation.Bean;
3741
import org.springframework.context.annotation.Configuration;
3842
import org.springframework.lang.Nullable;
43+
import org.springframework.test.util.ReflectionTestUtils;
3944
import software.amazon.awssdk.services.s3.S3Client;
4045
import software.amazon.awssdk.services.s3.S3ClientBuilder;
4146

@@ -82,68 +87,141 @@ void autoconfigurationIsNotTriggeredWhenS3ModuleIsNotOnClasspath() {
8287
});
8388
}
8489

85-
@Test
86-
void byDefaultCreatesCrossRegionS3Client() {
87-
this.contextRunner
88-
.run(context -> assertThat(context).getBean(S3Client.class).isInstanceOf(CrossRegionS3Client.class));
89-
}
90+
@Nested
91+
class S3ClientTests {
92+
@Test
93+
void byDefaultCreatesCrossRegionS3Client() {
94+
contextRunner.run(
95+
context -> assertThat(context).getBean(S3Client.class).isInstanceOf(CrossRegionS3Client.class));
96+
}
9097

91-
@Test
92-
void s3ClientCanBeOverwritten() {
93-
this.contextRunner.withUserConfiguration(CustomS3ClientConfiguration.class).run(context -> {
94-
assertThat(context).hasSingleBean(S3Client.class);
95-
assertThat(context).getBean(S3Client.class).isNotInstanceOf(CrossRegionS3Client.class);
96-
});
97-
}
98+
@Test
99+
void s3ClientCanBeOverwritten() {
100+
contextRunner.withUserConfiguration(CustomS3ClientConfiguration.class).run(context -> {
101+
assertThat(context).hasSingleBean(S3Client.class);
102+
assertThat(context).getBean(S3Client.class).isNotInstanceOf(CrossRegionS3Client.class);
103+
});
104+
}
98105

99-
@Test
100-
void byDefaultCreatesDiskBufferingS3OutputStreamProvider() {
101-
this.contextRunner.run(context -> assertThat(context).hasSingleBean(DiskBufferingS3OutputStreamProvider.class));
106+
@Test
107+
void createsStandardClientWhenCrossRegionModuleIsNotInClasspath() {
108+
contextRunner.withClassLoader(new FilteredClassLoader(CrossRegionS3Client.class)).run(context -> {
109+
assertThat(context).doesNotHaveBean(CrossRegionS3Client.class);
110+
assertThat(context).hasSingleBean(S3Client.class);
111+
});
112+
}
102113
}
103114

104-
@Test
105-
void customS3OutputStreamProviderCanBeConfigured() {
106-
this.contextRunner.withUserConfiguration(CustomS3OutputStreamProviderConfiguration.class)
107-
.run(context -> assertThat(context).hasSingleBean(CustomS3OutputStreamProvider.class));
115+
@Nested
116+
class OutputStreamProviderTests {
117+
@Test
118+
void byDefaultCreatesDiskBufferingS3OutputStreamProvider() {
119+
contextRunner.run(context -> assertThat(context).hasSingleBean(DiskBufferingS3OutputStreamProvider.class));
120+
}
121+
122+
@Test
123+
void customS3OutputStreamProviderCanBeConfigured() {
124+
contextRunner.withUserConfiguration(CustomS3OutputStreamProviderConfiguration.class)
125+
.run(context -> assertThat(context).hasSingleBean(CustomS3OutputStreamProvider.class));
126+
}
108127
}
109128

110-
@Test
111-
void createsStandardClientWhenCrossRegionModuleIsNotInClasspath() {
112-
this.contextRunner.withClassLoader(new FilteredClassLoader(CrossRegionS3Client.class)).run(context -> {
113-
assertThat(context).doesNotHaveBean(CrossRegionS3Client.class);
114-
assertThat(context).hasSingleBean(S3Client.class);
115-
});
129+
@Nested
130+
class EndpointConfigurationTests {
131+
@Test
132+
void withCustomEndpoint() {
133+
contextRunner.withPropertyValues("spring.cloud.aws.s3.endpoint:http://localhost:8090").run(context -> {
134+
S3ClientBuilder builder = context.getBean(S3ClientBuilder.class);
135+
ConfiguredAwsClient client = new ConfiguredAwsClient(builder.build());
136+
assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:8090"));
137+
assertThat(client.isEndpointOverridden()).isTrue();
138+
});
139+
}
140+
141+
@Test
142+
void withCustomGlobalEndpoint() {
143+
contextRunner.withPropertyValues("spring.cloud.aws.endpoint:http://localhost:8090").run(context -> {
144+
S3ClientBuilder builder = context.getBean(S3ClientBuilder.class);
145+
ConfiguredAwsClient client = new ConfiguredAwsClient(builder.build());
146+
assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:8090"));
147+
assertThat(client.isEndpointOverridden()).isTrue();
148+
});
149+
}
150+
151+
@Test
152+
void withCustomGlobalEndpointAndS3Endpoint() {
153+
contextRunner.withPropertyValues("spring.cloud.aws.endpoint:http://localhost:8090",
154+
"spring.cloud.aws.s3.endpoint:http://localhost:9999").run(context -> {
155+
S3ClientBuilder builder = context.getBean(S3ClientBuilder.class);
156+
ConfiguredAwsClient client = new ConfiguredAwsClient(builder.build());
157+
assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:9999"));
158+
assertThat(client.isEndpointOverridden()).isTrue();
159+
});
160+
}
116161
}
117162

118-
@Test
119-
void withCustomEndpoint() {
120-
this.contextRunner.withPropertyValues("spring.cloud.aws.s3.endpoint:http://localhost:8090").run(context -> {
121-
S3ClientBuilder builder = context.getBean(S3ClientBuilder.class);
122-
ConfiguredAwsClient client = new ConfiguredAwsClient(builder.build());
123-
assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:8090"));
124-
assertThat(client.isEndpointOverridden()).isTrue();
125-
});
163+
@Nested
164+
class S3TemplateAutoConfigurationTests {
165+
166+
@Test
167+
void withJacksonOnClasspathAutoconfiguresObjectConverter() {
168+
contextRunner.run(context -> {
169+
assertThat(context).hasSingleBean(S3ObjectConverter.class);
170+
assertThat(context).hasSingleBean(S3Template.class);
171+
});
172+
}
173+
174+
@Test
175+
void withoutJacksonOnClasspathDoesNotConfigureObjectConverter() {
176+
contextRunner.withClassLoader(new FilteredClassLoader(ObjectMapper.class)).run(context -> {
177+
assertThat(context).doesNotHaveBean(S3ObjectConverter.class);
178+
assertThat(context).doesNotHaveBean(S3Template.class);
179+
});
180+
}
181+
182+
@Test
183+
void usesCustomObjectMapperBean() {
184+
contextRunner.withUserConfiguration(CustomJacksonConfiguration.class).run(context -> {
185+
S3ObjectConverter bean = context.getBean(S3ObjectConverter.class);
186+
ObjectMapper objectMapper = (ObjectMapper) ReflectionTestUtils.getField(bean, "objectMapper");
187+
assertThat(objectMapper).isEqualTo(context.getBean("customObjectMapper"));
188+
});
189+
}
190+
191+
@Test
192+
void usesCustomS3ObjectConverter() {
193+
contextRunner
194+
.withUserConfiguration(CustomJacksonConfiguration.class, CustomS3ObjectConverterConfiguration.class)
195+
.run(context -> {
196+
S3ObjectConverter s3ObjectConverter = context.getBean(S3ObjectConverter.class);
197+
S3ObjectConverter customS3ObjectConverter = (S3ObjectConverter) context
198+
.getBean("customS3ObjectConverter");
199+
assertThat(s3ObjectConverter).isEqualTo(customS3ObjectConverter);
200+
201+
S3Template s3Template = context.getBean(S3Template.class);
202+
203+
S3ObjectConverter converter = (S3ObjectConverter) ReflectionTestUtils.getField(s3Template,
204+
"s3ObjectConverter");
205+
assertThat(converter).isEqualTo(customS3ObjectConverter);
206+
});
207+
}
126208
}
127209

128-
@Test
129-
void withCustomGlobalEndpoint() {
130-
this.contextRunner.withPropertyValues("spring.cloud.aws.endpoint:http://localhost:8090").run(context -> {
131-
S3ClientBuilder builder = context.getBean(S3ClientBuilder.class);
132-
ConfiguredAwsClient client = new ConfiguredAwsClient(builder.build());
133-
assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:8090"));
134-
assertThat(client.isEndpointOverridden()).isTrue();
135-
});
210+
@Configuration(proxyBeanMethods = false)
211+
static class CustomJacksonConfiguration {
212+
@Bean
213+
ObjectMapper customObjectMapper() {
214+
return new ObjectMapper();
215+
}
136216
}
137217

138-
@Test
139-
void withCustomGlobalEndpointAndS3Endpoint() {
140-
this.contextRunner.withPropertyValues("spring.cloud.aws.endpoint:http://localhost:8090",
141-
"spring.cloud.aws.s3.endpoint:http://localhost:9999").run(context -> {
142-
S3ClientBuilder builder = context.getBean(S3ClientBuilder.class);
143-
ConfiguredAwsClient client = new ConfiguredAwsClient(builder.build());
144-
assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:9999"));
145-
assertThat(client.isEndpointOverridden()).isTrue();
146-
});
218+
@Configuration(proxyBeanMethods = false)
219+
static class CustomS3ObjectConverterConfiguration {
220+
221+
@Bean
222+
S3ObjectConverter customS3ObjectConverter() {
223+
return mock(S3ObjectConverter.class);
224+
}
147225
}
148226

149227
@Configuration(proxyBeanMethods = false)
@@ -172,7 +250,6 @@ static class CustomS3OutputStreamProvider implements S3OutputStreamProvider {
172250
public S3OutputStream create(String bucket, String key, @Nullable ObjectMetadata metadata) throws IOException {
173251
return null;
174252
}
175-
176253
}
177254

178255
}

0 commit comments

Comments
 (0)