Skip to content

Commit b183504

Browse files
Stuart Camrusscam
authored andcommitted
Update settings available for s3 repository plugin (#3204)
This commit updates the settings available for the s3 repository plugin. Backport of #3203
1 parent 47df2da commit b183504

File tree

2 files changed

+202
-73
lines changed

2 files changed

+202
-73
lines changed

src/Nest/Modules/SnapshotAndRestore/Repositories/S3Repository.cs

Lines changed: 188 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -7,131 +7,260 @@ public interface IS3Repository : IRepository<IS3RepositorySettings> { }
77

88
public class S3Repository : IS3Repository
99
{
10-
public S3Repository(S3RepositorySettings settings)
11-
{
12-
Settings = settings;
13-
}
10+
public S3Repository(S3RepositorySettings settings) => Settings = settings;
1411

1512
public IS3RepositorySettings Settings { get; set; }
1613
public string Type { get; } = "s3";
1714
}
1815

1916
public interface IS3RepositorySettings : IRepositorySettings
2017
{
18+
/// <summary>
19+
/// The name of the bucket to be used for snapshots. This field is required
20+
/// </summary>
2121
[JsonProperty("bucket")]
2222
string Bucket { get; set; }
2323

24+
/// <summary>
25+
/// The region where bucket is located. Defaults to US Standard.
26+
/// </summary>
2427
[Obsolete("Removed in NEST 6.x.")]
2528
[JsonProperty("region")]
2629
string Region { get; set; }
2730

31+
/// <summary>
32+
/// The endpoint to the S3 API. Defaults to AWS's default S3 endpoint. Note that setting a region overrides the endpoint setting.
33+
/// </summary>
34+
[JsonProperty("endpoint")]
35+
string Endpoint { get; set; }
36+
37+
/// <summary>
38+
/// The protocol to use (http or https). Defaults to value of cloud.aws.protocol or cloud.aws.s3.protocol from elasticsearch.yml configuration.
39+
/// </summary>
40+
[JsonProperty("protocol")]
41+
string Protocol { get; set; }
42+
43+
/// <summary>
44+
/// Specifies the path within bucket to repository data.
45+
/// Defaults to value of repositories.s3.base_path or to root directory if not set.
46+
/// </summary>
2847
[JsonProperty("base_path")]
2948
string BasePath { get; set; }
3049

50+
/// <summary>
51+
/// The access key to use for authentication. Defaults to value of cloud.aws.access_key from elasticsearch.yml configuration.
52+
/// </summary>
3153
[JsonProperty("access_key")]
3254
string AccessKey { get; set; }
3355

56+
/// <summary>
57+
/// The secret key to use for authentication. Defaults to value of cloud.aws.secret_key from elasticsearch.yml configuration.
58+
/// </summary>
3459
[JsonProperty("secret_key")]
3560
string SecretKey { get; set; }
3661

62+
/// <summary>
63+
/// Big files can be broken down into chunks during snapshotting if needed.
64+
/// The chunk size can be specified in bytes or by using size value notation,
65+
/// i.e. 1gb, 10mb, 5kb. Defaults to 1gb.
66+
/// </summary>
67+
[JsonProperty("chunk_size")]
68+
string ChunkSize { get; set; }
69+
70+
/// <summary>
71+
/// When set to true metadata files are stored in compressed format.
72+
/// This setting doesn't affect index files that are already compressed by default.
73+
/// Defaults to false.
74+
/// </summary>
3775
[JsonProperty("compress")]
3876
bool? Compress { get; set; }
3977

4078
[JsonProperty("concurrent_streams")]
4179
int? ConcurrentStreams { get; set; }
4280

43-
[JsonProperty("chunk_size")]
44-
string ChunkSize { get; set; }
81+
/// <summary>
82+
/// When set to true files are encrypted on server side using AES256 algorithm.
83+
/// Defaults to false.
84+
/// </summary>
85+
[JsonProperty("server_side_encryption")]
86+
bool? ServerSideEncryption { get; set; }
87+
88+
/// <summary>
89+
/// Minimum threshold below which the chunk is uploaded using a single request.
90+
/// Beyond this threshold, the S3 repository will use the AWS Multipart Upload API to split the chunk into
91+
/// several parts, each of buffer_size length, and to upload each part in its own request. Note that setting a
92+
/// buffer size lower than 5mb is not allowed since it will prevent the use of the Multipart API and may result
93+
/// in upload errors. It is also not possible to set a buffer size greater than 5gb as it is the maximum upload
94+
/// size allowed by S3. Defaults to the minimum between 100mb and 5% of the heap size.
95+
/// </summary>
96+
[JsonProperty("buffer_size")]
97+
string BufferSize { get; set; }
98+
99+
/// <summary>
100+
/// Number of retries in case of S3 errors. Defaults to 3.
101+
/// </summary>
102+
[JsonProperty("max_retries")]
103+
int? MaximumRetries { get; set; }
104+
105+
/// <summary>
106+
/// Set to true if you want to throttle retries. Defaults to AWS SDK default value (true).
107+
/// </summary>
108+
[JsonProperty("use_throttle_retries")]
109+
bool? UseThrottleRetries { get; set; }
110+
111+
/// <summary>
112+
/// Makes repository read-only. Defaults to false.
113+
/// </summary>
114+
[JsonProperty("readonly")]
115+
bool? ReadOnly { get; set; }
116+
117+
/// <summary>
118+
/// Specify a canned ACL for the S3 bucket.
119+
/// The S3 repository supports all S3 canned ACLs : private, public-read, public-read-write, authenticated-read,
120+
/// log-delivery-write, bucket-owner-read, bucket-owner-full-control. Defaults to private.
121+
/// </summary>
122+
[JsonProperty("canned_acl")]
123+
string CannedAcl { get; set; }
124+
125+
/// <summary>
126+
/// Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy, standard_ia.
127+
/// Defaults to standard.
128+
/// </summary>
129+
[JsonProperty("storage_class")]
130+
string StorageClass { get; set; }
131+
132+
/// <summary>
133+
/// Activate path style access for virtual hosting of buckets. The default behaviour is to detect which access style to use based on the configured endpoint (an IP will result in path-style access) and the bucket being accessed (some buckets are not valid DNS names). Defaults to false.
134+
/// </summary>
135+
[JsonProperty("path_style_access")]
136+
bool? PathStyleAccess { get; set; }
45137
}
46138

47139
public class S3RepositorySettings : IS3RepositorySettings
48140
{
49141
internal S3RepositorySettings() { }
50142

51-
public S3RepositorySettings(string bucket)
52-
{
53-
this.Bucket = bucket;
54-
}
143+
public S3RepositorySettings(string bucket) => this.Bucket = bucket;
55144

145+
/// <inheritdoc />
56146
public string Bucket { get; set; }
57-
[Obsolete("Removed in NEST 6.x.")]
147+
/// <inheritdoc />
58148
public string Region { get; set; }
149+
/// <inheritdoc />
150+
public string Endpoint { get; set; }
151+
/// <inheritdoc />
152+
public string Protocol { get; set; }
153+
/// <inheritdoc />
59154
public string BasePath { get; set; }
155+
/// <inheritdoc />
60156
public string AccessKey { get; set; }
157+
/// <inheritdoc />
61158
public string SecretKey { get; set; }
159+
/// <inheritdoc />
160+
public string ChunkSize { get; set; }
161+
/// <inheritdoc />
62162
public bool? Compress { get; set; }
163+
/// <inheritdoc />
63164
public int? ConcurrentStreams { get; set; }
64-
public string ChunkSize { get; set; }
165+
/// <inheritdoc />
166+
public bool? ServerSideEncryption { get; set; }
167+
/// <inheritdoc />
168+
public string BufferSize { get; set; }
169+
/// <inheritdoc />
170+
public int? MaximumRetries { get; set; }
171+
/// <inheritdoc />
172+
public bool? UseThrottleRetries { get; set; }
173+
/// <inheritdoc />
174+
public bool? ReadOnly { get; set; }
175+
/// <inheritdoc />
176+
public string CannedAcl { get; set; }
177+
/// <inheritdoc />
178+
public string StorageClass { get; set; }
179+
/// <inheritdoc />
180+
public bool? PathStyleAccess { get; set; }
65181
}
66182

67183
public class S3RepositorySettingsDescriptor
68184
: DescriptorBase<S3RepositorySettingsDescriptor, IS3RepositorySettings>, IS3RepositorySettings
69185
{
70186
string IS3RepositorySettings.Bucket { get; set; }
71-
[Obsolete("Removed in NEST 6.x.")]
72187
string IS3RepositorySettings.Region { get; set; }
188+
string IS3RepositorySettings.Endpoint { get; set; }
189+
string IS3RepositorySettings.Protocol { get; set; }
73190
string IS3RepositorySettings.BasePath { get; set; }
74191
string IS3RepositorySettings.AccessKey { get; set; }
75192
string IS3RepositorySettings.SecretKey { get; set; }
76-
bool? IS3RepositorySettings.Compress { get; set; }
77-
int? IS3RepositorySettings.ConcurrentStreams { get; set; }
78193
string IS3RepositorySettings.ChunkSize { get; set; }
194+
int? IS3RepositorySettings.ConcurrentStreams { get; set; }
195+
bool? IS3RepositorySettings.Compress { get; set; }
196+
bool? IS3RepositorySettings.ServerSideEncryption { get; set; }
197+
string IS3RepositorySettings.BufferSize { get; set; }
198+
int? IS3RepositorySettings.MaximumRetries { get; set; }
199+
bool? IS3RepositorySettings.UseThrottleRetries { get; set; }
200+
bool? IS3RepositorySettings.ReadOnly { get; set; }
201+
string IS3RepositorySettings.CannedAcl { get; set; }
202+
string IS3RepositorySettings.StorageClass { get; set; }
203+
bool? IS3RepositorySettings.PathStyleAccess { get; set; }
79204

80-
/// <summary>
81-
/// The name of the bucket to be used for snapshots. (Mandatory)
82-
/// </summary>
83-
/// <param name="bucket"></param>
205+
public S3RepositorySettingsDescriptor() { }
206+
207+
public S3RepositorySettingsDescriptor(string bucket) => Self.Bucket = bucket;
208+
209+
/// <inheritdoc cref="IS3RepositorySettings.Bucket"/>
84210
public S3RepositorySettingsDescriptor Bucket(string bucket) => Assign(a => a.Bucket = bucket);
85211

86-
/// <summary>
87-
/// The region where bucket is located. Defaults to US Standard
88-
/// </summary>
89-
/// <param name="region"></param>
90-
/// <returns></returns>
91-
[Obsolete("Removed in NEST 6.x.")]
212+
/// <inheritdoc cref="IS3RepositorySettings.Region"/>
213+
[Obsolete("Removed in NEST 6.x")]
92214
public S3RepositorySettingsDescriptor Region(string region) => Assign(a => a.Region = region);
93215

94-
/// <summary>
95-
/// Specifies the path within bucket to repository data. Defaults to root directory.
96-
/// </summary>
97-
/// <param name="basePath"></param>
98-
/// <returns></returns>
216+
/// <inheritdoc cref="IS3RepositorySettings.Endpoint"/>
217+
public S3RepositorySettingsDescriptor Endpoint(string endpoint) => Assign(a => a.Endpoint = endpoint);
218+
219+
/// <inheritdoc cref="IS3RepositorySettings.Protocol"/>
220+
public S3RepositorySettingsDescriptor Protocol(string protocol) => Assign(a => a.Protocol = protocol);
221+
222+
/// <inheritdoc cref="IS3RepositorySettings.BasePath"/>
99223
public S3RepositorySettingsDescriptor BasePath(string basePath) => Assign(a => a.BasePath = basePath);
100224

101-
/// <summary>
102-
/// The access key to use for authentication. Defaults to value of cloud.aws.access_key.
103-
/// </summary>
104-
/// <param name="accessKey"></param>
105-
/// <returns></returns>
225+
/// <inheritdoc cref="IS3RepositorySettings.AccessKey"/>
106226
public S3RepositorySettingsDescriptor AccessKey(string accessKey) => Assign(a => a.AccessKey = accessKey);
107227

108-
/// <summary>
109-
/// The secret key to use for authentication. Defaults to value of cloud.aws.secret_key.
110-
/// </summary>
111-
/// <param name="secretKey"></param>
112-
/// <returns></returns>
228+
/// <inheritdoc cref="IS3RepositorySettings.SecretKey"/>
113229
public S3RepositorySettingsDescriptor SecretKey(string secretKey) => Assign(a => a.SecretKey = secretKey);
114230

115-
/// <summary>
116-
/// When set to true metadata files are stored in compressed format. This setting doesn't
117-
/// affect index files that are already compressed by default. Defaults to false.
118-
/// </summary>
119-
/// <param name="compress"></param>
120-
public S3RepositorySettingsDescriptor Compress(bool compress = true) => Assign(a => a.Compress = compress);
231+
/// <inheritdoc cref="IS3RepositorySettings.ConcurrentStreams"/>
232+
public S3RepositorySettingsDescriptor ConcurrentStreams(int? concurrentStreams) => Assign(a => a.ConcurrentStreams = concurrentStreams);
121233

122-
/// <summary>
123-
/// Throttles the number of streams (per node) preforming snapshot operation. Defaults to 5
124-
/// </summary>
125-
/// <param name="concurrentStreams"></param>
126-
public S3RepositorySettingsDescriptor ConcurrentStreams(int concurrentStreams) => Assign(a => a.ConcurrentStreams = concurrentStreams);
127-
128-
/// <summary>
129-
/// Big files can be broken down into chunks during snapshotting if needed.
130-
/// The chunk size can be specified in bytes or by using size value notation,
131-
/// i.e. 1g, 10m, 5k. Defaults to 100m.
132-
/// </summary>
133-
/// <param name="chunkSize"></param>
234+
/// <inheritdoc cref="IS3RepositorySettings.ChunkSize"/>
134235
public S3RepositorySettingsDescriptor ChunkSize(string chunkSize) => Assign(a => a.ChunkSize = chunkSize);
236+
237+
/// <inheritdoc cref="IS3RepositorySettings.Compress"/>
238+
public S3RepositorySettingsDescriptor Compress(bool? compress = true) => Assign(a => a.Compress = compress);
239+
240+
/// <inheritdoc cref="IS3RepositorySettings.ServerSideEncryption"/>
241+
public S3RepositorySettingsDescriptor ServerSideEncryption(bool? serverSideEncryption = true) =>
242+
Assign(a => a.ServerSideEncryption = serverSideEncryption);
243+
244+
/// <inheritdoc cref="IS3RepositorySettings.BufferSize"/>
245+
public S3RepositorySettingsDescriptor BufferSize(string bufferSize) => Assign(a => a.BufferSize = bufferSize);
246+
247+
/// <inheritdoc cref="IS3RepositorySettings.MaximumRetries"/>
248+
public S3RepositorySettingsDescriptor MaximumRetries(int maximumRetries) => Assign(a => a.MaximumRetries = maximumRetries);
249+
250+
/// <inheritdoc cref="IS3RepositorySettings.UseThrottleRetries"/>
251+
public S3RepositorySettingsDescriptor UseThrottleRetries(bool? useThrottleRetries = true) => Assign(a => a.UseThrottleRetries = useThrottleRetries);
252+
253+
/// <inheritdoc cref="IS3RepositorySettings.ReadOnly"/>
254+
public S3RepositorySettingsDescriptor ReadOnly(bool? @readonly = true) => Assign(a => a.ReadOnly = @readonly);
255+
256+
/// <inheritdoc cref="IS3RepositorySettings.CannedAcl"/>
257+
public S3RepositorySettingsDescriptor CannedAcl(string cannedAcl) => Assign(a => a.CannedAcl = cannedAcl);
258+
259+
/// <inheritdoc cref="IS3RepositorySettings.StorageClass"/>
260+
public S3RepositorySettingsDescriptor StorageClass(string storageClass) => Assign(a => a.StorageClass = storageClass);
261+
262+
/// <inheritdoc cref="IS3RepositorySettings.PathStyleAccess"/>
263+
public S3RepositorySettingsDescriptor PathStyleAccess(bool? pathStyleAccess = true) => Assign(a => a.PathStyleAccess = pathStyleAccess);
135264
}
136265

137266
public class S3RepositoryDescriptor
@@ -141,6 +270,6 @@ public class S3RepositoryDescriptor
141270
IS3RepositorySettings IRepository<IS3RepositorySettings>.Settings { get; set; }
142271

143272
public S3RepositoryDescriptor Settings(string bucket, Func<S3RepositorySettingsDescriptor, IS3RepositorySettings> settingsSelector = null) =>
144-
Assign(a => a.Settings = settingsSelector.InvokeOrDefault(new S3RepositorySettingsDescriptor().Bucket(bucket)));
273+
Assign(a => a.Settings = settingsSelector.InvokeOrDefault(new S3RepositorySettingsDescriptor(bucket)));
145274
}
146275
}

src/Tests/Modules/SnapshotAndRestore/Repositories/CreateRepository/CreateRepositoryApiTests.cs

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -252,13 +252,13 @@ protected override LazyResponses ClientUsage() => Calls(
252252
type = "s3",
253253
settings = new {
254254
bucket = "foobucket",
255-
region = "us-east",
256255
base_path = "some/path",
257-
access_key = "fooaccess",
258-
secret_key = "foosecret",
259256
compress = true,
260-
concurrent_streams = 5,
261-
chunk_size = "64mb"
257+
chunk_size = "64mb",
258+
server_side_encryption = true,
259+
buffer_size = "100mb",
260+
canned_acl = "authenticated-read",
261+
storage_class = "standard"
262262
}
263263
};
264264

@@ -267,27 +267,27 @@ protected override LazyResponses ClientUsage() => Calls(
267267
protected override Func<CreateRepositoryDescriptor, ICreateRepositoryRequest> Fluent => d => d
268268
.S3(fs => fs
269269
.Settings("foobucket", s => s
270-
.Region("us-east")
271270
.BasePath("some/path")
272-
.AccessKey("fooaccess")
273-
.SecretKey("foosecret")
274271
.Compress()
275-
.ConcurrentStreams(5)
276272
.ChunkSize("64mb")
273+
.ServerSideEncryption()
274+
.BufferSize("100mb")
275+
.CannedAcl("authenticated-read")
276+
.StorageClass("standard")
277277
)
278278
);
279279

280280
protected override CreateRepositoryRequest Initializer => new CreateRepositoryRequest(_name)
281281
{
282282
Repository = new S3Repository(new S3RepositorySettings("foobucket")
283283
{
284-
Region = "us-east",
285284
BasePath = "some/path",
286-
AccessKey = "fooaccess",
287-
SecretKey = "foosecret",
288285
Compress = true,
289-
ConcurrentStreams = 5,
290-
ChunkSize = "64mb"
286+
ChunkSize = "64mb",
287+
ServerSideEncryption = true,
288+
BufferSize = "100mb",
289+
CannedAcl = "authenticated-read",
290+
StorageClass = "standard"
291291
})
292292
};
293293
}

0 commit comments

Comments
 (0)