Merge feature/master/checksum-reuse (#6452)

* Reuse computed checksums across retries (#6413)

* Reuse computed checksums across retries

This commit adds the ability to reuse previously computed checksums for
a request across retries.

This ensures that if a request data stream is modified between attempts
that the server will reject the request.

As part of this change, the `http-auth-spi` package has been updated to
expose a new interface: `PayloadChecksumStore`. This is a simple storage
interface that allows signers to store and retrieve computed checksums.
Additionally, a new `SignerProperty` is introduced,
`SdkInternalHttpSignerProperty.CHECKSUM_CACHE` so that signers and
their callers can access this cache.

Note that both the interface and associated signer property are
`@SdkProtectedApi` and not intended to be used by non-SDK consumers of
`http-auth-spi`.

Finally, this adds a dependency on `checksums-spi` for `http-auth-spi`.

* Update core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java

Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com>

* Review comments

---------

Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com>

* Reuse computed checkums for async payloads (#6441)

* Reuse computed checkums for async payloads

This is a continuation of #6413 that adds support for checksum reuse on
async payloads.

Additionally, this commit renames references to "ChecksumCache" to
"ChecksumStore".

* Fix cross module issue

* Reuse checksums in legacy signing codepath (#6444)

* Reuse checksums in legacy signing codepath

This commit adds support for reusing calculated payload checksums over
retries in the legacy (i.e. non-SRA) signing codepaths.

* S3 testing with non-SRA

* Add changelog (#6454)

* Don't store x-amz-content-sha256 (#6458)

This doesn't correspond to a flexible checksum value, so this is not
required.

---------

Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com>

Author: dagnir

.changes/next-release/feature-AWSSDKforJavav2-9d16701.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "This update enables reusing the initially computed payload checksum of a request across all request attempts. This ensures that even if the content is changed from one attempt to the next, the checksum included in the request will remain the same and the request will be rejected by the service."
+}

core/auth/pom.xml

@@ -93,6 +93,11 @@
             <artifactId>http-auth-spi</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>checksums-spi</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.eventstream</groupId>
             <artifactId>eventstream</artifactId>

core/auth/src/main/java/software/amazon/awssdk/auth/signer/internal/chunkedencoding/AwsSignedChunkedEncodingInputStream.java

@@ -19,11 +19,13 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
 import software.amazon.awssdk.core.checksums.Algorithm;
 import software.amazon.awssdk.core.checksums.SdkChecksum;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.internal.chunked.AwsChunkedEncodingConfig;
 import software.amazon.awssdk.core.internal.io.AwsChunkedEncodingInputStream;
+import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
 import software.amazon.awssdk.utils.BinaryUtils;
 
 /**
@@ -60,12 +62,15 @@ public final class AwsSignedChunkedEncodingInputStream extends AwsChunkedEncodin
      * @param config          The configuration allows the user to customize chunk size and buffer size.
      *                        See {@link AwsChunkedEncodingConfig} for default values.
      */
-    private AwsSignedChunkedEncodingInputStream(InputStream in, SdkChecksum sdkChecksum,
+    private AwsSignedChunkedEncodingInputStream(InputStream in,
+                                                ChecksumAlgorithm checksumAlgorithm,
+                                                SdkChecksum sdkChecksum,
+                                                PayloadChecksumStore checksumStore,
                                                 String checksumHeaderForTrailer,
                                                 String headerSignature,
                                                 AwsChunkSigner chunkSigner,
                                                 AwsChunkedEncodingConfig config) {
-        super(in, sdkChecksum, checksumHeaderForTrailer, config);
+        super(in, checksumAlgorithm, sdkChecksum, checksumStore, checksumHeaderForTrailer, config);
         this.chunkSigner = chunkSigner;
         this.previousChunkSignature = headerSignature;
         this.headerSignature = headerSignature;
@@ -103,9 +108,14 @@ public Builder awsChunkSigner(AwsChunkSigner awsChunkSigner) {
 
         public AwsSignedChunkedEncodingInputStream build() {
 
-            return new AwsSignedChunkedEncodingInputStream(this.inputStream, this.sdkChecksum, this.checksumHeaderForTrailer,
+            return new AwsSignedChunkedEncodingInputStream(this.inputStream,
+                                                           this.checksumAlgorithm,
+                                                           this.sdkChecksum,
+                                                           this.checksumStore,
+                                                           this.checksumHeaderForTrailer,
                                                            this.headerSignature,
-                                                           this.awsChunkSigner, this.awsChunkedEncodingConfig);
+                                                           this.awsChunkSigner,
+                                                           this.awsChunkedEncodingConfig);
         }
     }
 

core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java

@@ -35,12 +35,15 @@
 import software.amazon.awssdk.http.Header;
 import software.amazon.awssdk.http.SdkHttpRequest;
 import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope;
+import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore;
 import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider;
 import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream;
 import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
 import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
 import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
+import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
 import software.amazon.awssdk.utils.BinaryUtils;
+import software.amazon.awssdk.utils.Logger;
 import software.amazon.awssdk.utils.Pair;
 import software.amazon.awssdk.utils.StringInputStream;
 import software.amazon.awssdk.utils.Validate;
@@ -51,16 +54,20 @@
  */
 @SdkInternalApi
 public final class AwsChunkedV4aPayloadSigner implements V4aPayloadSigner {
+    private static final Logger LOG = Logger.loggerFor(AwsChunkedV4aPayloadSigner.class);
 
     private final CredentialScope credentialScope;
     private final int chunkSize;
     private final ChecksumAlgorithm checksumAlgorithm;
+    private final PayloadChecksumStore payloadChecksumStore;
     private final List<Pair<String, List<String>>> preExistingTrailers = new ArrayList<>();
 
     private AwsChunkedV4aPayloadSigner(Builder builder) {
         this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope");
         this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize");
         this.checksumAlgorithm = builder.checksumAlgorithm;
+        this.payloadChecksumStore = builder.checksumStore == null ? NoOpPayloadChecksumStore.create() :
+                                    builder.checksumStore;
     }
 
     public static Builder builder() {
@@ -241,21 +248,41 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil
             return;
         }
         String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
+
+        String cachedChecksum = getCachedChecksum();
+
+        if (cachedChecksum != null) {
+            LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value",
+                                          checksumAlgorithm.algorithmId(), checksumHeaderName));
+            builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum)));
+            return;
+        }
+
         SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
         ChecksumInputStream checksumInputStream = new ChecksumInputStream(
             builder.inputStream(),
             Collections.singleton(sdkChecksum)
         );
 
-        TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName);
+        TrailerProvider checksumTrailer =
+            new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore);
 
         builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
     }
 
+    private String getCachedChecksum() {
+        byte[] checksumBytes = payloadChecksumStore.getChecksumValue(checksumAlgorithm);
+        if (checksumBytes != null) {
+            return BinaryUtils.toBase64(checksumBytes);
+        }
+        return null;
+    }
+
     static final class Builder {
         private CredentialScope credentialScope;
         private Integer chunkSize;
         private ChecksumAlgorithm checksumAlgorithm;
+        private PayloadChecksumStore checksumStore;
 
         public Builder credentialScope(CredentialScope credentialScope) {
             this.credentialScope = credentialScope;
@@ -272,6 +299,11 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) {
             return this;
         }
 
+        public Builder checksumStore(PayloadChecksumStore checksumStore) {
+            this.checksumStore = checksumStore;
+            return this;
+        }
+
         public AwsChunkedV4aPayloadSigner build() {
             return new AwsChunkedV4aPayloadSigner(this);
         }

core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java

@@ -33,6 +33,7 @@
 import static software.amazon.awssdk.http.auth.aws.internal.signer.util.CredentialUtils.sanitizeCredentials;
 import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION;
 import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
+import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_STORE;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -47,11 +48,13 @@
 import software.amazon.awssdk.http.SdkHttpRequest;
 import software.amazon.awssdk.http.auth.aws.internal.signer.Checksummer;
 import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope;
+import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore;
 import software.amazon.awssdk.http.auth.aws.signer.AwsV4aHttpSigner;
 import software.amazon.awssdk.http.auth.aws.signer.RegionSet;
 import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest;
 import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest;
 import software.amazon.awssdk.http.auth.spi.signer.BaseSignRequest;
+import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
 import software.amazon.awssdk.http.auth.spi.signer.SignRequest;
 import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
 import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
@@ -70,7 +73,7 @@ public final class DefaultAwsCrtV4aHttpSigner implements AwsV4aHttpSigner {
 
     @Override
     public SignedRequest sign(SignRequest<? extends AwsCredentialsIdentity> request) {
-        Checksummer checksummer = checksummer(request, null);
+        Checksummer checksummer = checksummer(request, null, checksumStore(request));
         V4aProperties v4aProperties = v4aProperties(request);
         AwsSigningConfig signingConfig = signingConfig(request, v4aProperties);
         V4aPayloadSigner payloadSigner = v4aPayloadSigner(request, v4aProperties);
@@ -104,7 +107,7 @@ private static V4aProperties v4aProperties(BaseSignRequest<?, ? extends AwsCrede
     }
 
     private static V4aPayloadSigner v4aPayloadSigner(
-        BaseSignRequest<?, ? extends AwsCredentialsIdentity> request,
+        SignRequest<? extends AwsCredentialsIdentity> request,
         V4aProperties v4aProperties) {
 
         boolean isPayloadSigning = isPayloadSigning(request);
@@ -117,6 +120,7 @@ private static V4aPayloadSigner v4aPayloadSigner(
                                              .credentialScope(v4aProperties.getCredentialScope())
                                              .chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES)
                                              .checksumAlgorithm(request.property(CHECKSUM_ALGORITHM))
+                                             .checksumStore(checksumStore(request))
                                              .build();
         }
 
@@ -252,4 +256,12 @@ private static V4aRequestSigningResult sign(SdkHttpRequest request, HttpRequest
             signingResult.getSignature(),
             signingConfig);
     }
+
+    private static PayloadChecksumStore checksumStore(SignRequest<? extends AwsCredentialsIdentity> request) {
+        PayloadChecksumStore cache = request.property(CHECKSUM_STORE);
+        if (cache == null) {
+            return NoOpPayloadChecksumStore.create();
+        }
+        return cache;
+    }
 }

core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java

@@ -45,7 +45,9 @@
 import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
 import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
 import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
+import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
 import software.amazon.awssdk.utils.BinaryUtils;
+import software.amazon.awssdk.utils.Logger;
 import software.amazon.awssdk.utils.Pair;
 import software.amazon.awssdk.utils.Validate;
 
@@ -55,16 +57,20 @@
  */
 @SdkInternalApi
 public final class AwsChunkedV4PayloadSigner implements V4PayloadSigner {
+    private static final Logger LOG = Logger.loggerFor(AwsChunkedV4PayloadSigner.class);
 
     private final CredentialScope credentialScope;
     private final int chunkSize;
     private final ChecksumAlgorithm checksumAlgorithm;
+    private final PayloadChecksumStore payloadChecksumStore;
     private final List<Pair<String, List<String>>> preExistingTrailers = new ArrayList<>();
 
     private AwsChunkedV4PayloadSigner(Builder builder) {
         this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope");
         this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize");
         this.checksumAlgorithm = builder.checksumAlgorithm;
+        this.payloadChecksumStore = builder.checksumStore == null ? NoOpPayloadChecksumStore.create() :
+                                    builder.checksumStore;
     }
 
     public static Builder builder() {
@@ -259,22 +265,43 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil
         if (checksumAlgorithm == null) {
             return;
         }
+
         String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
+
+        String cachedChecksum = getCachedChecksum();
+
+        if (cachedChecksum != null) {
+            LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value",
+                                          checksumAlgorithm.algorithmId(), checksumHeaderName));
+            builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum)));
+            return;
+        }
+
         SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
         ChecksumInputStream checksumInputStream = new ChecksumInputStream(
             builder.inputStream(),
             Collections.singleton(sdkChecksum)
         );
 
-        TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName);
+        TrailerProvider checksumTrailer =
+            new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore);
 
         builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
     }
 
+    private String getCachedChecksum() {
+        byte[] checksumBytes = payloadChecksumStore.getChecksumValue(checksumAlgorithm);
+        if (checksumBytes != null) {
+            return BinaryUtils.toBase64(checksumBytes);
+        }
+        return null;
+    }
+
     static class Builder {
         private CredentialScope credentialScope;
         private Integer chunkSize;
         private ChecksumAlgorithm checksumAlgorithm;
+        private PayloadChecksumStore checksumStore;
 
         public Builder credentialScope(CredentialScope credentialScope) {
             this.credentialScope = credentialScope;
@@ -291,6 +318,11 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) {
             return this;
         }
 
+        public Builder checksumStore(PayloadChecksumStore checksumStore) {
+            this.checksumStore = checksumStore;
+            return this;
+        }
+
         public AwsChunkedV4PayloadSigner build() {
             return new AwsChunkedV4PayloadSigner(this);
         }

core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/Checksummer.java

@@ -29,6 +29,7 @@
 import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
 import software.amazon.awssdk.http.ContentStreamProvider;
 import software.amazon.awssdk.http.SdkHttpRequest;
+import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
 import software.amazon.awssdk.utils.BinaryUtils;
 
 /**
@@ -49,6 +50,7 @@ public interface Checksummer {
      */
     static Checksummer create() {
         return new FlexibleChecksummer(
+            NoOpPayloadChecksumStore.create(),
             option().headerName(X_AMZ_CONTENT_SHA256).algorithm(SHA256).formatter(BinaryUtils::toHex).build()
         );
     }
@@ -57,9 +59,10 @@ static Checksummer create() {
      * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and the SHA-256 checksum. It places
      * the SHA-256 checksum in x-amz-content-sha256 header, and the given checksum-algorithm in the x-amz-checksum-[name] header.
      */
-    static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm) {
+    static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm, PayloadChecksumStore cache) {
         if (checksumAlgorithm != null) {
             return new FlexibleChecksummer(
+                cache,
                 option().headerName(X_AMZ_CONTENT_SHA256).algorithm(SHA256).formatter(BinaryUtils::toHex)
                         .build(),
                 option().headerName(checksumHeaderName(checksumAlgorithm)).algorithm(checksumAlgorithm)
@@ -82,9 +85,12 @@ static Checksummer forPrecomputed256Checksum(String precomputedSha256) {
      * given checksum string. It places the precomputed checksum in x-amz-content-sha256 header, and the given checksum-algorithm
      * in the x-amz-checksum-[name] header.
      */
-    static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorithm checksumAlgorithm) {
+    static Checksummer forFlexibleChecksum(String precomputedSha256,
+                                           ChecksumAlgorithm checksumAlgorithm,
+                                           PayloadChecksumStore cache) {
         if (checksumAlgorithm != null) {
             return new FlexibleChecksummer(
+                cache,
                 option().headerName(X_AMZ_CONTENT_SHA256).algorithm(new ConstantChecksumAlgorithm(precomputedSha256))
                         .formatter(b -> new String(b, StandardCharsets.UTF_8)).build(),
                 option().headerName(checksumHeaderName(checksumAlgorithm)).algorithm(checksumAlgorithm)
@@ -96,7 +102,7 @@ static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorit
     }
 
     static Checksummer forNoOp() {
-        return new FlexibleChecksummer();
+        return new FlexibleChecksummer(NoOpPayloadChecksumStore.create());
     }
 
     /**