10000 Fix for Issue #3569 where the user set Content-Encoding was over writ… · aws/aws-sdk-java-v2@303a375 · GitHub
[go: up one dir, main page]
Skip to content

Commit 303a375

Browse files
joviegasjoviegas
authored
Fix for Issue #3569 where the user set Content-Encoding was over written to aws-chunked with Checksum algorithm enabled (#3720)
1 parent ee51283 commit 303a375

File tree

13 files changed

+115
-14
lines changed

13 files changed

+115
-14
lines changed

‎.changes/next-release/bugfix-AWSSDKforJavav2-16b5911.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "bugfix",
3+
"category": "AWS SDK for Java v2",
4+
"contributor": "",
5+
"description": "Append Content-encoding header instead of over writing the header when Checksum algorithm is selected along with user set Content-encoding"
6+
}

‎core/auth-crt/src/main/java/software/amazon/awssdk/authcrt/signer/internal/DefaultAwsCrtS3V4aSigner.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,6 @@ private static long getChecksumTrailerLength(SignerChecksumParams signerParams,
224224
private static void updateRequestWithTrailer(SignerChecksumParams signerChecksumParams,
225225
SdkHttpFullRequest.Builder mutableRequest) {
226226
mutableRequest.putHeader("x-amz-trailer", signerChecksumParams.chec 10000 ksumHeaderName());
227-
mutableRequest.putHeader("Content-Encoding", "aws-chunked");
227+
mutableRequest.appendHeader("Content-Encoding", "aws-chunked");
228228
}
229229
}

‎core/auth/src/main/java/software/amazon/awssdk/auth/signer/internal/AbstractAwsS3V4Signer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ protected String calculateContentHash(SdkHttpFullRequest.Builder mutableRequest,
236236
ChecksumSpecs.builder().headerName(signerParams.checksumParams().checksumHeaderName()).build())) {
237237
isTrailingChecksum = true;
238238
mutableRequest.putHeader("x-amz-trailer", headerForTrailerChecksumLocation);
239-
mutableRequest.putHeader("Content-Encoding", "aws-chunked");
239+
mutableRequest.appendHeader("Content-Encoding", "aws-chunked");
240240
}
241241
}
242242
// Make sure "Content-Length" header is not empty so that HttpClient

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/interceptor/AsyncRequestBodyHttpChecksumTrailerInterceptor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ private static SdkHttpRequest updateHeadersForTrailerChecksum(Context.ModifyHttp
105105

106106
return context.httpRequest().copy(r ->
107107
r.putHeader(HttpChecksumConstant.HEADER_FOR_TRAILER_REFERENCE, checksum.headerName())
108-
.putHeader("Content-encoding", HttpChecksumConstant.AWS_CHUNKED_HEADER)
108+
.appendHeader("Content-encoding", HttpChecksumConstant.AWS_CHUNKED_HEADER)
109109
.putHeader("x-amz-content-sha256", HttpChecksumConstant.CONTENT_SHA_256_FOR_UNSIGNED_TRAILER)
110110
.putHeader("x-amz-decoded-content-length", Long.toString(originalContentLength))
111111
.putHeader(Header.CONTENT_LENGTH,

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/interceptor/SyncHttpChecksumInTrailerInterceptor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, Execu
108108
long originalContentLength = context.requestBody().get().optionalContentLength().orElse(0L);
109109
return context.httpRequest().copy(
110110
r -> r.putHeader(HttpChecksumConstant.HEADER_FOR_TRAILER_REFERENCE, checksum.headerName())
111-
.putHeader("Content-encoding", AWS_CHUNKED_HEADER)
111+
.appendHeader("Content-encoding", AWS_CHUNKED_HEADER)
112112
.putHeader("x-amz-content-sha256", CONTENT_SHA_256_FOR_UNSIGNED_TRAILER)
113113
.putHeader("x-amz-decoded-content-length", Long.toString(originalContentLength))
114114
.putHeader(CONTENT_LENGTH,

‎services/s3/src/it/java/software/amazon/awssdk/services/s3/checksum/AsyncHttpChecksumIntegrationTest.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
5050
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
5151
import software.amazon.awssdk.services.s3.utils.CaptureChecksumValidationInterceptor;
52+
import software.amazon.awssdk.services.s3.utils.ChecksumUtils;
5253
import software.amazon.awssdk.testutils.RandomTempFile;
5354

5455
public class AsyncHttpChecksumIntegrationTest extends S3IntegrationTestBase {
@@ -115,6 +116,7 @@ void asyncHttpsValidUnsignedTrailerChecksumCalculatedBySdkClient_withSmallReques
115116
.key(KEY).checksumMode(ChecksumMode.ENABLED)
116117
.build(), AsyncResponseTransformer.toBytes()).join().asUtf8String();
117118
assertThat(interceptor.validationAlgorithm()).isEqualTo(Algorithm.CRC32);
119+
assertThat(interceptor.contentEncoding()).isEqualTo("aws-chunked");
118120
assertThat(interceptor.responseValidation()).isEqualTo(ChecksumValidation.VALIDATED);
119121
assertThat(response).isEqualTo("Hello world");
120122
}
@@ -126,10 +128,12 @@ void asyncHttpsValidUnsignedTrailerChecksumCalculatedBySdkClient_withHugeRequest
126128
s3Async.putObject(PutObjectRequest.builder()
127129
.bucket(BUCKET)
128130
.key(KEY)
131+
.contentEncoding("gzip")
129132
.checksumAlgorithm(ChecksumAlgorithm.CRC32)
130133
.build(), AsyncRequestBody.fromString(createDataOfSize(64 * KB, 'a'))).join();
131134
assertThat(interceptor.requestChecksumInTrailer()).isEqualTo("x-amz-checksum-crc32");
132135
assertThat(interceptor.requestChecksumInHeader()).isNull();
136+
assertThat(interceptor.contentEncoding()).isEqualTo("gzip,aws-chunked");
133137

134138
String response = s3Async.getObject(GetObjectRequest.builder().bucket(BUCKET)
135139
.key(KEY).checksumMode(ChecksumMode.ENABLED)

‎services/s3/src/it/java/software/amazon/awssdk/services/s3/checksum/HttpChecksumIntegrationTest.java

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.junit.jupiter.api.BeforeAll;
3535
import org. 10000 junit.jupiter.api.Test;
3636
import software.amazon.awssdk.authcrt.signer.internal.DefaultAwsCrtS3V4aSigner;
37+
import software.amazon.awssdk.core.HttpChecksumConstant;
3738
import software.amazon.awssdk.core.ResponseInputStream;
3839
import software.amazon.awssdk.core.checksums.Algorithm;
3940
import software.amazon.awssdk.core.checksums.ChecksumValidation;
@@ -45,13 +46,12 @@
4546
import software.amazon.awssdk.services.s3.model.ChecksumMode;
4647
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
4748
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
48-
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
4949
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
5050
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
5151
import software.amazon.awssdk.services.s3.model.S3Exception;
5252
import software.amazon.awssdk.services.s3.utils.CaptureChecksumValidationInterceptor;
53+
import software.amazon.awssdk.services.s3.utils.ChecksumUtils;
5354
import software.amazon.awssdk.testutils.RandomTempFile;
54-
import software.amazon.awssdk.testutils.Waiter;
5555

5656
public class HttpChecksumIntegrationTest extends S3IntegrationTestBase {
5757

@@ -102,6 +102,7 @@ public void validHeaderChecksumCalculatedBySdkClient() {
102102
.key(KEY)
103103
.build(), RequestBody.fromString("Hello world"));
104104
assertThat(interceptor.requestChecksumInTrailer()).isEqualTo("x-amz-checksum-crc32");
105+
assertThat(interceptor.contentEncoding()).isEqualTo(HttpChecksumConstant.AWS_CHUNKED_HEADER);
105106
assertThat(interceptor.requestChecksumInHeader()).isNull();
106107
assertThat(putObjectResponse.sdkHttpResponse().firstMatchingHeader("x-amz-checksum-crc32"))
107108
.hasValue("i9aeUg==");
@@ -112,11 +113,13 @@ public void validHeaderChecksumSentDirectlyInTheField() {
112113
PutObjectResponse putObjectResponse = s3Https.putObject(PutObjectRequest.builder()
113114
.bucket(BUCKET)
114115
.checksumAlgorithm(ChecksumAlgorithm.CRC32)
116+
.contentEncoding("gzip")
115117
.checksumCRC32("i9aeUg==")
116118
.key(KEY)
117119
.build(), RequestBody.fromString("Hello world"));
118120
assertThat(interceptor.requestChecksumInHeader()).isEqualTo("i9aeUg==");
119121
assertThat(interceptor.requestChecksumInTrailer()).isNull();
122+
assertThat(interceptor.contentEncoding()).isEqualTo("gzip");
120123
assertThat(putObjectResponse.sdkHttpResponse().firstMatchingHeader("x-amz-checksum-crc32")).hasValue("i9aeUg==");
121124
}
122125

@@ -191,6 +194,7 @@ public void syncValidSignedTrailerChecksumCalculatedBySdkClient() {
191194
new InputStreamReader(s3HttpsObject, StandardCharsets.UTF_8))
192195
.lines()
193196
.collect(Collectors.joining("\n"));
197+
assertThat(interceptor.contentEncoding()).isEmpty();
194198
assertThat(interceptor.validationAlgorithm()).isEqualTo(Algorithm.CRC32);
195199
assertThat(interceptor.responseValidation()).isEqualTo(ChecksumValidation.VALIDATED);
196200
assertThat(text).isEqualTo("Hello world");
@@ -204,10 +208,12 @@ public void syncValidSignedTrailerChecksumCalculatedBySdkClient_Empty_String() {
204208
s3.putObject(PutObjectRequest.builder()
205209
.bucket(BUCKET)
206210
.key(KEY)
211+
.contentEncoding("gzip")
207212
.checksumAlgorithm(ChecksumAlgorithm.CRC32)
208213
.build(), RequestBody.fromString(""));
209214

210215
assertThat(interceptor.requestChecksumInTrailer()).isEqualTo("x-amz-checksum-crc32");
216+
assertThat(interceptor.contentEncoding()).isEqualTo("gzip,aws-chunked");
211217
assertThat(interceptor.requestChecksumInHeader()).isNull();
212218

213219
ResponseInputStream<GetObjectResponse> s3HttpsObject =
@@ -247,16 +253,18 @@ public void syncValidSignedTrailerChecksumCalculatedBySdkClientWithSigv4a() {
247253
}
248254

249255
@Test
250-
public void syncValidUnsignedTrailerChecksumCalculatedBySdkClientWithSigv4a() {
256+
public void syncValidSignedTrailerChecksumCalculatedBySdkClientWithSigv4a_withContentEncoding() {
251257

252258
s3.putObject(PutObjectRequest.builder()
253259
.bucket(BUCKET)
254260
.key(KEY)
255261
.checksumAlgorithm(ChecksumAlgorithm.CRC32)
262+
.contentEncoding("gzip")
256263
.overrideConfiguration(o -> o.signer(DefaultAwsCrtS3V4aSigner.create()))
257264
.build(), RequestBody.fromString("Hello world"));
258265

259266
assertThat(interceptor.requestChecksumInTrailer()).isEqualTo("x-amz-checksum-crc32");
267+
assertThat(interceptor.contentEncoding()).isEqualTo("gzip,aws-chunked");
260268
assertThat(interceptor.requestChecksumInHeader()).isNull();
261269

262270
ResponseInputStream<GetObjectResponse> s3HttpsObject =
@@ -276,9 +284,11 @@ public void syncUnsignedPayloadForHugeMessage() throws InterruptedException {
276284
.bucket(BUCKET)
277285
.key(KEY)
278286
.checksumAlgorithm(ChecksumAlgorithm.CRC32)
287+
.contentEncoding("gzip")
279288
.build(), RequestBody.fromString(createDataOfSize(HUGE_MSG_SIZE, 'a')));
280289

281290
assertThat(interceptor.requestChecksumInTrailer()).isEqualTo("x-amz-checksum-crc32");
291+
assertThat(interceptor.contentEncoding()).isEqualTo("gzip,aws-chunked");
282292
assertThat(interceptor.requestChecksumInHeader()).isNull();
283293

284294
Thread.sleep(1000);

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/S3SignerTest.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,28 @@ public void payloadSigningWithChecksum() {
102102
.withRequestBody(containing("0;")));
103103
}
104104

105+
@Test
106+
public void payloadSigningWithChecksumWithContentEncodingSuppliedByUser() {
107+
108+
S3Client s3Client = getS3Client(true, true, URI.create(getEndpoint()));
109+
stubFor(any(urlMatching(".*"))
110+
.willReturn(response()));
111+
s3Client.putObject(PutObjectRequest.builder()
112+
.checksumAlgorithm(ChecksumAlgorithm.CRC32).contentEncoding("deflate")
113+
.bucket("test").key("test").build(), RequestBody.fromBytes("abc".getBytes()));
114+
verify(putRequestedFor(anyUrl()).withHeader(CONTENT_TYPE, equalTo(Mimetype.MIMETYPE_OCTET_STREAM)));
115+
verify(putRequestedFor(anyUrl()).withHeader(CONTENT_LENGTH, equalTo("296")));
116+
verify(putRequestedFor(anyUrl()).withHeader("x-amz-trailer", equalTo(CRC32_TRAILER.headerName())));
117+
verify(putRequestedFor(anyUrl()).withHeader("x-amz-decoded-content-length", equalTo("3")));
118+
verify(putRequestedFor(anyUrl()).withHeader("x-amz-content-sha256", equalTo("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
119+
)));
120+
verify(putRequestedFor(anyUrl()).withHeader("Content-Encoding", equalTo("aws-chunked")));
121+
verify(putRequestedFor(anyUrl()).withHeader("Content-Encoding", equalTo("deflate")));
122+
verify(putRequestedFor(anyUrl()).withRequestBody(containing("x-amz-checksum-crc32:NSRBwg=="))
123+
.withRequestBody(containing("x-amz-trailer-signature:"))
124+
.withRequestBody(containing("0;")));
125+
}
126+
105127
@Test
106128
public void payloadSigningWithNoChecksum() {
107129

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/utils/CaptureChecksumValidationInterceptor.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ public class CaptureChecksumValidationInterceptor implements ExecutionIntercepto
2727
private ChecksumValidation responseValidation;
2828
private String requestChecksumInTrailer;
2929
private String requestChecksumInHeader;
30+
private String contentEncoding;
31+
32+
public String contentEncoding() {
33+
return contentEncoding;
34+
}
3035

3136
public Algorithm validationAlgorithm() {
3237
return validationAlgorithm;
@@ -64,6 +69,7 @@ public void afterExecution(Context.AfterExecution context, ExecutionAttributes e
6469
executionAttributes.getOptionalAttribute(SdkExecutionAttribute.HTTP_CHECKSUM_VALIDATION_ALGORITHM).orElse(null);
6570
responseValidation =
6671
executionAttributes.getOptionalAttribute(SdkExecutionAttribute.HTTP_RESPONSE_CHECKSUM_VALIDATION).orElse(null);
72+
contentEncoding = String.join(",", context.httpRequest().matchingHeaders("content-encoding"));
6773
}
6874

6975
@Override

‎test/codegen-generated-classes-test/src/main/resources/codegen-resources/customresponsemetadata/service-2.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -907,6 +907,11 @@
907907
"location":"header",
908908
"locationName":"x-amz-checksum-mode"
909909
},
910+
"ContentEncoding":{
911+
"shape":"String",
912+
"location":"header",
913+
"locationName":"Content-Encoding"
914+
},
910915
"ChecksumAlgorithm":{
911916
"shape":"ChecksumAlgorithm",
912917
"location":"header",
@@ -925,6 +930,11 @@
925930
"location":"header",
926931
"locationName":"x-amz-checksum-mode"
927932
},
933+
"ContentEncoding":{
934+
"shape":"String",
935+
"location":"header",
936+
"locationName":"Content-Encoding"
937+
},
928938
"ChecksumAlgorithm":{
929939
"shape":"ChecksumAlgorithm",
930940
"location":"header",

0 commit comments

Comments
 (0)
0