From 60dee0f32bce6d0b73bbd9880d8aa577d8a84c5b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 26 Dec 2023 10:48:58 -0600 Subject: [PATCH 01/10] remove unused val --- .../aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt index 4d62b0b1d..f1db98d26 100644 --- a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt +++ b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt @@ -193,9 +193,6 @@ internal class DefaultCanonicalizer(private val sha256Supplier: HashSupplier = : } } -/** The number of bytes to read at a time during SHA256 calculation on streaming bodies. */ -private const val STREAM_CHUNK_BYTES = 16384 // 16KB - /** * Canonicalizes a path from this [Url.Builder]. * @param config The signing configuration to use From 42e07698a7d311f006f18289a42dcf1fa9e1d270 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 26 Dec 2023 10:49:55 -0600 Subject: [PATCH 02/10] Copy flexible checksums SHA256 to signing config HashSpecification --- .../aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt index ebd1a465b..70bd4d3bf 100644 --- a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt +++ b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt @@ -18,6 +18,8 @@ import aws.smithy.kotlin.runtime.http.HttpBody import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder +import aws.smithy.kotlin.runtime.text.encoding.decodeBase64Bytes +import aws.smithy.kotlin.runtime.text.encoding.encodeToHex import aws.smithy.kotlin.runtime.time.Instant import kotlin.time.Duration @@ -163,6 +165,12 @@ public class AwsHttpSigner(private val config: Config) : HttpSigner { HashSpecification.StreamingAws4HmacSha256Payload } } + request.headers["x-amz-checksum-sha256"] != null -> { + // If it's set, reuse flexible checksums SHA256 + HashSpecification.Precalculated( + request.headers["x-amz-checksum-sha256"]!!.decodeBase64Bytes().encodeToHex() + ) + } config.isUnsignedPayload -> HashSpecification.UnsignedPayload // use the payload to compute the hash else -> HashSpecification.CalculateFromPayload From bd59c5da612c1670cab703420bdc21de07b5bd63 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 26 Dec 2023 11:05:43 -0600 Subject: [PATCH 03/10] update comment --- .../src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt index 70bd4d3bf..977f80d50 100644 --- a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt +++ b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt @@ -166,7 +166,7 @@ public class AwsHttpSigner(private val config: Config) : HttpSigner { } } request.headers["x-amz-checksum-sha256"] != null -> { - // If it's set, reuse flexible checksums SHA256 + // Re-use flexible checksums SHA256 if it's set HashSpecification.Precalculated( request.headers["x-amz-checksum-sha256"]!!.decodeBase64Bytes().encodeToHex() ) From db302c129490948984e8eaebc3eb0289bc4c3d96 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 26 Dec 2023 11:06:30 -0600 Subject: [PATCH 04/10] changelog --- .changes/0b435166-944d-4836-8683-964034b823cc.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/0b435166-944d-4836-8683-964034b823cc.json diff --git a/.changes/0b435166-944d-4836-8683-964034b823cc.json b/.changes/0b435166-944d-4836-8683-964034b823cc.json new file mode 100644 index 000000000..3a4ba0905 --- /dev/null +++ b/.changes/0b435166-944d-4836-8683-964034b823cc.json @@ -0,0 +1,5 @@ +{ + "id": "0b435166-944d-4836-8683-964034b823cc", + "type": "feature", + "description": "Re-use SHA256 checksum during signing" +} \ No newline at end of file From c591554cad6b56934a30c4774562329397646d85 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 26 Dec 2023 11:09:55 -0600 Subject: [PATCH 05/10] ktlint --- .../src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt index 977f80d50..88b8c5db8 100644 --- a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt +++ b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt @@ -168,7 +168,7 @@ public class AwsHttpSigner(private val config: Config) : HttpSigner { request.headers["x-amz-checksum-sha256"] != null -> { // Re-use flexible checksums SHA256 if it's set HashSpecification.Precalculated( - request.headers["x-amz-checksum-sha256"]!!.decodeBase64Bytes().encodeToHex() + request.headers["x-amz-checksum-sha256"]!!.decodeBase64Bytes().encodeToHex(), ) } config.isUnsignedPayload -> HashSpecification.UnsignedPayload From 615f9c2fba6cad72b075949bacb5c6c5a173a13a Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 27 Dec 2023 15:21:15 -0600 Subject: [PATCH 06/10] Don't require contentLength for aws-chunked streaming --- .../kotlin/runtime/auth/awssigning/internal/AwsChunkedUtil.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedUtil.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedUtil.kt index c37c85a01..8b6a1c8a6 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedUtil.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedUtil.kt @@ -43,8 +43,8 @@ internal fun SdkBuffer.writeTrailerSignature(signature: String) { */ @InternalApi public val HttpBody.isEligibleForAwsChunkedStreaming: Boolean - get() = (this is HttpBody.SourceContent || this is HttpBody.ChannelContent) && contentLength != null && - (isOneShot || contentLength!! > AWS_CHUNKED_THRESHOLD) + get() = (this is HttpBody.SourceContent || this is HttpBody.ChannelContent) && + (isOneShot || (contentLength?.compareTo(AWS_CHUNKED_THRESHOLD) ?: 0) > 0) /** * @return a boolean representing if the signing configuration is configured (via [HashSpecification]) for aws-chunked content encoding From cf34645051e24f58f8fc92675e1304364e517148 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 27 Dec 2023 16:22:05 -0600 Subject: [PATCH 07/10] Remove old optimization --- .../aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt index 88b8c5db8..2d395d37d 100644 --- a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt +++ b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt @@ -165,12 +165,6 @@ public class AwsHttpSigner(private val config: Config) : HttpSigner { HashSpecification.StreamingAws4HmacSha256Payload } } - request.headers["x-amz-checksum-sha256"] != null -> { - // Re-use flexible checksums SHA256 if it's set - HashSpecification.Precalculated( - request.headers["x-amz-checksum-sha256"]!!.decodeBase64Bytes().encodeToHex(), - ) - } config.isUnsignedPayload -> HashSpecification.UnsignedPayload // use the payload to compute the hash else -> HashSpecification.CalculateFromPayload From ab7750dbc71b537d81374fd29e13a81709e14f1b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 27 Dec 2023 16:26:07 -0600 Subject: [PATCH 08/10] ktlint --- .../src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt index 2d395d37d..ebd1a465b 100644 --- a/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt +++ b/runtime/auth/http-auth-aws/common/src/aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner.kt @@ -18,8 +18,6 @@ import aws.smithy.kotlin.runtime.http.HttpBody import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder -import aws.smithy.kotlin.runtime.text.encoding.decodeBase64Bytes -import aws.smithy.kotlin.runtime.text.encoding.encodeToHex import aws.smithy.kotlin.runtime.time.Instant import kotlin.time.Duration From 8251f58f31364eb6d7cf0b0a14b3f7a55a399322 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 27 Dec 2023 16:45:07 -0600 Subject: [PATCH 09/10] copy logic to interceptor too --- .../http/interceptors/FlexibleChecksumsRequestInterceptor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsRequestInterceptor.kt b/runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsRequestInterceptor.kt index 7ee18cdf7..8eff9f98a 100644 --- a/runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsRequestInterceptor.kt +++ b/runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsRequestInterceptor.kt @@ -114,8 +114,8 @@ public class FlexibleChecksumsRequestInterceptor( // FIXME this duplicates the logic from aws-signing-common, but can't import from there due to circular import. private val HttpBody.isEligibleForAwsChunkedStreaming: Boolean - get() = (this is HttpBody.SourceContent || this is HttpBody.ChannelContent) && contentLength != null && - (isOneShot || contentLength!! > 65536 * 16) + get() = (this is HttpBody.SourceContent || this is HttpBody.ChannelContent) && + (isOneShot || (contentLength?.compareTo(65_536 * 16) ?: 0) > 0) /** * @return if the [HashFunction] is supported by flexible checksums From ee75a601671c50584e52f6af352775186b8cc25c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 27 Dec 2023 16:45:58 -0600 Subject: [PATCH 10/10] update changelog --- .changes/0b435166-944d-4836-8683-964034b823cc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/0b435166-944d-4836-8683-964034b823cc.json b/.changes/0b435166-944d-4836-8683-964034b823cc.json index 3a4ba0905..04d19b100 100644 --- a/.changes/0b435166-944d-4836-8683-964034b823cc.json +++ b/.changes/0b435166-944d-4836-8683-964034b823cc.json @@ -1,5 +1,5 @@ { "id": "0b435166-944d-4836-8683-964034b823cc", "type": "feature", - "description": "Re-use SHA256 checksum during signing" + "description": "Enable `aws-chunked` content encoding for streaming requests without a content length" } \ No newline at end of file