From a4734c7551694dd90a71aa5d1e4818ec9ae7b47f Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:30:46 -0500 Subject: [PATCH 1/8] fix: create JMESPATH flattenIfPossible functions for Lists (#1169) --- .../smithy/kotlin/runtime/util/JMESPath.kt | 10 ++++ .../kotlin/runtime/util/JmesPathTest.kt | 49 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/util/JmesPathTest.kt diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt index 2a7cdd9791..9b691a79a2 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt @@ -66,6 +66,7 @@ public fun Any?.type(): String = when (this) { else -> throw Exception("Undetected type for: $this") } +// Collection `flattenIfPossible` functions @InternalApi @JvmName("noOpUnnestedCollection") public inline fun Collection.flattenIfPossible(): Collection = this @@ -73,3 +74,12 @@ public inline fun Collection.flattenIfPossible(): Collection = @InternalApi @JvmName("flattenNestedCollection") public inline fun Collection>.flattenIfPossible(): Collection = flatten() + +// List `flattenIfPossible` functions +@InternalApi +@JvmName("noOpUnnestedCollection") +public inline fun List.flattenIfPossible(): List = this + +@InternalApi +@JvmName("flattenNestedCollection") +public inline fun List>.flattenIfPossible(): List = flatten() diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/util/JmesPathTest.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/util/JmesPathTest.kt new file mode 100644 index 0000000000..e867f1e8a1 --- /dev/null +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/util/JmesPathTest.kt @@ -0,0 +1,49 @@ +package aws.smithy.kotlin.runtime.util + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class JmesPathTest { + @Test + fun flattenNestedLists() { + val nestedList = listOf( + listOf(1, 2, 3), + listOf(4, 5), + listOf(6), + ) + val flattenedList = nestedList.flattenIfPossible() + assertEquals(listOf(1, 2, 3, 4, 5, 6), flattenedList) + } + + @Test + fun flattenEmptyNestedLists() { + val nestedList = listOf( + listOf(), + listOf(), + listOf(), + ) + val flattenedList = nestedList.flattenIfPossible() + assertTrue(flattenedList.isEmpty()) + } + + @Test + fun flattenNestedEmptyAndNonEmptyNestedLists() { + val nestedList = listOf( + listOf(1, 2), + listOf(), + listOf(3, 4, 5), + ) + val flattenedList = nestedList.flattenIfPossible() + assertEquals(listOf(1, 2, 3, 4, 5), flattenedList) + } + + @Test + fun flattenList() { + val nestedList = listOf( + listOf(1, 2, 3), + ) + val flattenedList = nestedList.flattenIfPossible() + assertEquals(listOf(1, 2, 3), flattenedList) + } +} From 7911a94dd5f6ee27f2652aeb7c8031e2dce79215 Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Mon, 16 Dec 2024 16:51:49 +0000 Subject: [PATCH 2/8] chore: release 1.3.30 --- CHANGELOG.md | 2 ++ gradle.properties | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 892e2b1248..5633a30d73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [1.3.30] - 12/16/2024 + ## [1.3.29] - 12/12/2024 ## [1.3.28] - 12/03/2024 diff --git a/gradle.properties b/gradle.properties index f35cfc1d5a..377ff6666b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.30-SNAPSHOT +sdkVersion=1.3.30 # codegen -codegenVersion=0.33.30-SNAPSHOT \ No newline at end of file +codegenVersion=0.33.30 \ No newline at end of file From 7085c8ab0d4b34d3ddb77bbb31a2fd41c47c288c Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Mon, 16 Dec 2024 16:51:50 +0000 Subject: [PATCH 3/8] chore: bump snapshot version to 1.3.31-SNAPSHOT --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 377ff6666b..3628520e5a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.30 +sdkVersion=1.3.31-SNAPSHOT # codegen -codegenVersion=0.33.30 \ No newline at end of file +codegenVersion=0.33.31-SNAPSHOT \ No newline at end of file From 9b624ac6d4a258226067b44d0f4ceabea0ab3ad7 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 17 Dec 2024 11:56:42 -0500 Subject: [PATCH 4/8] gzip init --- .../aws/smithy/kotlin/runtime/compression/GzipNative.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt index a9fc036399..6c96909006 100644 --- a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt +++ b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt @@ -5,6 +5,7 @@ package aws.smithy.kotlin.runtime.compression import aws.smithy.kotlin.runtime.content.ByteStream +import platform.zlib.* /** * The gzip compression algorithm. @@ -13,10 +14,8 @@ import aws.smithy.kotlin.runtime.content.ByteStream * See: https://en.wikipedia.org/wiki/Gzip */ public actual class Gzip : CompressionAlgorithm { - actual override val id: String - get() = TODO("Not yet implemented") - actual override val contentEncoding: String - get() = TODO("Not yet implemented") + actual override val id: String = "gzip" + actual override val contentEncoding: String = "gzip" actual override fun compress(stream: ByteStream): ByteStream { TODO("Not yet implemented") From 45750232626bbe4e279142e486be8bbbfccab33f Mon Sep 17 00:00:00 2001 From: Matas Date: Wed, 18 Dec 2024 14:20:08 -0500 Subject: [PATCH 5/8] misc: enhance support for replayable instances of `InputStream` (#1197) --- .../2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json | 8 ++++++ .../kotlin/runtime/content/ByteStreamJVM.kt | 23 +++++++++++++-- .../runtime/content/ByteStreamJVMTest.kt | 28 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json diff --git a/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json b/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json new file mode 100644 index 0000000000..fb16fa5843 --- /dev/null +++ b/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json @@ -0,0 +1,8 @@ +{ + "id": "2de8162f-e5a0-4618-b00d-8da3cfdbc6a2", + "type": "feature", + "description": "Enhance support for replayable instances of `InputStream`", + "issues": [ + "https://github.com/awslabs/aws-sdk-kotlin/issues/1473" + ] +} \ No newline at end of file diff --git a/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt b/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt index 1d6124357a..5647ac15af 100644 --- a/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt +++ b/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt @@ -114,11 +114,30 @@ public fun ByteStream.Companion.fromInputStream( * @param contentLength If specified, indicates how many bytes remain in this stream. Defaults to `null`. */ public fun InputStream.asByteStream(contentLength: Long? = null): ByteStream.SourceStream { - val source = source() + if (markSupported() && contentLength != null) { + mark(contentLength.toInt()) + } + return object : ByteStream.SourceStream() { override val contentLength: Long? = contentLength override val isOneShot: Boolean = !markSupported() - override fun readFrom(): SdkSource = source + override fun readFrom(): SdkSource { + if (markSupported() && contentLength != null) { + reset() + mark(contentLength.toInt()) + return object : SdkSource by source() { + /* + * This is a no-op close to prevent body hashing from closing the underlying InputStream, which causes + * `IOException: Stream closed` on subsequent reads. Consider making [ByteStream.ChannelStream]/[ByteStream.SourceStream] + * (or possibly even [ByteStream] itself) implement [Closeable] to better handle closing streams. + * This should allow us to clean up our usage of [ByteStream.cancel()]. + */ + override fun close() { } + } + } + + return source() + } } } diff --git a/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt b/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt index 054387b2e5..e8324fb11f 100644 --- a/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt +++ b/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt @@ -5,8 +5,11 @@ package aws.smithy.kotlin.runtime.content +import aws.smithy.kotlin.runtime.io.readToByteArray import aws.smithy.kotlin.runtime.testing.RandomTempFile import kotlinx.coroutines.test.runTest +import java.io.BufferedInputStream +import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.InputStream import java.io.OutputStream @@ -228,6 +231,31 @@ class ByteStreamJVMTest { assertFalse(sos.closed) } + // https://github.com/awslabs/aws-sdk-kotlin/issues/1473 + @Test + fun testReplayableInputStreamAsByteStream() = runTest { + val content = "Hello, Bytes!".encodeToByteArray() + val byteArrayIns = ByteArrayInputStream(content) + val nonReplayableIns = NonReplayableInputStream(byteArrayIns) + + // buffer the non-replayable stream, making it replayable... + val bufferedIns = BufferedInputStream(nonReplayableIns) + + val byteStream = bufferedIns.asByteStream(content.size.toLong()) + + // Test that it can be read at least twice (e.g. once for hashing the body, once for transmitting the body) + assertContentEquals(content, byteStream.readFrom().use { it.readToByteArray() }) + assertContentEquals(content, byteStream.readFrom().use { it.readToByteArray() }) + } + + private class NonReplayableInputStream(val inputStream: InputStream) : InputStream() { + override fun markSupported(): Boolean = false // not replayable + + override fun read(): Int = inputStream.read() + override fun mark(readlimit: Int) = inputStream.mark(readlimit) + override fun reset() = inputStream.reset() + } + private class StatusTrackingOutputStream(val os: OutputStream) : OutputStream() { var closed: Boolean = false From b0a4bacc3626bf2c9e8cfe9607ef8d5ece03ee4e Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Wed, 18 Dec 2024 19:23:06 +0000 Subject: [PATCH 6/8] chore: release 1.3.31 --- .changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json | 8 -------- CHANGELOG.md | 5 +++++ gradle.properties | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) delete mode 100644 .changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json diff --git a/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json b/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json deleted file mode 100644 index fb16fa5843..0000000000 --- a/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id": "2de8162f-e5a0-4618-b00d-8da3cfdbc6a2", - "type": "feature", - "description": "Enhance support for replayable instances of `InputStream`", - "issues": [ - "https://github.com/awslabs/aws-sdk-kotlin/issues/1473" - ] -} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5633a30d73..5473a56d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [1.3.31] - 12/18/2024 + +### Features +* [#1473](https://github.com/awslabs/aws-sdk-kotlin/issues/1473) Enhance support for replayable instances of `InputStream` + ## [1.3.30] - 12/16/2024 ## [1.3.29] - 12/12/2024 diff --git a/gradle.properties b/gradle.properties index 3628520e5a..cf967edd0b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.31-SNAPSHOT +sdkVersion=1.3.31 # codegen -codegenVersion=0.33.31-SNAPSHOT \ No newline at end of file +codegenVersion=0.33.31 \ No newline at end of file From 80f453814d8b0c4e20ef7e729d47a364ca8b5753 Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Wed, 18 Dec 2024 19:23:08 +0000 Subject: [PATCH 7/8] chore: bump snapshot version to 1.3.32-SNAPSHOT --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index cf967edd0b..f902a36719 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.31 +sdkVersion=1.3.32-SNAPSHOT # codegen -codegenVersion=0.33.31 \ No newline at end of file +codegenVersion=0.33.32-SNAPSHOT \ No newline at end of file From e9d16a98b8030468e35a05f63c79cae2444e6759 Mon Sep 17 00:00:00 2001 From: Matas Date: Wed, 18 Dec 2024 16:28:30 -0500 Subject: [PATCH 8/8] fix: CBOR protocol test assertions / blob serialization (#1198) --- .changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json | 5 +++++ .../protocol/HttpProtocolUnitTestRequestGenerator.kt | 4 ++-- .../codegen/rendering/serde/SerializeStructGenerator.kt | 1 - .../codegen/rendering/serde/SerializeStructGeneratorTest.kt | 2 +- .../src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 .changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json diff --git a/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json b/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json new file mode 100644 index 0000000000..0fad5e0349 --- /dev/null +++ b/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json @@ -0,0 +1,5 @@ +{ + "id": "1021e75a-45f3-4f3a-820c-700d9ec6e782", + "type": "bugfix", + "description": "Fix serialization of CBOR blobs" +} \ No newline at end of file diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt index 74bd7db782..6fcc47c3e5 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt @@ -117,11 +117,11 @@ open class HttpProtocolUnitTestRequestGenerator protected constructor(builder: B write("return") } write("requireNotNull(expectedBytes) { #S }", "expected application/cbor body cannot be null") - write("requireNotNull(expectedBytes) { #S }", "actual application/cbor body cannot be null") + write("requireNotNull(actualBytes) { #S }", "actual application/cbor body cannot be null") write("") write("val expectedRequest = #L(#T(expectedBytes))", inputDeserializer.name, RuntimeTypes.Serde.SerdeCbor.CborDeserializer) - write("val actualRequest = #L(#T(expectedBytes))", inputDeserializer.name, RuntimeTypes.Serde.SerdeCbor.CborDeserializer) + write("val actualRequest = #L(#T(actualBytes))", inputDeserializer.name, RuntimeTypes.Serde.SerdeCbor.CborDeserializer) write("assertEquals(expectedRequest, actualRequest)") } writer.write("") diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt index e7f2e261ef..d8810044a5 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt @@ -647,7 +647,6 @@ open class SerializeStructGenerator( val target = member.targetOrSelf(ctx.model) val encoded = when { - target.type == ShapeType.BLOB -> writer.format("#L.#T()", identifier, RuntimeTypes.Core.Text.Encoding.encodeBase64String) target.type == ShapeType.TIMESTAMP -> { writer.addImport(RuntimeTypes.Core.TimestampFormat) val tsFormat = member diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt index 40d9c9db7d..347da007cb 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt @@ -1822,7 +1822,7 @@ class SerializeStructGeneratorTest { val expected = """ serializer.serializeStruct(OBJ_DESCRIPTOR) { - input.fooBlob?.let { field(FOOBLOB_DESCRIPTOR, it.encodeBase64String()) } + input.fooBlob?.let { field(FOOBLOB_DESCRIPTOR, it) } } """.trimIndent() diff --git a/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt b/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt index 35618cb49e..d1356f848c 100644 --- a/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt +++ b/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt @@ -134,7 +134,7 @@ public class XmlSerializer(private val xmlWriter: XmlStreamWriter = xmlStreamWri field(descriptor, value.format(format)) override fun field(descriptor: SdkFieldDescriptor, value: ByteArray): Unit = - field(descriptor, value) + field(descriptor, value.encodeBase64String()) override fun field(descriptor: SdkFieldDescriptor, value: Document?): Unit = throw SerializationException( "cannot serialize field ${descriptor.serialName}; Document type is not supported by xml encoding",