fix: ignore __type when deserializing union for AWS Json protocols

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt

@@ -233,6 +233,8 @@ object RuntimeTypes {
         val asSdkSerializable = symbol("asSdkSerializable")
         val field = symbol("field")
 
+        val IgnoreKey = symbol("IgnoreKey")
+
         object SerdeJson : RuntimeTypePackage(KotlinDependency.SERDE_JSON) {
             val JsonSerialName = symbol("JsonSerialName")
             val JsonSerializer = symbol("JsonSerializer")

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/JsonParserGenerator.kt

@@ -24,6 +24,13 @@ open class JsonParserGenerator(
 
     open val defaultTimestampFormat: TimestampFormatTrait.Format = TimestampFormatTrait.Format.EPOCH_SECONDS
 
+    open fun descriptorGenerator(
+        ctx: ProtocolGenerator.GenerationContext,
+        shape: Shape,
+        members: List<MemberShape>,
+        writer: KotlinWriter,
+    ): JsonSerdeDescriptorGenerator = JsonSerdeDescriptorGenerator(ctx.toRenderingContext(protocolGenerator, shape, writer), members, supportsJsonNameTrait)
+
     override fun operationDeserializer(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, members: List<MemberShape>): Symbol {
         val outputSymbol = op.output.get().let { ctx.symbolProvider.toSymbol(ctx.model.expectShape(it)) }
         return op.bodyDeserializer(ctx.settings) { writer ->
@@ -127,7 +134,7 @@ open class JsonParserGenerator(
         members: List<MemberShape>,
         writer: KotlinWriter,
     ) {
-        JsonSerdeDescriptorGenerator(ctx.toRenderingContext(protocolGenerator, shape, writer), members, supportsJsonNameTrait).render()
+        descriptorGenerator(ctx, shape, members, writer).render()
         if (shape.isUnionShape) {
             val name = ctx.symbolProvider.toSymbol(shape).name
             DeserializeUnionGenerator(ctx, name, members, writer, defaultTimestampFormat).render()

runtime/serde/common/src/aws/smithy/kotlin/runtime/serde/SdkFieldDescriptor.kt

@@ -6,23 +6,6 @@ package aws.smithy.kotlin.runtime.serde
 
 import aws.smithy.kotlin.runtime.InternalApi
 
-/**
- * This tag interface provides a mechanism to attach type-specific metadata to any field.
- * See [aws.smithy.kotlin.runtime.serde.xml.XmlList] for an example implementation.
- *
- * For example, to specify that a list should be serialized in XML such that values are wrapped
- * in a tag called "boo", pass an instance of XmlList to the FieldDescriptor of `XmlList(elementName="boo")`.
- */
-@InternalApi
-public interface FieldTrait
-
-/**
- * Denotes that a Map or List may contain null values
- * Details at https://awslabs.github.io/smithy/1.0/spec/core/type-refinement-traits.html#sparse-trait
- */
-@InternalApi
-public object SparseValues : FieldTrait
-
 /**
  * A protocol-agnostic type description of a field.
  */

runtime/serde/common/src/aws/smithy/kotlin/runtime/serde/SerdeTraits.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.smithy.kotlin.runtime.serde
+
+import aws.smithy.kotlin.runtime.InternalApi
+
+/**
+ * This tag interface provides a mechanism to attach type-specific metadata to any field.
+ * See [aws.smithy.kotlin.runtime.serde.xml.XmlList] for an example implementation.
+ *
+ * For example, to specify that a list should be serialized in XML such that values are wrapped
+ * in a tag called "boo", pass an instance of XmlList to the FieldDescriptor of `XmlList(elementName="boo")`.
+ */
+@InternalApi
+public interface FieldTrait
+
+/**
+ * Indicates to deserializers to ignore field/key
+ *
+ * @param key The key to ignore in the payload
+ * @param regardlessOfInModel If true will ignore key even though the model indicates key should be there
+ *
+ */
+@InternalApi
+public data class IgnoreKey(public val key: String, public val regardlessOfInModel: Boolean = true) : FieldTrait
+
+/**
+ * Denotes that a Map or List may contain null values
+ * Details at https://awslabs.github.io/smithy/1.0/spec/core/type-refinement-traits.html#sparse-trait
+ */
+@InternalApi
+public object SparseValues : FieldTrait

runtime/serde/serde-json/common/src/aws/smithy/kotlin/runtime/serde/json/JsonDeserializer.kt

@@ -198,7 +198,20 @@ private class JsonFieldIterator(
                 val token = reader.nextTokenOf<JsonToken.Name>()
                 val propertyName = token.value
                 val field = descriptor.fields.find { it.serialName == propertyName }
-                field?.index ?: Deserializer.FieldIterator.UNKNOWN_FIELD
+
+                if (descriptor.traits.contains(IgnoreKey(propertyName, false))) {
+                    if (field == null) { // not in the model
+                        reader.skipNext()
+                        return findNextFieldIndex()
+                    } else {
+                        field.index
+                    }
+                } else if (descriptor.traits.contains(IgnoreKey(propertyName, true))) {
+                    reader.skipNext() // the value of the ignored key
+                    return findNextFieldIndex()
+                } else {
+                    field?.index ?: Deserializer.FieldIterator.UNKNOWN_FIELD
+                }
             }
         }
 

runtime/serde/serde-json/common/test/aws/smithy/kotlin/runtime/serde/json/JsonDeserializerIgnoresKeysTest.kt

@@ -0,0 +1,245 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.smithy.kotlin.runtime.serde.json
+
+import aws.smithy.kotlin.runtime.serde.*
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class JsonDeserializerIgnoresKeysTest {
+    class IgnoresKeysTest {
+        companion object {
+            val X_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("x"))
+            val Y_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("y"))
+            val Z_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("z"))
+            val OBJ_DESCRIPTOR = SdkObjectDescriptor.build {
+                trait(IgnoreKey("z")) // <----
+                field(X_DESCRIPTOR)
+                field(Y_DESCRIPTOR)
+                field(Z_DESCRIPTOR)
+            }
+        }
+    }
+
+    @Test
+    fun itIgnoresKeys() {
+        val payload = """
+        {
+            "x": 1,
+            "y": 2,
+            "z": 3
+        }
+        """.trimIndent().encodeToByteArray()
+
+        val deserializer = JsonDeserializer(payload)
+        var x: Int? = null
+        var y: Int? = null
+        var z: Int? = null
+        deserializer.deserializeStruct(IgnoresKeysTest.OBJ_DESCRIPTOR) {
+            loop@ while (true) {
+                when (findNextFieldIndex()) {
+                    IgnoresKeysTest.X_DESCRIPTOR.index -> x = deserializeInt()
+                    IgnoresKeysTest.Y_DESCRIPTOR.index -> y = deserializeInt()
+                    IgnoresKeysTest.Z_DESCRIPTOR.index -> z = deserializeInt()
+                    null -> break@loop
+                }
+            }
+        }
+
+        assertEquals(1, x)
+        assertEquals(2, y)
+        assertEquals(null, z)
+    }
+
+    @Test
+    fun itIgnoresKeysOutOfOrder() {
+        val payload = """
+        {
+            "z": 3,
+            "x": 1,
+            "y": 2
+        }
+        """.trimIndent().encodeToByteArray()
+
+        val deserializer = JsonDeserializer(payload)
+        var x: Int? = null
+        var y: Int? = null
+        var z: Int? = null
+        deserializer.deserializeStruct(IgnoresKeysTest.OBJ_DESCRIPTOR) {
+            loop@ while (true) {
+                when (findNextFieldIndex()) {
+                    IgnoresKeysTest.X_DESCRIPTOR.index -> x = deserializeInt()
+                    IgnoresKeysTest.Y_DESCRIPTOR.index -> y = deserializeInt()
+                    IgnoresKeysTest.Z_DESCRIPTOR.index -> z = deserializeInt()
+                    null -> break@loop
+                }
+            }
+        }
+
+        assertEquals(1, x)
+        assertEquals(2, y)
+        assertEquals(null, z)
+    }
+
+    @Test
+    fun itIgnoresKeysManyTimes() {
+        val payload = """
+        {
+            "x": 1,
+            "y": 2,
+            "z": 3,
+            "z": 3,
+            "z": 3
+        }
+        """.trimIndent().encodeToByteArray()
+
+        val deserializer = JsonDeserializer(payload)
+        var x: Int? = null
+        var y: Int? = null
+        var z: Int? = null
+        deserializer.deserializeStruct(IgnoresKeysTest.OBJ_DESCRIPTOR) {
+            loop@ while (true) {
+                when (findNextFieldIndex()) {
+                    IgnoresKeysTest.X_DESCRIPTOR.index -> x = deserializeInt()
+                    IgnoresKeysTest.Y_DESCRIPTOR.index -> y = deserializeInt()
+                    IgnoresKeysTest.Z_DESCRIPTOR.index -> z = deserializeInt()
+                    null -> break@loop
+                }
+            }
+        }
+
+        assertEquals(1, x)
+        assertEquals(2, y)
+        assertEquals(null, z)
+    }
+
+    class IgnoresMultipleKeysTest {
+        companion object {
+            val W_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("w"))
+            val X_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("x"))
+            val Y_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("y"))
+            val Z_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("z"))
+            val OBJ_DESCRIPTOR = SdkObjectDescriptor.build {
+                trait(IgnoreKey("w")) // <----
+                trait(IgnoreKey("z")) // <----
+                field(W_DESCRIPTOR)
+                field(X_DESCRIPTOR)
+                field(Y_DESCRIPTOR)
+                field(Z_DESCRIPTOR)
+            }
+        }
+    }
+
+    @Test
+    fun itIgnoresMultipleKeys() {
+        val payload = """
+        {
+            "w": 0,
+            "x": 1,
+            "y": 2,
+            "z": 3
+        }
+        """.trimIndent().encodeToByteArray()
+
+        val deserializer = JsonDeserializer(payload)
+        var w: Int? = null
+        var x: Int? = null
+        var y: Int? = null
+        var z: Int? = null
+        deserializer.deserializeStruct(IgnoresMultipleKeysTest.OBJ_DESCRIPTOR) {
+            loop@ while (true) {
+                when (findNextFieldIndex()) {
+                    IgnoresMultipleKeysTest.W_DESCRIPTOR.index -> w = deserializeInt()
+                    IgnoresMultipleKeysTest.X_DESCRIPTOR.index -> x = deserializeInt()
+                    IgnoresMultipleKeysTest.Y_DESCRIPTOR.index -> y = deserializeInt()
+                    IgnoresMultipleKeysTest.Z_DESCRIPTOR.index -> z = deserializeInt()
+                    null -> break@loop
+                }
+            }
+        }
+
+        assertEquals(null, w)
+        assertEquals(1, x)
+        assertEquals(2, y)
+        assertEquals(null, z)
+    }
+
+    // Now testing `regardlessOfInModel = false` option
+    class IgnoresKeysConsideringModelTest {
+        companion object {
+            val X_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("x"))
+            val Y_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("y"))
+            val Z_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("z"))
+            val OBJ_DESCRIPTOR = SdkObjectDescriptor.build {
+                trait(IgnoreKey("x", false)) // <----
+                field(X_DESCRIPTOR)
+                field(Y_DESCRIPTOR)
+                field(Z_DESCRIPTOR)
+            }
+        }
+    }
+
+    @Test
+    fun itDoesNotIgnoreKeyBecauseItWasInTheModel() {
+        val payload = """
+        {
+            "x": 0
+        }
+        """.trimIndent().encodeToByteArray()
+
+        val deserializer = JsonDeserializer(payload)
+        var unionValue: Int? = null
+        deserializer.deserializeStruct(IgnoresKeysConsideringModelTest.OBJ_DESCRIPTOR) {
+            loop@ while (true) {
+                when (findNextFieldIndex()) {
+                    IgnoresKeysConsideringModelTest.X_DESCRIPTOR.index -> unionValue = deserializeInt()
+                    IgnoresKeysConsideringModelTest.Y_DESCRIPTOR.index -> unionValue = deserializeInt()
+                    IgnoresKeysConsideringModelTest.Z_DESCRIPTOR.index -> unionValue = deserializeInt()
+                    null -> break@loop
+                    else -> unionValue = deserializeInt()
+                }
+            }
+        }
+
+        assertEquals(0, unionValue)
+    }
+
+    class IgnoresKeysBecauseNotInModelTest {
+        companion object {
+            val Y_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("y"))
+            val Z_DESCRIPTOR = SdkFieldDescriptor(SerialKind.Integer, JsonSerialName("z"))
+            val OBJ_DESCRIPTOR = SdkObjectDescriptor.build {
+                trait(IgnoreKey("x", false)) // <----
+                field(Y_DESCRIPTOR)
+                field(Z_DESCRIPTOR)
+            }
+        }
+    }
+
+    @Test
+    fun itIgnoresKeyBecauseWasNotInTheModel() {
+        val payload = """
+        {
+            "x": 0
+        }
+        """.trimIndent().encodeToByteArray()
+
+        val deserializer = JsonDeserializer(payload)
+        var unionValue: Int? = null
+        deserializer.deserializeStruct(IgnoresKeysBecauseNotInModelTest.OBJ_DESCRIPTOR) {
+            loop@ while (true) {
+                when (findNextFieldIndex()) {
+                    IgnoresKeysBecauseNotInModelTest.Y_DESCRIPTOR.index -> unionValue = deserializeInt()
+                    IgnoresKeysBecauseNotInModelTest.Z_DESCRIPTOR.index -> unionValue = deserializeInt()
+                    null -> break@loop
+                    else -> unionValue = deserializeInt()
+                }
+            }
+        }
+
+        assertEquals(null, unionValue)
+    }
+}