diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d463bff2c..7999eda95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,8 +140,8 @@ jobs: env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} PGP_SECRET: ${{ secrets.PGP_SECRET }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.S01_SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.S01_SONATYPE_USERNAME }} microsite: name: Publish Docs Microsite diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f201a404..61e644bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ Previously they'd be named after the **member target**, now they will use the na There's usually only one instance of `EncoderK[F, A]` for a particular `F[_]`, and interpreters don't need to know what `A` is. For convenience, the type parameter has been moved to a type member. +# 0.18.24 + +* Adds missing nanoseconds in Document encoding of EPOCH_SECOND timestamps +* Add support for `alloy#jsonUnknown`, allowing structures to capture unknown JSON fields in one of their members. +* Add `getMessage` implementation in `Smithy4sThrowable` which will be overridden in cases where the error structure contains a message field, but otherwise will be used to prevent a useless `null` result when `getMessage` is called. + # 0.18.23 ## Validated newtypes [#1454](https://github.com/disneystreaming/smithy4s/pull/1454) diff --git a/modules/bootstrapped/resources/smithy4s.example.DiscriminatedService.json b/modules/bootstrapped/resources/smithy4s.example.DiscriminatedService.json index 46689ed95..37ee95dd2 100644 --- a/modules/bootstrapped/resources/smithy4s.example.DiscriminatedService.json +++ b/modules/bootstrapped/resources/smithy4s.example.DiscriminatedService.json @@ -54,52 +54,51 @@ "TestBiggerUnion": { "oneOf": [ { - "allOf": [ - { - "$ref": "#/components/schemas/One" - }, - { - "type": "object", - "properties": { - "tpe": { - "type": "string", - "enum": [ - "one" - ] - } - }, - "required": [ - "tpe" - ] - } - ] + "$ref": "#/components/schemas/TestBiggerUnionOne" }, { - "allOf": [ - { - "$ref": "#/components/schemas/Two" - }, - { - "type": "object", - "properties": { - "tpe": { - "type": "string", - "enum": [ - "two" - ] - } - }, - "required": [ - "tpe" - ] - } - ] + "$ref": "#/components/schemas/TestBiggerUnionTwo" } ], "discriminator": { - "propertyName": "tpe" + "propertyName": "tpe", + "mapping": { + "one": "#/components/schemas/TestBiggerUnionOne", + "two": "#/components/schemas/TestBiggerUnionTwo" + } } }, + "TestBiggerUnionMixin": { + "type": "object", + "properties": { + "tpe": { + "type": "string" + } + }, + "required": [ + "tpe" + ] + }, + "TestBiggerUnionOne": { + "allOf": [ + { + "$ref": "#/components/schemas/One" + }, + { + "$ref": "#/components/schemas/TestBiggerUnionMixin" + } + ] + }, + "TestBiggerUnionTwo": { + "allOf": [ + { + "$ref": "#/components/schemas/Two" + }, + { + "$ref": "#/components/schemas/TestBiggerUnionMixin" + } + ] + }, "Two": { "type": "object", "properties": { diff --git a/modules/bootstrapped/src/generated/smithy4s/example/AdditionalProperties.scala b/modules/bootstrapped/src/generated/smithy4s/example/AdditionalProperties.scala new file mode 100644 index 000000000..c90b01243 --- /dev/null +++ b/modules/bootstrapped/src/generated/smithy4s/example/AdditionalProperties.scala @@ -0,0 +1,18 @@ +package smithy4s.example + +import smithy4s.Document +import smithy4s.Hints +import smithy4s.Newtype +import smithy4s.Schema +import smithy4s.ShapeId +import smithy4s.schema.Schema.bijection +import smithy4s.schema.Schema.document +import smithy4s.schema.Schema.map +import smithy4s.schema.Schema.string + +object AdditionalProperties extends Newtype[Map[String, Document]] { + val id: ShapeId = ShapeId("smithy4s.example", "AdditionalProperties") + val hints: Hints = Hints.empty + val underlyingSchema: Schema[Map[String, Document]] = map(string, document).withId(id).addHints(hints) + implicit val schema: Schema[AdditionalProperties] = bijection(underlyingSchema, asBijection) +} diff --git a/modules/bootstrapped/src/generated/smithy4s/example/JsonUnknownExample.scala b/modules/bootstrapped/src/generated/smithy4s/example/JsonUnknownExample.scala new file mode 100644 index 000000000..3fc02409a --- /dev/null +++ b/modules/bootstrapped/src/generated/smithy4s/example/JsonUnknownExample.scala @@ -0,0 +1,27 @@ +package smithy4s.example + +import smithy4s.Document +import smithy4s.Hints +import smithy4s.Schema +import smithy4s.ShapeId +import smithy4s.ShapeTag +import smithy4s.schema.Schema.int +import smithy4s.schema.Schema.string +import smithy4s.schema.Schema.struct + +final case class JsonUnknownExample(s: Option[String] = None, i: Option[Int] = None, additionalProperties: Option[Map[String, Document]] = None) + +object JsonUnknownExample extends ShapeTag.Companion[JsonUnknownExample] { + val id: ShapeId = ShapeId("smithy4s.example", "JsonUnknownExample") + + val hints: Hints = Hints.empty + + // constructor using the original order from the spec + private def make(s: Option[String], i: Option[Int], additionalProperties: Option[Map[String, Document]]): JsonUnknownExample = JsonUnknownExample(s, i, additionalProperties) + + implicit val schema: Schema[JsonUnknownExample] = struct( + string.optional[JsonUnknownExample]("s", _.s), + int.optional[JsonUnknownExample]("i", _.i), + AdditionalProperties.underlyingSchema.optional[JsonUnknownExample]("additionalProperties", _.additionalProperties).addHints(alloy.JsonUnknown()), + )(make).withId(id).addHints(hints) +} diff --git a/modules/bootstrapped/src/generated/smithy4s/example/package.scala b/modules/bootstrapped/src/generated/smithy4s/example/package.scala index 52d37ec2c..a914f9cf4 100644 --- a/modules/bootstrapped/src/generated/smithy4s/example/package.scala +++ b/modules/bootstrapped/src/generated/smithy4s/example/package.scala @@ -43,6 +43,7 @@ package object example { /** This is a simple example of a "quoted string" */ type AString = smithy4s.example.AString.Type + type AdditionalProperties = smithy4s.example.AdditionalProperties.Type type Age = smithy4s.example.Age.Type /** Multiple line doc comment for another string * Containing a random \*\/ here. diff --git a/modules/bootstrapped/test/src/smithy4s/DocumentSpec.scala b/modules/bootstrapped/test/src/smithy4s/DocumentSpec.scala index 6b3b8e941..e65e1605f 100644 --- a/modules/bootstrapped/test/src/smithy4s/DocumentSpec.scala +++ b/modules/bootstrapped/test/src/smithy4s/DocumentSpec.scala @@ -20,6 +20,7 @@ import smithy.api.JsonName import smithy.api.Default import smithy4s.example.IntList import alloy.Discriminated +import alloy.JsonUnknown import munit._ import smithy4s.example.DefaultNullsOperationOutput import alloy.Untagged @@ -581,6 +582,31 @@ class DocumentSpec() extends FunSuite { ) } + test("Document encoder - timestamp epoch seconds with nanos") { + val timestampWithNanos = + Timestamp(1716459630L, 500 * 1000 * 1000 /* half a second */ ) + val result = Document.Encoder + .withExplicitDefaultsEncoding(false) + .fromSchema(TimestampOperationInput.schema) + .encode( + TimestampOperationInput( + timestampWithNanos, + timestampWithNanos, + timestampWithNanos + ) + ) + expect.same( + Document.obj( + "httpDate" -> Document.fromString("Thu, 23 May 2024 10:20:30.500 GMT"), + "dateTime" -> Document.fromString("2024-05-23T10:20:30.500Z"), + "epochSeconds" -> Document.fromBigDecimal( + BigDecimal("1716459630.500000") + ) + ), + result + ) + } + test("Document decoder - timestamp defaults") { val doc = Document.obj() val result = Document.Decoder @@ -638,4 +664,239 @@ class DocumentSpec() extends FunSuite { assertEquals(niceSyntaxDocument, expectedDocument) } + case class JsonUnknownExample( + s: String, + i: Int, + others: Map[String, Document] + ) + + object JsonUnknownExample { + implicit val jsonUnknownExampleSchema: Schema[JsonUnknownExample] = { + val s = string.required[JsonUnknownExample]("s", _.s) + val i = int.required[JsonUnknownExample]("i", _.i) + val others = map(string, document) + .required[JsonUnknownExample]("others", _.others) + .addHints(JsonUnknown()) + struct(s, i, others)(JsonUnknownExample.apply) + } + } + + object JsonUnknownExampleWithDefault { + implicit val jsonUnknownExampleSchema: Schema[JsonUnknownExample] = { + val s = string.required[JsonUnknownExample]("s", _.s) + val i = int.required[JsonUnknownExample]("i", _.i) + val others = map(string, document) + .required[JsonUnknownExample]("others", _.others) + .addHints( + JsonUnknown(), + Default(Document.obj("default" -> Document.fromBoolean(true))) + ) + struct(s, i, others)(JsonUnknownExample.apply) + } + } + + case class JsonUnknownExampleOptional( + s: String, + i: Int, + others: Option[Map[String, Document]] + ) + + object JsonUnknownExampleOptional { + implicit val jsonUnknownExampleOptionalSchema + : Schema[JsonUnknownExampleOptional] = { + val s = string.required[JsonUnknownExampleOptional]("s", _.s) + val i = int.required[JsonUnknownExampleOptional]("i", _.i) + val others = map(string, document) + .optional[JsonUnknownExampleOptional]("others", _.others) + .addHints(JsonUnknown()) + struct(s, i, others)(JsonUnknownExampleOptional.apply) + } + } + + object JsonUnknownExampleOptionalWithDefault { + implicit val jsonUnknownExampleOptionalSchema + : Schema[JsonUnknownExampleOptional] = { + val s = string.required[JsonUnknownExampleOptional]("s", _.s) + val i = int.required[JsonUnknownExampleOptional]("i", _.i) + val others = map(string, document) + .optional[JsonUnknownExampleOptional]("others", _.others) + .addHints( + JsonUnknown(), + Default(Document.obj("default" -> Document.fromBoolean(true))) + ) + struct(s, i, others)(JsonUnknownExampleOptional.apply) + } + } + + test("unknown field decoding: no unknown field in payload") { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67) + ) + val expected = JsonUnknownExample("foo", 67, Map.empty) + + val res = Document.Decoder + .fromSchema(JsonUnknownExample.jsonUnknownExampleSchema) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test("unknown field decoding: no unknown field in payload with default") { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67) + ) + val expected = JsonUnknownExample( + "foo", + 67, + Map("default" -> Document.fromBoolean(true)) + ) + + val res = Document.Decoder + .fromSchema(JsonUnknownExampleWithDefault.jsonUnknownExampleSchema) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test( + "unknown field decoding: no unknown field in payload, optional field" + ) { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67) + ) + val expected = JsonUnknownExampleOptional("foo", 67, None) + + val res = Document.Decoder + .fromSchema(JsonUnknownExampleOptional.jsonUnknownExampleOptionalSchema) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test( + "unknown field decoding: no unknown field in payload, optional field with default" + ) { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67) + ) + val expected = JsonUnknownExampleOptional( + "foo", + 67, + Some(Map("default" -> Document.fromBoolean(true))) + ) + + val res = Document.Decoder + .fromSchema( + JsonUnknownExampleOptionalWithDefault.jsonUnknownExampleOptionalSchema + ) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test("unknown field decoding: with unknown fields in payload") { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67), + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75) + ) + val expected = JsonUnknownExample( + "foo", + 67, + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75) + ) + ) + + val res = Document.Decoder + .fromSchema(JsonUnknownExample.jsonUnknownExampleSchema) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test( + "unknown field decoding: with unknown fields in payload, optional field" + ) { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67), + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75) + ) + val expected = JsonUnknownExampleOptional( + "foo", + 67, + Some( + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75) + ) + ) + ) + + val res = Document.Decoder + .fromSchema(JsonUnknownExampleOptional.jsonUnknownExampleOptionalSchema) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test("unknown field decoding: with unknow field explicitely set in payload") { + val doc = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67), + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + val expected = JsonUnknownExample( + "foo", + 67, + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + ) + + val res = Document.Decoder + .fromSchema(JsonUnknownExample.jsonUnknownExampleSchema) + .decode(doc) + + assertEquals(res, Right(expected)) + } + + test("unknown field encoding") { + val in = JsonUnknownExample( + "foo", + 67, + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + ) + + val expected = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67), + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + + val doc = Document.Encoder + .fromSchema(JsonUnknownExample.jsonUnknownExampleSchema) + .encode(in) + + assertEquals(doc, expected) + } + } diff --git a/modules/bootstrapped/test/src/smithy4s/ErrorMessageTraitSpec.scala b/modules/bootstrapped/test/src/smithy4s/ErrorMessageTraitSpec.scala index 221157343..25c1ae100 100644 --- a/modules/bootstrapped/test/src/smithy4s/ErrorMessageTraitSpec.scala +++ b/modules/bootstrapped/test/src/smithy4s/ErrorMessageTraitSpec.scala @@ -73,11 +73,20 @@ class ErrorMessageTraitSpec extends FunSuite { ) } - test("Generated getMessage") { + test("Generated - no message") { val e = ClientError(400, "oopsy") val expected = "smithy4s.example.ClientError(400, oopsy)" - expect.eql(e.getMessage, null) + expect.eql(e.getMessage, expected) + expect.eql(e.toString, s"smithy4s.example.ClientError: $expected") + } + + test("Generated - has message") { + val e = + ErrorCustomTypeMessage(Some(CustomErrorMessageType("This is a test."))) + + val expected = "smithy4s.example.ErrorCustomTypeMessage: This is a test." + expect.eql(e.getMessage, "This is a test.") expect.eql(e.toString, expected) } diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/update-lsp-config/expected.json b/modules/codegen-plugin/src/sbt-test/codegen-plugin/update-lsp-config/expected.json index 2b2e64f24..471476fcc 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/update-lsp-config/expected.json +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/update-lsp-config/expected.json @@ -6,7 +6,7 @@ ], "maven" : { "dependencies" : [ - "com.disneystreaming.alloy:alloy-core:0.3.9" + "com.disneystreaming.alloy:alloy-core:0.3.13" ], "repositories" : [ { diff --git a/modules/core/src/smithy4s/Smithy4sThrowable.scala b/modules/core/src/smithy4s/Smithy4sThrowable.scala index c4d9c1a75..390d05962 100644 --- a/modules/core/src/smithy4s/Smithy4sThrowable.scala +++ b/modules/core/src/smithy4s/Smithy4sThrowable.scala @@ -18,14 +18,8 @@ package smithy4s trait Smithy4sThrowable extends Throwable { self: Product => - /** - * implementing toString, because implementing getMessage - * lead to `smithy4s.example.ClientError: smithy4s.example.ClientError(400, "oops")` - * which felt weird - */ - override def toString(): String = { + private def show(message: String): String = { val name = getClass().getName() - val message = getLocalizedMessage() if (message == null) { val sb = new StringBuilder() sb.append(name) @@ -41,4 +35,16 @@ trait Smithy4sThrowable extends Throwable { self: Product => s"$name: $message" } } + + override def getMessage(): String = this.show(message = null) + + /** + * implementing toString, because implementing getMessage + * lead to `smithy4s.example.ClientError: smithy4s.example.ClientError(400, "oops")` + * which felt weird + */ + override def toString(): String = { + val message = getLocalizedMessage() + this.show(message = message) + } } diff --git a/modules/core/src/smithy4s/internals/DocumentDecoderSchemaVisitor.scala b/modules/core/src/smithy4s/internals/DocumentDecoderSchemaVisitor.scala index 541bd198c..bd3849ef8 100644 --- a/modules/core/src/smithy4s/internals/DocumentDecoderSchemaVisitor.scala +++ b/modules/core/src/smithy4s/internals/DocumentDecoderSchemaVisitor.scala @@ -32,8 +32,11 @@ import smithy4s.schema._ import java.util.Base64 import java.util.UUID +import java.{util => ju} import scala.collection.immutable.ListMap import alloy.Untagged +import alloy.JsonUnknown +import scala.collection.mutable.ListBuffer trait DocumentDecoder[A] { self => def apply(history: List[PayloadPath.Segment], document: Document): A @@ -307,6 +310,9 @@ class DocumentDecoderSchemaVisitor( } } + private def isForJsonUnknown[Z, A](field: Field[Z, A]): Boolean = + field.hints.has(JsonUnknown) + override def struct[S]( shapeId: ShapeId, hints: Hints, @@ -316,58 +322,116 @@ class DocumentDecoderSchemaVisitor( def jsonLabel[A](field: Field[S, A]): String = field.hints.get(JsonName).map(_.value).getOrElse(field.label) - def fieldDecoder[A]( - field: Field[S, A] - ): ( - List[PayloadPath.Segment], - Any => Unit, - Map[String, Document] - ) => Unit = { + type Handler = + (List[PayloadPath.Segment], Document, ju.HashMap[String, Any]) => Unit + + val labelledFields = fields.map { field => val jLabel = jsonLabel(field) + val decoded = field.schema.getDefaultValue + val default = decoded.orNull + (field, jLabel, default) + } + + def fieldHandler[A](field: Field[S, A], jLabel: String): Handler = { + val decoder = apply(field.schema) + val label = field.label + (parentPath, in, mmap) => + val _ = mmap.put( + label, { + val path = PayloadPath.Segment(jLabel) :: parentPath + decoder(path, in) + } + ) + } - field.getDefaultValue match { - case Some(defaultValue) => - ( - pp: List[PayloadPath.Segment], - buffer: Any => Unit, - fields: Map[String, Document] - ) => - val path = PayloadPath.Segment(jLabel) :: pp - fields - .get(jLabel) match { - case Some(document) => - buffer(apply(field.schema)(path, document)) - case None => - buffer(defaultValue) + val (fieldsForUnknown, knownFields) = labelledFields.partition { + case (field, _, _) => isForJsonUnknown(field) + } + + val handlers = + new ju.HashMap[String, Handler](knownFields.length << 1, 0.5f) { + knownFields.foreach { case (field, jLabel, _) => + put(jLabel, fieldHandler(field, jLabel)) + } + } + + if (fieldsForUnknown.isEmpty) { + DocumentDecoder.instance("Structure", "Object") { + case (pp, DObject(value)) => + val buffer = new ju.HashMap[String, Any](handlers.size << 1, 0.5f) + value.foreach { case (key, value) => + val handler = handlers.get(key) + if (handler != null) { + handler(pp, value, buffer) } - case None => - ( - pp: List[PayloadPath.Segment], - buffer: Any => Unit, - fields: Map[String, Document] - ) => - val path = PayloadPath.Segment(jLabel) :: pp - fields - .get(jLabel) match { - case Some(document) => - buffer(apply(field.schema)(path, document)) - case None => - throw new PayloadError( - PayloadPath(path.reverse), - "", - "Required field not found" - ) + } + val orderedBuffer = Vector.newBuilder[Any] + labelledFields.foreach { case (field, jLabel, default) => + orderedBuffer += { + val value = buffer.get(field.label) + if (value == null) { + if (default == null) { + throw new PayloadError( + PayloadPath((PayloadPath.Segment(jLabel) :: pp).reverse), + jLabel, + "Required field not found" + ) + } else default + } else value } + } + make(orderedBuffer.result()) + } + } else { + val fieldForUnknownDocDecoders = fieldsForUnknown.map { + case (field, jLabel, _) => + jLabel -> apply(field.schema).asInstanceOf[DocumentDecoder[Any]] + }.toMap + DocumentDecoder.instance("Structure", "Object") { + case (pp, DObject(value)) => + val buffer = new ju.HashMap[String, Any](handlers.size << 1, 0.5f) + val unknownValues = ListBuffer[(String, Document)]() + value.foreach { case (key, value) => + val handler = handlers.get(key) + if (handler == null) { + unknownValues += (key -> value) + } else { + handler(pp, value, buffer) + } + } + val orderedBuffer = Vector.newBuilder[Any] + val unknownValue = + if (unknownValues.nonEmpty) Document.obj(unknownValues) else null + labelledFields.foreach { case (field, jLabel, default) => + orderedBuffer += { + fieldForUnknownDocDecoders.get(jLabel) match { + case None => + val value = buffer.get(field.label) + if (value == null) { + if (default == null) { + throw new PayloadError( + PayloadPath( + (PayloadPath.Segment(jLabel) :: pp).reverse + ), + jLabel, + "Required field not found" + ) + } else default + } else value + case Some(decoder) => + if (unknownValue == null) { + if (default == null) { + decoder(Nil, Document.obj()) + } else default + } else { + decoder(Nil, unknownValue) + } + } + } + } + make(orderedBuffer.result()) } - } - - val fieldDecoders = fields.map(field => fieldDecoder(field)) - DocumentDecoder.instance("Structure", "Object") { - case (pp, DObject(value)) => - val buffer = Vector.newBuilder[Any] - fieldDecoders.foreach(fd => fd(pp, buffer.+=(_), value)) - make(buffer.result()) } } diff --git a/modules/core/src/smithy4s/internals/DocumentEncoderSchemaVisitor.scala b/modules/core/src/smithy4s/internals/DocumentEncoderSchemaVisitor.scala index 87dca9963..841ce4297 100644 --- a/modules/core/src/smithy4s/internals/DocumentEncoderSchemaVisitor.scala +++ b/modules/core/src/smithy4s/internals/DocumentEncoderSchemaVisitor.scala @@ -17,13 +17,14 @@ package smithy4s package internals -import alloy.Discriminated import alloy.Untagged import smithy.api.JsonName import smithy.api.TimestampFormat import smithy.api.TimestampFormat.DATE_TIME import smithy.api.TimestampFormat.EPOCH_SECONDS import smithy.api.TimestampFormat.HTTP_DATE +import alloy.Discriminated +import alloy.JsonUnknown import smithy4s.capability.EncoderK import smithy4s.schema.Primitive.PBigDecimal import smithy4s.schema.Primitive.PBigInt @@ -100,9 +101,14 @@ class DocumentEncoderSchemaVisitor( hints .get(TimestampFormat) .getOrElse(TimestampFormat.EPOCH_SECONDS) match { - case DATE_TIME => ts => DString(ts.format(DATE_TIME)) - case HTTP_DATE => ts => DString(ts.format(HTTP_DATE)) - case EPOCH_SECONDS => ts => DNumber(BigDecimal(ts.epochSecond)) + case DATE_TIME => ts => DString(ts.format(DATE_TIME)) + case HTTP_DATE => ts => DString(ts.format(HTTP_DATE)) + case EPOCH_SECONDS => + ts => + val epochSecondsWithNanos = + BigDecimal(ts.epochSecond) + (BigDecimal(ts.nano) * BigDecimal(10) + .pow(-9)) + DNumber(epochSecondsWithNanos) } case PDocument => from(identity) case PFloat => from(float => DNumber(BigDecimal(float.toDouble))) @@ -181,6 +187,9 @@ class DocumentEncoderSchemaVisitor( from(e => DString(value(e))) } + private def isForJsonUnknown(field: Field[_, _]): Boolean = + field.hints.has(JsonUnknown) + override def struct[S]( shapeId: ShapeId, hints: Hints, @@ -210,7 +219,37 @@ class DocumentEncoderSchemaVisitor( } } - val encoders = fields.map(field => fieldEncoder(field)) + def jsonUnknownFieldEncoder[A]( + field: Field[S, A] + ): (S, Builder[(String, Document), Map[String, Document]]) => Unit = { + val encoder = apply(field.schema) + (s, builder) => { + if (explicitDefaultsEncoding) { + encoder(field.get(s)) match { + case Document.DObject(value) => value.foreach(builder += _) + case _ => + throw new IllegalArgumentException( + s"Failed encoding field ${field.label} because it cannot be converted to a JSON object" + ) + } + } else { + field.foreachUnlessDefault(s) { a => + encoder(a) match { + case Document.DObject(value) => value.foreach(builder += _) + case _ => + throw new IllegalArgumentException( + s"Failed encoding field ${field.label} because it cannot be converted to a JSON object" + ) + } + } + } + } + } + + val (fieldsForUnknown, knownFields) = fields.partition(isForJsonUnknown) + + val encoders = knownFields.map(field => fieldEncoder(field)) ++ + fieldsForUnknown.map(field => jsonUnknownFieldEncoder(field)) new DocumentEncoder[S] { def apply(s: S): Document = { val builder = Map.newBuilder[String, Document] diff --git a/modules/dynamic/test/src-jvm/smithy4s/dynamic/DynamicJsonServerSpec.scala b/modules/dynamic/test/src-jvm/smithy4s/dynamic/DynamicJsonServerSpec.scala index 621b33ac3..122e3f8af 100644 --- a/modules/dynamic/test/src-jvm/smithy4s/dynamic/DynamicJsonServerSpec.scala +++ b/modules/dynamic/test/src-jvm/smithy4s/dynamic/DynamicJsonServerSpec.scala @@ -81,7 +81,7 @@ class DynamicJsonServerSpec() extends DummyIO.Suite { testJsonIO("Dynamic service is correctly wired: Bad Json Input") { jsonIO => val expected = PayloadError( PayloadPath("key"), - "", + "key", "Required field not found" ) diff --git a/modules/json/src/smithy4s/json/JsoniterCodecCompiler.scala b/modules/json/src/smithy4s/json/JsoniterCodecCompiler.scala index 6cabb6d68..a8a8ee8a7 100644 --- a/modules/json/src/smithy4s/json/JsoniterCodecCompiler.scala +++ b/modules/json/src/smithy4s/json/JsoniterCodecCompiler.scala @@ -94,7 +94,8 @@ object JsoniterCodecCompiler { DiscriminatedUnionMember, Default, Required, - Nullable + Nullable, + JsonUnknown ) } diff --git a/modules/json/src/smithy4s/json/internals/SchemaVisitorJCodec.scala b/modules/json/src/smithy4s/json/internals/SchemaVisitorJCodec.scala index 2bc37f071..cf0ff7736 100644 --- a/modules/json/src/smithy4s/json/internals/SchemaVisitorJCodec.scala +++ b/modules/json/src/smithy4s/json/internals/SchemaVisitorJCodec.scala @@ -26,6 +26,7 @@ import com.github.plokhotnyuk.jsoniter_scala.core.JsonWriter import smithy.api.JsonName import smithy.api.TimestampFormat import alloy.Discriminated +import alloy.JsonUnknown import alloy.Nullable import alloy.Untagged import smithy4s.internals.DiscriminatedUnionMember @@ -1351,6 +1352,9 @@ private[smithy4s] class SchemaVisitorJCodec( case Some(x) => x.value } + private def isForJsonUnknown[Z, A](field: Field[Z, A]): Boolean = + field.hints.has(JsonUnknown) + private type Handler = (Cursor, JsonReader, util.HashMap[String, Any]) => Unit private def fieldHandler[Z, A]( @@ -1369,27 +1373,47 @@ private[smithy4s] class SchemaVisitorJCodec( ) } + private def writeLabel(label: String, out: JsonWriter): Unit = + if (label.forall(JsonWriter.isNonEscapedAscii)) { + out.writeNonEscapedAsciiKey(label) + } else out.writeKey(label) + private def fieldEncoder[Z, A]( field: Field[Z, A] ): (Z, JsonWriter) => Unit = { val codec = apply(field.schema) val jLabel = jsonLabel(field) - val writeLabel: JsonWriter => Unit = - if (jLabel.forall(JsonWriter.isNonEscapedAscii)) { - _.writeNonEscapedAsciiKey(jLabel) - } else _.writeKey(jLabel) - if (explicitDefaultsEncoding) { (z: Z, out: JsonWriter) => - writeLabel(out) + writeLabel(jLabel, out) codec.encodeValue(field.get(z), out) } else { (z: Z, out: JsonWriter) => field.foreachUnlessDefault(z) { (a: A) => - writeLabel(out) + writeLabel(jLabel, out) codec.encodeValue(a, out) } } } + private def jsonUnknownFieldEncoder[Z, A]( + field: Field[Z, A] + ): (Z, JsonWriter) => Unit = { + val docEncoder = Document.Encoder.fromSchema(field.schema) + (z: Z, out: JsonWriter) => + field.foreachUnlessDefault(z) { a => + docEncoder.encode(a) match { + case Document.DObject(value) => + value.foreach { case (label: String, value: Document) => + writeLabel(label, out) + documentJCodec.encodeValue(value, out) + } + case _ => + out.encodeError( + s"Failed encoding field ${field.label} because it cannot be converted to a JSON object" + ) + } + } + } + private type Fields[Z] = Vector[Field[Z, _]] private type LabelledFields[Z] = Vector[(Field[Z, _], String, Any)] private def labelledFields[Z](fields: Fields[Z]): LabelledFields[Z] = @@ -1400,7 +1424,124 @@ private[smithy4s] class SchemaVisitorJCodec( (field, jLabel, default) } - private def nonPayloadStruct[Z]( + private def structRetainUnknownFields[Z]( + allFields: LabelledFields[Z], + knownFields: LabelledFields[Z], + fieldsForUnknown: LabelledFields[Z], + structHints: Hints + )( + const: Vector[Any] => Z, + encode: (Z, JsonWriter, Vector[(Z, JsonWriter) => Unit]) => Unit + ): JCodec[Z] = + new JCodec[Z] { + + private val fieldForUnknownDocumentDecoders = fieldsForUnknown.map { + case (field, label, _) => + label -> Document.Decoder + .fromSchema(field.schema) + .asInstanceOf[Document.Decoder[Any]] + }.toMap + + private[this] val handlers = + new util.HashMap[String, Handler](knownFields.length << 1, 0.5f) { + knownFields.foreach { case (field, jLabel, _) => + put(jLabel, fieldHandler(field)) + } + } + + private[this] val documentEncoders = + knownFields.map(labelledField => fieldEncoder(labelledField._1)) ++ + fieldsForUnknown.map(f => jsonUnknownFieldEncoder(f._1)) + + def expecting: String = "object" + + override def canBeKey = false + + def decodeValue(cursor: Cursor, in: JsonReader): Z = + decodeValue_(cursor, in)(emptyMetadata) + + private def decodeValue_( + cursor: Cursor, + in: JsonReader + ): scala.collection.Map[String, Any] => Z = { + val unknownValues = ListBuffer[(String, Document)]() + val buffer = new util.HashMap[String, Any](handlers.size << 1, 0.5f) + if (in.isNextToken('{')) { + if (!in.isNextToken('}')) { + in.rollbackToken() + while ({ + val key = in.readKeyAsString() + val handler = handlers.get(key) + if (handler eq null) { + val value = documentJCodec.decodeValue(cursor, in) + unknownValues += (key -> value) + } else handler(cursor, in, buffer) + in.isNextToken(',') + }) () + if (!in.isCurrentToken('}')) in.objectEndOrCommaError() + } + } else in.decodeError("Expected JSON object") + + // At this point, we have parsed the json and retrieved + // all the values that interest us for the construction + // of our domain object. + // We re-order the values following the order of the schema + // fields before calling the constructor. + { (meta: scala.collection.Map[String, Any]) => + meta.foreach(kv => buffer.put(kv._1, kv._2)) + val stage2 = new VectorBuilder[Any] + val unknownValue = + if (unknownValues.nonEmpty) Document.obj(unknownValues) else null + + allFields.foreach { case (f, jsonLabel, default) => + stage2 += { + fieldForUnknownDocumentDecoders.get(jsonLabel) match { + case None => + val value = buffer.get(f.label) + if (value == null) { + if (default == null) + cursor.requiredFieldError(jsonLabel, jsonLabel) + else default + } else value + + case Some(docDecoder) => + if (unknownValue == null) { + if (default == null) { + docDecoder + .decode(Document.obj()) + .getOrElse( + in.decodeError( + s"${cursor.getPath(Nil)} Failed translating a Document.DObject to the type targeted by ${f.label}." + ) + ) + } else default + } else { + docDecoder + .decode(unknownValue) + .getOrElse( + in.decodeError( + s"${cursor.getPath(Nil)} Failed translating a Document.DObject to the type targeted by ${f.label}." + ) + ) + } + } + } + } + const(stage2.result()) + } + } + + def encodeValue(z: Z, out: JsonWriter): Unit = + encode(z, out, documentEncoders) + + def decodeKey(in: JsonReader): Z = + in.decodeError("Cannot use products as keys") + + def encodeKey(x: Z, out: JsonWriter): Unit = + out.encodeError("Cannot use products as keys") + } + + private def structIgnoreUnknownFields[Z]( fields: LabelledFields[Z], structHints: Hints )( @@ -1476,6 +1617,28 @@ private[smithy4s] class SchemaVisitorJCodec( out.encodeError("Cannot use products as keys") } + private def nonPayloadStruct[Z]( + fields: LabelledFields[Z], + structHints: Hints + )( + const: Vector[Any] => Z, + encode: (Z, JsonWriter, Vector[(Z, JsonWriter) => Unit]) => Unit + ): JCodec[Z] = { + val (fieldsForUnknown, knownFields) = fields.partition { + case (field, _, _) => isForJsonUnknown(field) + } + + if (fieldsForUnknown.isEmpty) + structIgnoreUnknownFields(fields, structHints)(const, encode) + else + structRetainUnknownFields( + fields, + knownFields, + fieldsForUnknown, + structHints + )(const, encode) + } + private def basicStruct[A, S]( fields: LabelledFields[S], structHints: Hints diff --git a/modules/json/test/src/smithy4s/json/SchemaVisitorJCodecTests.scala b/modules/json/test/src/smithy4s/json/SchemaVisitorJCodecTests.scala index f868dfad3..a32aaabdf 100644 --- a/modules/json/test/src/smithy4s/json/SchemaVisitorJCodecTests.scala +++ b/modules/json/test/src/smithy4s/json/SchemaVisitorJCodecTests.scala @@ -18,6 +18,7 @@ package smithy4s package json import alloy.Discriminated +import alloy.JsonUnknown import com.github.plokhotnyuk.jsoniter_scala.core.{readFromString => _, _} import munit.FunSuite import smithy.api.Default @@ -109,6 +110,70 @@ class SchemaVisitorJCodecTests() extends FunSuite { ) } + case class JsonUnknownExample( + s: String, + i: Int, + others: Map[String, Document] + ) + + object JsonUnknownExample { + implicit val jsonUnknownExampleSchema: Schema[JsonUnknownExample] = { + val s = string.required[JsonUnknownExample]("s", _.s) + val i = int.required[JsonUnknownExample]("i", _.i) + val others = map(string, document) + .required[JsonUnknownExample]("others", _.others) + .addHints(JsonUnknown()) + struct(s, i, others)(JsonUnknownExample.apply) + } + } + + object JsonUnknownExampleWithDefault { + implicit val jsonUnknownExampleSchema: Schema[JsonUnknownExample] = { + val s = string.required[JsonUnknownExample]("s", _.s) + val i = int.required[JsonUnknownExample]("i", _.i) + val others = map(string, document) + .required[JsonUnknownExample]("others", _.others) + .addHints( + JsonUnknown(), + Default(Document.obj("default" -> Document.fromBoolean(true))) + ) + struct(s, i, others)(JsonUnknownExample.apply) + } + } + + case class JsonUnknownExampleOptional( + s: String, + i: Int, + others: Option[Map[String, Document]] + ) + + object JsonUnknownExampleOptional { + implicit val jsonUnknownExampleOptionalSchema + : Schema[JsonUnknownExampleOptional] = { + val s = string.required[JsonUnknownExampleOptional]("s", _.s) + val i = int.required[JsonUnknownExampleOptional]("i", _.i) + val others = map(string, document) + .optional[JsonUnknownExampleOptional]("others", _.others) + .addHints(JsonUnknown()) + struct(s, i, others)(JsonUnknownExampleOptional.apply) + } + } + + object JsonUnknownExampleOptionalWithDefault { + implicit val jsonUnknownExampleOptionalSchema + : Schema[JsonUnknownExampleOptional] = { + val s = string.required[JsonUnknownExampleOptional]("s", _.s) + val i = int.required[JsonUnknownExampleOptional]("i", _.i) + val others = map(string, document) + .optional[JsonUnknownExampleOptional]("others", _.others) + .addHints( + JsonUnknown(), + Default(Document.obj("default" -> Document.fromBoolean(true))) + ) + struct(s, i, others)(JsonUnknownExampleOptional.apply) + } + } + test( "Compiling a codec for a recursive type should not blow up the stack" ) { @@ -669,4 +734,136 @@ class SchemaVisitorJCodecTests() extends FunSuite { assertEquals(fromJson, Right(patchable)) } + test("unknown field decoding: no unknown field in payload") { + val jsonString = """{"s": "foo", "i": 67}""" + val expected = JsonUnknownExample("foo", 67, Map.empty) + + val res = readFromString[JsonUnknownExample](jsonString) + + assertEquals(res, expected) + } + + test("unknown field decoding: no unknown field in payload with default") { + import JsonUnknownExampleWithDefault._ + val jsonString = """{"s": "foo", "i": 67}""" + val expected = JsonUnknownExample( + "foo", + 67, + Map("default" -> Document.fromBoolean(true)) + ) + + val res = readFromString[JsonUnknownExample](jsonString) + + assertEquals(res, expected) + } + + test( + "unknown field decoding: no unknown field in payload, optional field" + ) { + val jsonString = """{"s": "foo", "i": 67}""" + val expected = JsonUnknownExampleOptional("foo", 67, None) + + val res = readFromString[JsonUnknownExampleOptional](jsonString) + + assertEquals(res, expected) + } + + test( + "unknown field decoding: no unknown field in payload, optional field with default" + ) { + import JsonUnknownExampleOptionalWithDefault._ + val jsonString = """{"s": "foo", "i": 67}""" + val expected = JsonUnknownExampleOptional( + "foo", + 67, + Some(Map("default" -> Document.fromBoolean(true))) + ) + + val res = readFromString[JsonUnknownExampleOptional](jsonString) + + assertEquals(res, expected) + } + + test("unknown field decoding: with unknown fields in payload") { + val jsonString = + """{"s": "foo", "i": 67, "someField": {"a": "b"}, "someOtherField": 75}""" + val expected = JsonUnknownExample( + "foo", + 67, + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75) + ) + ) + + val res = readFromString[JsonUnknownExample](jsonString) + + assertEquals(res, expected) + } + + test( + "unknown field decoding: with unknown fields in payload, optional field" + ) { + val jsonString = + """{"s": "foo", "i": 67, "someField": {"a": "b"}, "someOtherField": 75}""" + val expected = JsonUnknownExampleOptional( + "foo", + 67, + Some( + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75) + ) + ) + ) + + val res = readFromString[JsonUnknownExampleOptional](jsonString) + + assertEquals(res, expected) + } + + test("unknown field decoding: with unknow field explicitely set in payload") { + val jsonString = + """{"s": "foo", "i": 67, "someField": {"a": "b"}, "someOtherField": 75, "others": {}}""" + val expected = JsonUnknownExample( + "foo", + 67, + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + ) + + val res = readFromString[JsonUnknownExample](jsonString) + + assertEquals(res, expected) + } + + test("unknown field encoding") { + val in = JsonUnknownExample( + "foo", + 67, + Map( + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + ) + + val expected = Document.obj( + "s" -> Document.fromString("foo"), + "i" -> Document.fromInt(67), + "someField" -> Document.obj("a" -> Document.fromString("b")), + "someOtherField" -> Document.fromInt(75), + "others" -> Document.obj() + ) + + val jsonStr = writeToString[JsonUnknownExample](in) + + val doc = readFromString[Document](jsonStr) + + assertEquals(doc, expected) + } + } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index dfb923897..5284fba62 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -31,7 +31,7 @@ object Dependencies { val Alloy = new { val org = "com.disneystreaming.alloy" - val alloyVersion = "0.3.9" + val alloyVersion = "0.3.13" val core = org % "alloy-core" % alloyVersion val openapi = org %% "alloy-openapi" % alloyVersion val protobuf = org % "alloy-protobuf" % alloyVersion diff --git a/sampleSpecs/jsonUnknown.smithy b/sampleSpecs/jsonUnknown.smithy new file mode 100644 index 000000000..07c55e78b --- /dev/null +++ b/sampleSpecs/jsonUnknown.smithy @@ -0,0 +1,17 @@ +$version: "2" + +namespace smithy4s.example + +use alloy#jsonUnknown + +structure JsonUnknownExample { + s: String + i: Integer + @jsonUnknown + additionalProperties: AdditionalProperties +} + +map AdditionalProperties { + key: String + value: Document +}