From 26bb289a166d23d346abe09939eeb7fa31397ea6 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Fri, 21 Jul 2023 16:42:23 +0100 Subject: [PATCH 1/4] Fix Json codec of maps with lazy key schema (#586) * Fix Json codec of maps with lazy key schema * readme --- README.md | 10 +++++----- .../scala/zio/schema/codec/JsonCodec.scala | 2 ++ .../zio/schema/codec/JsonCodecSpec.scala | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 243bf44ee..243558c74 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,13 @@ _ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, In order to use this library, we need to add the following lines in our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.11" +libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.12" // Required for automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.11", +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.12", libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` diff --git a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala index 8c22480fa..fb56ec666 100644 --- a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala +++ b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala @@ -273,6 +273,7 @@ object JsonCodec { case Schema.Primitive(StandardType.StringType, _) => Option(JsonFieldEncoder.string) case Schema.Primitive(StandardType.LongType, _) => Option(JsonFieldEncoder.long) case Schema.Primitive(StandardType.IntType, _) => Option(JsonFieldEncoder.int) + case Schema.Lazy(inner) => jsonFieldEncoder(inner()) case _ => None } @@ -598,6 +599,7 @@ object JsonCodec { case Schema.Primitive(StandardType.StringType, _) => Option(JsonFieldDecoder.string) case Schema.Primitive(StandardType.LongType, _) => Option(JsonFieldDecoder.long) case Schema.Primitive(StandardType.IntType, _) => Option(JsonFieldDecoder.int) + case Schema.Lazy(inner) => jsonFieldDecoder(inner()) case _ => None } diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index d35adb72d..933c6ccb3 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -96,6 +96,15 @@ object JsonCodecSpec extends ZIOSpecDefault { """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" ) ) + }, + test("of simple keys and values where the key's schema is lazy") { + assertEncodes( + Schema.map[Int, Value](Schema.defer(Schema[Int]), Schema[Value]), + Map(0 -> Value(0, true), 1 -> Value(1, false)), + charSequenceToByteChunk( + """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" + ) + ) } ), suite("Set")( @@ -469,6 +478,15 @@ object JsonCodecSpec extends ZIOSpecDefault { """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" ) ) + }, + test("of simple keys and values where the key schema is lazy") { + assertDecodes( + Schema.map[Int, Value](Schema.defer(Schema[Int]), Schema[Value]), + Map(0 -> Value(0, true), 1 -> Value(1, false)), + charSequenceToByteChunk( + """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" + ) + ) } ) ) From 9420122d0fea19d03e9899664b43df2cebccc15e Mon Sep 17 00:00:00 2001 From: Gabriel Ciuloaica Date: Tue, 22 Aug 2023 14:07:33 +0300 Subject: [PATCH 2/4] fixed enum support in avro codec (#578) * fixed enum support in avro codec * fixed tests * linted * fixed README * fixed scal 3x compilation * fixed enum support in avro codec * fixed tests * linted * fixed scal 3x compilation * fix: fixed README * fix: fixed README --------- Co-authored-by: Daniel Vigovszky --- README.md | 10 +- .../scala/zio/schema/codec/AvroCodec.scala | 114 +++++++++++++----- .../zio/schema/codec/AvroSchemaCodec.scala | 4 +- .../zio/schema/codec/AvroCodecSpec.scala | 14 ++- .../schema/codec/AvroSchemaCodecSpec.scala | 2 +- 5 files changed, 101 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 243558c74..2e33a70fe 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,13 @@ _ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, In order to use this library, we need to add the following lines in our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.12" -libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.13" +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.13" // Required for automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.12", +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.13", libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` diff --git a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala index 97a499e7d..6caec830c 100644 --- a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala +++ b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala @@ -209,16 +209,31 @@ object AvroCodec { private def decodeCaseClass1[A, Z](raw: Any, schema: Schema.CaseClass1[A, Z]) = decodeValue(raw, schema.field.schema).map(schema.defaultConstruct) - private def decodeEnum[Z](raw: Any, cases: Schema.Case[Z, _]*): Either[DecodeError, Any] = { - val generic = raw.asInstanceOf[GenericData.Record] - val enumCaseName = generic.getSchema.getFullName - val enumCaseValue = generic.get("value") + private def decodeEnum[Z](raw: Any, cases: Schema.Case[Z, _]*): Either[DecodeError, Any] = + raw match { + case enums: GenericData.EnumSymbol => + decodeGenericEnum(enums.toString, None, cases: _*) + case gr: GenericData.Record => + val enumCaseName = gr.getSchema.getFullName + if (gr.hasField("value")) { + val enumCaseValue = gr.get("value") + decodeGenericEnum[Z](enumCaseName, Some(enumCaseValue), cases: _*) + } else { + decodeGenericEnum[Z](enumCaseName, None, cases: _*) + } + case _ => Left(DecodeError.MalformedFieldWithPath(Chunk.single("Error"), s"Unknown enum: $raw")) + } + + private def decodeGenericEnum[Z]( + enumCaseName: String, + enumCaseValue: Option[AnyRef], + cases: Schema.Case[Z, _]* + ): Either[DecodeError, Any] = cases .find(_.id == enumCaseName) - .map(s => decodeValue(enumCaseValue, s.schema)) + .map(s => decodeValue(enumCaseValue.getOrElse(s), s.schema)) .toRight(DecodeError.MalformedFieldWithPath(Chunk.single("Error"), s"Unknown enum value: $enumCaseName")) .flatMap(identity) - } private def decodeRecord[A](value: A, schema: Schema.Record[_]) = { val record = value.asInstanceOf[GenericRecord] @@ -454,41 +469,41 @@ object AvroCodec { else decodeValue(value, schema).map(Some(_)) private def encodeValue[A](a: A, schema: Schema[A]): Any = schema match { - case Schema.Enum1(_, c1, _) => encodeEnum(a, c1) - case Schema.Enum2(_, c1, c2, _) => encodeEnum(a, c1, c2) - case Schema.Enum3(_, c1, c2, c3, _) => encodeEnum(a, c1, c2, c3) - case Schema.Enum4(_, c1, c2, c3, c4, _) => encodeEnum(a, c1, c2, c3, c4) - case Schema.Enum5(_, c1, c2, c3, c4, c5, _) => encodeEnum(a, c1, c2, c3, c4, c5) - case Schema.Enum6(_, c1, c2, c3, c4, c5, c6, _) => encodeEnum(a, c1, c2, c3, c4, c5, c6) + case Schema.Enum1(_, c1, _) => encodeEnum(schema, a, c1) + case Schema.Enum2(_, c1, c2, _) => encodeEnum(schema, a, c1, c2) + case Schema.Enum3(_, c1, c2, c3, _) => encodeEnum(schema, a, c1, c2, c3) + case Schema.Enum4(_, c1, c2, c3, c4, _) => encodeEnum(schema, a, c1, c2, c3, c4) + case Schema.Enum5(_, c1, c2, c3, c4, c5, _) => encodeEnum(schema, a, c1, c2, c3, c4, c5) + case Schema.Enum6(_, c1, c2, c3, c4, c5, c6, _) => encodeEnum(schema, a, c1, c2, c3, c4, c5, c6) case Schema.Enum7(_, c1, c2, c3, c4, c5, c6, c7, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7) case Schema.Enum8(_, c1, c2, c3, c4, c5, c6, c7, c8, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8) case Schema.Enum9(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9) case Schema.Enum10(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) case Schema.Enum11(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) case Schema.Enum12(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12) case Schema.Enum13(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) case Schema.Enum14(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14) case Schema.Enum15(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15) case Schema.Enum16(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16) case Schema.Enum17(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17) case Schema.Enum18(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18) case Schema.Enum19(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19) case Schema .Enum20(_, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, _) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20) + encodeEnum(schema, a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20) case Schema.Enum21( _, c1, @@ -514,7 +529,31 @@ object AvroCodec { c21, _ ) => - encodeEnum(a, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, c21) + encodeEnum( + schema, + a, + c1, + c2, + c3, + c4, + c5, + c6, + c7, + c8, + c9, + c10, + c11, + c12, + c13, + c14, + c15, + c16, + c17, + c18, + c19, + c20, + c21 + ) case Schema.Enum22( _, c1, @@ -542,6 +581,7 @@ object AvroCodec { _ ) => encodeEnum( + schema, a, c1, c2, @@ -580,9 +620,10 @@ object AvroCodec { case Schema.Optional(schema, _) => encodeOption(schema, a) case Schema.Tuple2(left, right, _) => encodeTuple2(left.asInstanceOf[Schema[Any]], right.asInstanceOf[Schema[Any]], a) - case Schema.Either(left, right, _) => encodeEither(left, right, a) - case Schema.Lazy(schema0) => encodeValue(a, schema0()) - case Schema.CaseClass0(_, _, _) => encodePrimitive((), StandardType.UnitType) + case Schema.Either(left, right, _) => encodeEither(left, right, a) + case Schema.Lazy(schema0) => encodeValue(a, schema0()) + case Schema.CaseClass0(_, _, _) => + encodeCaseClass(schema, a, Seq.empty: _*) //encodePrimitive((), StandardType.UnitType) case Schema.CaseClass1(_, f, _, _) => encodeCaseClass(schema, a, f) case Schema.CaseClass2(_, f0, f1, _, _) => encodeCaseClass(schema, a, f0, f1) case Schema.CaseClass3(_, f0, f1, f2, _, _) => encodeCaseClass(schema, a, f0, f1, f2) @@ -926,11 +967,20 @@ object AvroCodec { record } - private def encodeEnum[Z](value: Z, cases: Schema.Case[Z, _]*): Any = { + private def encodeEnum[Z](schemaRaw: Schema[Z], value: Z, cases: Schema.Case[Z, _]*): Any = { + val schema = AvroSchemaCodec + .encodeToApacheAvro(schemaRaw) + .getOrElse(throw new Exception("Avro schema could not be generated for Enum.")) val fieldIndex = cases.indexWhere(c => c.deconstructOption(value).isDefined) if (fieldIndex >= 0) { val subtypeCase = cases(fieldIndex) - encodeValue(subtypeCase.deconstruct(value), subtypeCase.schema.asInstanceOf[Schema[Any]]) + if (schema.getType == SchemaAvro.Type.ENUM) { + GenericData.get.createEnum(schema.getEnumSymbols.get(fieldIndex), schema) + } else { + + encodeValue(subtypeCase.deconstruct(value), subtypeCase.schema.asInstanceOf[Schema[Any]]) + + } } else { throw new Exception("Could not find matching case for enum value.") } diff --git a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroSchemaCodec.scala b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroSchemaCodec.scala index 6630e6177..bb64aeac7 100644 --- a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroSchemaCodec.scala +++ b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroSchemaCodec.scala @@ -517,8 +517,8 @@ object AvroSchemaCodec extends AvroSchemaCodec { } def hasAvroEnumAnnotation(annotations: Chunk[Any]): Boolean = annotations.exists { - case AvroAnnotations.avroEnum => true - case _ => false + case AvroAnnotations.avroEnum() => true + case _ => false } def wrapAvro(schemaAvro: SchemaAvro, name: String, marker: AvroPropMarker): SchemaAvro = { diff --git a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala index 5790916c8..d0796b28b 100644 --- a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala +++ b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala @@ -21,9 +21,9 @@ import java.time.{ import java.util.UUID import zio._ +import zio.schema.codec.AvroAnnotations.avroEnum import zio.schema.{ DeriveSchema, Schema } import zio.stream.ZStream -import zio.test.TestAspect.failing import zio.test._ object AvroCodecSpec extends ZIOSpecDefault { @@ -106,10 +106,12 @@ object AvroCodecSpec extends ZIOSpecDefault { case class BooleanValue(value: Boolean) extends OneOf + case object NullValue extends OneOf + implicit val schemaOneOf: Schema[OneOf] = DeriveSchema.gen[OneOf] } - sealed trait Enums + @avroEnum() sealed trait Enums object Enums { case object A extends Enums @@ -649,12 +651,18 @@ object AvroCodecSpec extends ZIOSpecDefault { val result = codec.decode(bytes) assertTrue(result == Right(OneOf.BooleanValue(true))) }, + test("Decode Enum3 - case object") { + val codec = AvroCodec.schemaBasedBinaryCodec[OneOf] + val bytes = codec.encode(OneOf.NullValue) + val result = codec.decode(bytes) + assertTrue(result == Right(OneOf.NullValue)) + }, test("Decode Enum5") { val codec = AvroCodec.schemaBasedBinaryCodec[Enums] val bytes = codec.encode(Enums.A) val result = codec.decode(bytes) assertTrue(result == Right(Enums.A)) - } @@ failing, // TODO: the case object from a sealed trait are not properly encoded and decoded. + }, test("Decode Person") { val codec = AvroCodec.schemaBasedBinaryCodec[Person] val bytes = codec.encode(Person("John", 42)) diff --git a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroSchemaCodecSpec.scala b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroSchemaCodecSpec.scala index 31060202f..25895f44d 100644 --- a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroSchemaCodecSpec.scala +++ b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroSchemaCodecSpec.scala @@ -60,7 +60,7 @@ object AvroSchemaCodecSpec extends ZIOSpecDefault { }, test("encodes sealed trait objects only as enum when avroEnum annotation is present") { - val schema = DeriveSchema.gen[SpecTestData.CaseObjectsOnlyAdt].annotate(AvroAnnotations.avroEnum) + val schema = DeriveSchema.gen[SpecTestData.CaseObjectsOnlyAdt].annotate(AvroAnnotations.avroEnum()) val result = AvroSchemaCodec.encode(schema) val expected = """{"type":"enum","name":"MyEnum","symbols":["A","B","MyC"]}""" From 620d57d876be3dc4c3a6e65f7818b43e04c95ebf Mon Sep 17 00:00:00 2001 From: Gabriel Ciuloaica Date: Tue, 22 Aug 2023 16:29:43 +0300 Subject: [PATCH 3/4] =?UTF-8?q?added=20API=20to=20allow=20encoding=20and?= =?UTF-8?q?=20decoding=20to/from=20Generic=20record,=20to=20a=E2=80=A6=20(?= =?UTF-8?q?#573)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added API to allow encoding and decoding to/from Generic record, to allow integration with Schema Registry (AWS Glue) * fixed imports * added API to allow encoding and decoding to/from Generic record, to allow integration with Schema Registry (AWS Glue) * fixed imports * fix: fixed README --------- Co-authored-by: John A. De Goes --- .../main/scala/zio/schema/codec/AvroCodec.scala | 15 +++++++++++++-- .../scala-2/zio/schema/codec/AvroCodecSpec.scala | 14 +++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala index 6caec830c..4359d0a21 100644 --- a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala +++ b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala @@ -25,8 +25,13 @@ import zio.{ Chunk, Unsafe, ZIO } object AvroCodec { - implicit def schemaBasedBinaryCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = - new BinaryCodec[A] { + trait ExtendedBinaryCodec[A] extends BinaryCodec[A] { + def encodeGenericRecord(value: A)(implicit schema: Schema[A]): GenericData.Record + def decodeGenericRecord(value: GenericRecord)(implicit schema: Schema[A]): Either[DecodeError, A] + } + + implicit def schemaBasedBinaryCodec[A](implicit schema: Schema[A]): ExtendedBinaryCodec[A] = + new ExtendedBinaryCodec[A] { val avroSchema: SchemaAvro = AvroSchemaCodec.encodeToApacheAvro(schema).getOrElse(throw new Exception("Avro schema could not be generated.")) @@ -59,6 +64,12 @@ object AvroCodec { decode(chunk).map(Chunk(_)) ) } + + override def encodeGenericRecord(value: A)(implicit schema: Schema[A]): GenericData.Record = + encodeValue(value, schema).asInstanceOf[GenericData.Record] + + override def decodeGenericRecord(value: GenericRecord)(implicit schema: Schema[A]): Either[DecodeError, A] = + decodeValue(value, schema) } private def decodeValue[A](raw: Any, schema: Schema[A]): Either[DecodeError, A] = schema match { diff --git a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala index d0796b28b..8f1526aaf 100644 --- a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala +++ b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala @@ -20,6 +20,8 @@ import java.time.{ } import java.util.UUID +import org.apache.avro.generic.GenericData + import zio._ import zio.schema.codec.AvroAnnotations.avroEnum import zio.schema.{ DeriveSchema, Schema } @@ -140,7 +142,8 @@ object AvroCodecSpec extends ZIOSpecDefault { sequenceDecoderSpec, genericRecordDecoderSpec, enumDecoderSpec, - streamEncodingDecodingSpec + streamEncodingDecodingSpec, + genericRecordEncodeDecodeSpec ) private val primitiveEncoderSpec = suite("Avro Codec - Encoder primitive spec")( @@ -695,4 +698,13 @@ object AvroCodecSpec extends ZIOSpecDefault { }) + private val genericRecordEncodeDecodeSpec = suite("AvroCodec - encode/decode Generic Record")( + test("Encode/Decode") { + val codec = AvroCodec.schemaBasedBinaryCodec[Record] + val generic: GenericData.Record = codec.encodeGenericRecord(Record("John", 42)) + val result = codec.decodeGenericRecord(generic) + assertTrue(result == Right(Record("John", 42))) + } + ) + } From 1861252ace7f3c673f8d3bc847fe439878a89366 Mon Sep 17 00:00:00 2001 From: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> Date: Thu, 31 Aug 2023 01:12:59 +0200 Subject: [PATCH 4/4] Add Schema for `zio.json.ast.Json` (#599) --- .../main/scala/zio/schema/codec/package.scala | 85 +++++++++++++ .../zio/schema/codec/JsonCodecSpec.scala | 117 ++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 zio-schema-json/shared/src/main/scala/zio/schema/codec/package.scala diff --git a/zio-schema-json/shared/src/main/scala/zio/schema/codec/package.scala b/zio-schema-json/shared/src/main/scala/zio/schema/codec/package.scala new file mode 100644 index 000000000..d1eeb7135 --- /dev/null +++ b/zio-schema-json/shared/src/main/scala/zio/schema/codec/package.scala @@ -0,0 +1,85 @@ +package zio.schema.codec + +import java.util.Base64 + +import scala.collection.immutable.ListMap + +import zio.Chunk +import zio.json.ast.Json +import zio.schema.annotation.directDynamicMapping +import zio.schema.{ DynamicValue, Schema, StandardType, TypeId } + +package object json { + implicit val schemaJson: Schema[Json] = + Schema.dynamicValue.annotate(directDynamicMapping()).transform(toJson, fromJson) + + private def toJson(dv: DynamicValue): Json = + dv match { + case DynamicValue.Record(_, values) => + values.foldLeft(Json.Obj()) { case (obj, (name, value)) => (name, toJson(value)) +: obj } + case DynamicValue.Enumeration(_, _) => + throw new Exception("DynamicValue.Enumeration is unsupported") + case DynamicValue.Sequence(values) => + Json.Arr(values.map(toJson)) + case DynamicValue.Dictionary(_) => + throw new Exception("DynamicValue.Dictionary is unsupported") + case DynamicValue.SetValue(values) => + Json.Arr(Chunk.fromIterable(values.map(toJson))) + case DynamicValue.Primitive(value, standardType) => + standardType.asInstanceOf[StandardType[_]] match { + case StandardType.UnitType => Json.Obj() + case StandardType.StringType => Json.Str(value.asInstanceOf[String]) + case StandardType.BoolType => Json.Bool(value.asInstanceOf[Boolean]) + case StandardType.ByteType => Json.Num(value.asInstanceOf[Byte]) + case StandardType.ShortType => Json.Num(value.asInstanceOf[Short]) + case StandardType.IntType => Json.Num(value.asInstanceOf[Int]) + case StandardType.LongType => Json.Num(value.asInstanceOf[Long]) + case StandardType.FloatType => Json.Num(value.asInstanceOf[Float]) + case StandardType.DoubleType => Json.Num(value.asInstanceOf[Double]) + case StandardType.BinaryType => + Json.Str(Base64.getEncoder.encodeToString(value.asInstanceOf[Chunk[Byte]].toArray)) + case StandardType.CharType => Json.Str(value.asInstanceOf[Char].toString) + case StandardType.UUIDType => Json.Str(value.asInstanceOf[java.util.UUID].toString) + case StandardType.BigDecimalType => Json.Num(value.asInstanceOf[java.math.BigDecimal]) + case StandardType.BigIntegerType => Json.Num(BigDecimal(value.asInstanceOf[java.math.BigInteger])) + case StandardType.DayOfWeekType => Json.Str(value.asInstanceOf[java.time.DayOfWeek].toString) + case StandardType.MonthType => Json.Str(value.asInstanceOf[java.time.Month].toString) + case StandardType.MonthDayType => Json.Str(value.asInstanceOf[java.time.MonthDay].toString) + case StandardType.PeriodType => Json.Str(value.asInstanceOf[java.time.Period].toString) + case StandardType.YearType => Json.Num(value.asInstanceOf[java.time.Year].getValue) + case StandardType.YearMonthType => Json.Str(value.asInstanceOf[java.time.YearMonth].toString) + case StandardType.ZoneIdType => Json.Str(value.asInstanceOf[java.time.ZoneId].toString) + case StandardType.ZoneOffsetType => Json.Str(value.asInstanceOf[java.time.ZoneOffset].toString) + case StandardType.DurationType => Json.Str(value.asInstanceOf[java.time.Duration].toString) + case StandardType.InstantType => Json.Str(value.asInstanceOf[java.time.Instant].toString) + case StandardType.LocalDateType => Json.Str(value.asInstanceOf[java.time.LocalDate].toString) + case StandardType.LocalTimeType => Json.Str(value.asInstanceOf[java.time.LocalTime].toString) + case StandardType.LocalDateTimeType => Json.Str(value.asInstanceOf[java.time.LocalDateTime].toString) + case StandardType.OffsetTimeType => Json.Str(value.asInstanceOf[java.time.OffsetTime].toString) + case StandardType.OffsetDateTimeType => Json.Str(value.asInstanceOf[java.time.OffsetDateTime].toString) + case StandardType.ZonedDateTimeType => Json.Str(value.asInstanceOf[java.time.ZonedDateTime].toString) + } + case DynamicValue.Singleton(_) => Json.Obj() + case DynamicValue.SomeValue(value) => toJson(value) + case DynamicValue.NoneValue => Json.Null + case DynamicValue.Tuple(left, right) => Json.Arr(Chunk(toJson(left), toJson(right))) + case DynamicValue.LeftValue(value) => Json.Obj("Left" -> toJson(value)) + case DynamicValue.RightValue(value) => Json.Obj("Right" -> toJson(value)) + case DynamicValue.DynamicAst(_) => throw new Exception("DynamicValue.DynamicAst is unsupported") + case DynamicValue.Error(_) => throw new Exception("DynamicValue.Error is unsupported") + } + + private def fromJson(json: Json): DynamicValue = + json match { + case Json.Null => DynamicValue.NoneValue + case Json.Bool(value) => DynamicValue.Primitive(value, StandardType.BoolType) + case Json.Num(value) => DynamicValue.Primitive(value, StandardType.BigDecimalType) + case Json.Str(value) => DynamicValue.Primitive(value, StandardType.StringType) + case Json.Arr(values) => DynamicValue.Sequence(values.map(fromJson)) + case Json.Obj(values) => + DynamicValue.Record( + TypeId.parse("Json.Obj"), + ListMap(values.map { case (name, value) => (name, fromJson(value)) }.toList: _*) + ) + } +} diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index 933c6ccb3..b3b66901f 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -7,6 +7,7 @@ import scala.collection.immutable.ListMap import zio.Console._ import zio._ import zio.json.JsonDecoder.JsonError +import zio.json.ast.Json import zio.json.{ DeriveJsonEncoder, JsonEncoder } import zio.schema.CaseSet._ import zio.schema._ @@ -251,6 +252,64 @@ object JsonCodecSpec extends ZIOSpecDefault { charSequenceToByteChunk("""{"foo":"s","bar":1}""") ) } + ), + suite("zio.json.ast.Json encoding")( + test("Json.Obj") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Obj("foo" -> Json.Str("bar"), "null" -> Json.Null), + charSequenceToByteChunk("""{"foo":"bar","null":null}""") + ) + }, + test("Json.Arr") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Arr(Json.Str("foo"), Json.Num(1)), + charSequenceToByteChunk("""["foo",1]""") + ) + }, + test("Json.Num Int") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Num(1), + charSequenceToByteChunk("""1""") + ) + }, + test("Json.Num Long") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Num(1L), + charSequenceToByteChunk("""1""") + ) + }, + test("Json.Num Double") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Num(1.1), + charSequenceToByteChunk("""1.1""") + ) + }, + test("Json.Str") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Str("foo"), + charSequenceToByteChunk(""""foo"""") + ) + }, + test("Json.Bool") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Bool(true), + charSequenceToByteChunk("""true""") + ) + }, + test("Json.Null") { + assertEncodes( + zio.schema.codec.json.schemaJson, + Json.Null, + charSequenceToByteChunk("""null""") + ) + } ) ) @@ -488,6 +547,64 @@ object JsonCodecSpec extends ZIOSpecDefault { ) ) } + ), + suite("zio.json.ast.Json decoding")( + test("Json.Obj") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Obj("foo" -> Json.Str("bar"), "null" -> Json.Null), + charSequenceToByteChunk("""{"foo":"bar","null":null}""") + ) + }, + test("Json.Arr") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Arr(Json.Str("foo"), Json.Num(1)), + charSequenceToByteChunk("""["foo",1]""") + ) + }, + test("Json.Num Int") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Num(1), + charSequenceToByteChunk("""1""") + ) + }, + test("Json.Num Long") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Num(1L), + charSequenceToByteChunk("""1""") + ) + }, + test("Json.Num Double") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Num(1.1), + charSequenceToByteChunk("""1.1""") + ) + }, + test("Json.Str") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Str("foo"), + charSequenceToByteChunk(""""foo"""") + ) + }, + test("Json.Bool") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Bool(true), + charSequenceToByteChunk("""true""") + ) + }, + test("Json.Null") { + assertDecodes( + zio.schema.codec.json.schemaJson, + Json.Null, + charSequenceToByteChunk("""null""") + ) + } ) )