From cc1dd262b1bb3e6e3cfbbeb57a772f2d6e0ec9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Tue, 16 Jul 2024 15:16:58 +0200 Subject: [PATCH 1/3] parse all edge cases of ints and floats --- .../org/virtuslab/yaml/YamlDecoder.scala | 49 ++++++++++++++--- .../virtuslab/yaml/decoder/DecoderSuite.scala | 54 +++++++++++++++++++ 2 files changed, 96 insertions(+), 7 deletions(-) diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala b/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala index 13cd05345..7c1ec068e 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala @@ -85,27 +85,62 @@ object YamlDecoder extends YamlDecoderCompanionCrossCompat { ) implicit def forInt: YamlDecoder[Int] = YamlDecoder { case s @ ScalarNode(value, _) => - Try(java.lang.Integer.decode(value.replaceAll("_", "")).toInt).toEither.left + val normalizedValue = + if value.startsWith("0o") then value.stripPrefix("0o").prepended('0') else value + + Try(java.lang.Integer.decode(normalizedValue.replaceAll("_", "")).toInt).toEither.left .map(ConstructError.from(_, "Int", s)) } implicit def forLong: YamlDecoder[Long] = YamlDecoder { case s @ ScalarNode(value, _) => - Try(java.lang.Long.decode(value.replaceAll("_", "")).toLong).toEither.left + val normalizedValue = + if value.startsWith("0o") then value.stripPrefix("0o").prepended('0') else value + + Try(java.lang.Long.decode(normalizedValue.replaceAll("_", "")).toLong).toEither.left .map(ConstructError.from(_, "Long", s)) } + private val infinityRegex = """([-+]?)(\.inf|\.Inf|\.INF)""".r + + private val nanRegex = """(\.nan|\.NaN|\.NAN)""".r + implicit def forDouble: YamlDecoder[Double] = YamlDecoder { case s @ ScalarNode(value, _) => - Try(java.lang.Double.parseDouble(value.replaceAll("_", ""))).toEither.left - .map(ConstructError.from(_, "Double", s)) + infinityRegex.findFirstMatchIn(value) match + case Some(m) => + Right(m.group(1) match + case "-" => Double.NegativeInfinity + case _ => Double.PositiveInfinity + ) + case None => + nanRegex.findFirstMatchIn(value) match + case Some(_) => + Right(Double.NaN) + case None => + Try(java.lang.Double.parseDouble(value.replaceAll("_", ""))).toEither.left + .map(ConstructError.from(_, "Double", s)) } implicit def forFloat: YamlDecoder[Float] = YamlDecoder { case s @ ScalarNode(value, _) => - Try(java.lang.Float.parseFloat(value.replaceAll("_", ""))).toEither.left - .map(ConstructError.from(_, "Float", s)) + infinityRegex.findFirstMatchIn(value) match + case Some(m) => + Right(m.group(1) match + case "-" => Float.NegativeInfinity + case _ => Float.PositiveInfinity + ) + case None => + nanRegex.findFirstMatchIn(value) match + case Some(_) => + Right(Float.NaN) + case None => + Try(java.lang.Float.parseFloat(value.replaceAll("_", ""))).toEither.left + .map(ConstructError.from(_, "Float", s)) } implicit def forShort: YamlDecoder[Short] = YamlDecoder { case s @ ScalarNode(value, _) => - Try(java.lang.Short.decode(value.replaceAll("_", "")).toShort).toEither.left + val normalizedValue = + if value.startsWith("0o") then value.stripPrefix("0o").prepended('0') else value + + Try(java.lang.Short.decode(normalizedValue.replaceAll("_", "")).toShort).toEither.left .map(ConstructError.from(_, "Short", s)) } diff --git a/core/shared/src/test/scala-3/org/virtuslab/yaml/decoder/DecoderSuite.scala b/core/shared/src/test/scala-3/org/virtuslab/yaml/decoder/DecoderSuite.scala index 4a276c0da..1f19a5f27 100644 --- a/core/shared/src/test/scala-3/org/virtuslab/yaml/decoder/DecoderSuite.scala +++ b/core/shared/src/test/scala-3/org/virtuslab/yaml/decoder/DecoderSuite.scala @@ -360,6 +360,60 @@ class DecoderSuite extends munit.FunSuite: assertEquals(foo, Right(List(Some(Foo(1, "1")), None))) } + test("issue 222 - parse edge cases of booleans floats doubles and integers") { + case class Data( + booleans: List[Boolean], + integers: List[Int], + floats: List[Float], + `also floats`: List[Float], + `also doubles`: List[Double] + ) derives YamlCodec + + val yaml = """booleans: [ true, True, false, FALSE ] + |integers: [ 0, 0o7, 0x3A, -19 ] + |floats: [ + | 0., -0.0, .5, +12e03, -2E+05 ] + |also floats: [ + | .inf, -.Inf, +.INF, .NAN, .nan, .NaN] + |also doubles: [ + | .inf, -.Inf, +.INF, .NAN, .nan, .NaN]""".stripMargin + + val expected = Data( + booleans = List(true, true, false, false), + integers = List(0, 7, 58, -19), + floats = List(0.0f, -0.0f, 0.5f, 12000.0f, -200000.0f), + `also floats` = List( + Float.PositiveInfinity, + Float.NegativeInfinity, + Float.PositiveInfinity, + Float.NaN, + Float.NaN, + Float.NaN + ), + `also doubles` = List( + Double.PositiveInfinity, + Double.NegativeInfinity, + Double.PositiveInfinity, + Double.NaN, + Double.NaN, + Double.NaN + ) + ) + + yaml.as[Data] match + case Left(error: YamlError) => throw error + case Right(data) => + assertEquals(data.booleans, expected.booleans) + assertEquals(data.integers, expected.integers) + assertEquals(data.floats, expected.floats) + data.`also floats`.zipAll(expected.`also floats`, 0f, 0f).foreach { case (a, b) => + assertEqualsFloat(a, b, 0f) + } + data.`also doubles`.zipAll(expected.`also doubles`, 0.0d, 0.0d).foreach { case (a, b) => + assertEqualsDouble(a, b, 0.0d) + } + } + test("issue 281 - parse multiline string") { case class Data(description: String) derives YamlCodec From 5eed294b0f1b9bdf6306fef6e4e6f2e33b442922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Tue, 16 Jul 2024 15:22:12 +0200 Subject: [PATCH 2/3] use scala 2.13 syntax for cross-compile --- .../org/virtuslab/yaml/YamlDecoder.scala | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala b/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala index 7c1ec068e..6cdfac67c 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala @@ -86,7 +86,7 @@ object YamlDecoder extends YamlDecoderCompanionCrossCompat { implicit def forInt: YamlDecoder[Int] = YamlDecoder { case s @ ScalarNode(value, _) => val normalizedValue = - if value.startsWith("0o") then value.stripPrefix("0o").prepended('0') else value + if (value.startsWith("0o")) value.stripPrefix("0o").prepended('0') else value Try(java.lang.Integer.decode(normalizedValue.replaceAll("_", "")).toInt).toEither.left .map(ConstructError.from(_, "Int", s)) @@ -94,7 +94,7 @@ object YamlDecoder extends YamlDecoderCompanionCrossCompat { implicit def forLong: YamlDecoder[Long] = YamlDecoder { case s @ ScalarNode(value, _) => val normalizedValue = - if value.startsWith("0o") then value.stripPrefix("0o").prepended('0') else value + if (value.startsWith("0o")) value.stripPrefix("0o").prepended('0') else value Try(java.lang.Long.decode(normalizedValue.replaceAll("_", "")).toLong).toEither.left .map(ConstructError.from(_, "Long", s)) @@ -105,40 +105,44 @@ object YamlDecoder extends YamlDecoderCompanionCrossCompat { private val nanRegex = """(\.nan|\.NaN|\.NAN)""".r implicit def forDouble: YamlDecoder[Double] = YamlDecoder { case s @ ScalarNode(value, _) => - infinityRegex.findFirstMatchIn(value) match + infinityRegex.findFirstMatchIn(value) match { case Some(m) => - Right(m.group(1) match + Right(m.group(1) match { case "-" => Double.NegativeInfinity case _ => Double.PositiveInfinity - ) + }) case None => - nanRegex.findFirstMatchIn(value) match + nanRegex.findFirstMatchIn(value) match { case Some(_) => Right(Double.NaN) case None => Try(java.lang.Double.parseDouble(value.replaceAll("_", ""))).toEither.left .map(ConstructError.from(_, "Double", s)) + } + } } implicit def forFloat: YamlDecoder[Float] = YamlDecoder { case s @ ScalarNode(value, _) => - infinityRegex.findFirstMatchIn(value) match + infinityRegex.findFirstMatchIn(value) match { case Some(m) => - Right(m.group(1) match + Right(m.group(1) match { case "-" => Float.NegativeInfinity case _ => Float.PositiveInfinity - ) + }) case None => - nanRegex.findFirstMatchIn(value) match + nanRegex.findFirstMatchIn(value) match { case Some(_) => Right(Float.NaN) case None => Try(java.lang.Float.parseFloat(value.replaceAll("_", ""))).toEither.left .map(ConstructError.from(_, "Float", s)) + } + } } implicit def forShort: YamlDecoder[Short] = YamlDecoder { case s @ ScalarNode(value, _) => val normalizedValue = - if value.startsWith("0o") then value.stripPrefix("0o").prepended('0') else value + if (value.startsWith("0o")) value.stripPrefix("0o").prepended('0') else value Try(java.lang.Short.decode(normalizedValue.replaceAll("_", "")).toShort).toEither.left .map(ConstructError.from(_, "Short", s)) From dcecaa3ea84d80bceaa10a06203c0c88ac7210c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Tue, 16 Jul 2024 22:45:35 +0200 Subject: [PATCH 3/3] decided to disregard the lowercase problem --- .../org/virtuslab/yaml/YamlDecoder.scala | 50 +++++++------------ 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala b/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala index 6cdfac67c..770331038 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala @@ -100,43 +100,29 @@ object YamlDecoder extends YamlDecoderCompanionCrossCompat { .map(ConstructError.from(_, "Long", s)) } - private val infinityRegex = """([-+]?)(\.inf|\.Inf|\.INF)""".r - - private val nanRegex = """(\.nan|\.NaN|\.NAN)""".r - implicit def forDouble: YamlDecoder[Double] = YamlDecoder { case s @ ScalarNode(value, _) => - infinityRegex.findFirstMatchIn(value) match { - case Some(m) => - Right(m.group(1) match { - case "-" => Double.NegativeInfinity - case _ => Double.PositiveInfinity - }) - case None => - nanRegex.findFirstMatchIn(value) match { - case Some(_) => - Right(Double.NaN) - case None => - Try(java.lang.Double.parseDouble(value.replaceAll("_", ""))).toEither.left - .map(ConstructError.from(_, "Double", s)) - } + val lowercased = value.toLowerCase + if (lowercased.endsWith("inf")) { + if (value.startsWith("-")) Right(Double.NegativeInfinity) + else Right(Double.PositiveInfinity) + } else if (lowercased.endsWith("nan")) { + Right(Double.NaN) + } else { + Try(java.lang.Double.parseDouble(value.replaceAll("_", ""))).toEither.left + .map(ConstructError.from(_, "Double", s)) } } implicit def forFloat: YamlDecoder[Float] = YamlDecoder { case s @ ScalarNode(value, _) => - infinityRegex.findFirstMatchIn(value) match { - case Some(m) => - Right(m.group(1) match { - case "-" => Float.NegativeInfinity - case _ => Float.PositiveInfinity - }) - case None => - nanRegex.findFirstMatchIn(value) match { - case Some(_) => - Right(Float.NaN) - case None => - Try(java.lang.Float.parseFloat(value.replaceAll("_", ""))).toEither.left - .map(ConstructError.from(_, "Float", s)) - } + val lowercased = value.toLowerCase + if (lowercased.endsWith("inf")) { + if (value.startsWith("-")) Right(Float.NegativeInfinity) + else Right(Float.PositiveInfinity) + } else if (lowercased.endsWith("nan")) { + Right(Float.NaN) + } else { + Try(java.lang.Float.parseFloat(value.replaceAll("_", ""))).toEither.left + .map(ConstructError.from(_, "Float", s)) } }