Fix nested list & map of structs using case classes (#267)

* Fix nested list & map of structs Signed-off-by: Andres Gomez Ferrer <[email protected]> * Rename value to v in lambdas Signed-off-by: Andres Gomez Ferrer <[email protected]> * Make it compatible with scala 2.12 Signed-off-by: Andres Gomez Ferrer <[email protected]> --------- Signed-off-by: Andres Gomez Ferrer <[email protected]>
flyteorg · Nov 23, 2023 · acac5fe · acac5fe
1 parent 1760eff
commit acac5fe
Show file tree

Hide file tree

Showing 2 changed files with 80 additions and 5 deletions.
diff --git a/flytekit-scala-tests/src/test/scala/org/flyte/flytekitscala/SdkScalaTypeTest.scala b/flytekit-scala-tests/src/test/scala/org/flyte/flytekitscala/SdkScalaTypeTest.scala
@@ -52,7 +52,9 @@ import org.flyte.flytekitscala.SdkLiteralTypes.{
 case class ScalarNested(
     foo: String,
     bar: Option[String],
-    nestedNested: Option[ScalarNestedNested]
+    nestedNested: Option[ScalarNestedNested],
+    nestedNestedList: List[ScalarNestedNested],
+    nestedNestedMap: Map[String, ScalarNestedNested]
 )
 case class ScalarNestedNested(foo: String, bar: Option[String])
 
@@ -191,6 +193,32 @@ class SdkScalaTypeTest {
                     "bar" -> Struct.Value.ofStringValue("bar")
                   ).asJava
                 )
+              ),
+              "nestedNestedList" -> Struct.Value.ofListValue(
+                List(
+                  Struct.Value.ofStructValue(
+                    Struct.of(
+                      Map(
+                        "foo" -> Struct.Value.ofStringValue("foo"),
+                        "bar" -> Struct.Value.ofStringValue("bar")
+                      ).asJava
+                    )
+                  )
+                ).asJava
+              ),
+              "nestedNestedMap" -> Struct.Value.ofStructValue(
+                Struct.of(
+                  Map(
+                    "foo" -> Struct.Value.ofStructValue(
+                      Struct.of(
+                        Map(
+                          "foo" -> Struct.Value.ofStringValue("foo"),
+                          "bar" -> Struct.Value.ofStringValue("bar")
+                        ).asJava
+                      )
+                    )
+                  ).asJava
+                )
               )
             ).asJava
           )
@@ -212,7 +240,9 @@ class SdkScalaTypeTest {
           ScalarNested(
             "foo",
             None,
-            Some(ScalarNestedNested("foo", Some("bar")))
+            Some(ScalarNestedNested("foo", Some("bar"))),
+            List(ScalarNestedNested("foo", Some("bar"))),
+            Map("foo" -> ScalarNestedNested("foo", Some("bar")))
           )
         )
       )
@@ -238,7 +268,9 @@ class SdkScalaTypeTest {
           ScalarNested(
             "foo",
             Some("bar"),
-            Some(ScalarNestedNested("foo", Some("bar")))
+            Some(ScalarNestedNested("foo", Some("bar"))),
+            List(ScalarNestedNested("foo", Some("bar"))),
+            Map("foo" -> ScalarNestedNested("foo", Some("bar")))
           )
         )
       )
@@ -274,6 +306,32 @@ class SdkScalaTypeTest {
                     "bar" -> Struct.Value.ofStringValue("bar")
                   ).asJava
                 )
+              ),
+              "nestedNestedList" -> Struct.Value.ofListValue(
+                List(
+                  Struct.Value.ofStructValue(
+                    Struct.of(
+                      Map(
+                        "foo" -> Struct.Value.ofStringValue("foo"),
+                        "bar" -> Struct.Value.ofStringValue("bar")
+                      ).asJava
+                    )
+                  )
+                ).asJava
+              ),
+              "nestedNestedMap" -> Struct.Value.ofStructValue(
+                Struct.of(
+                  Map(
+                    "foo" -> Struct.Value.ofStructValue(
+                      Struct.of(
+                        Map(
+                          "foo" -> Struct.Value.ofStringValue("foo"),
+                          "bar" -> Struct.Value.ofStringValue("bar")
+                        ).asJava
+                      )
+                    )
+                  ).asJava
+                )
               )
             ).asJava
           )
@@ -317,7 +375,9 @@ class SdkScalaTypeTest {
         ScalarNested(
           "foo",
           Some("bar"),
-          Some(ScalarNestedNested("foo", Some("bar")))
+          Some(ScalarNestedNested("foo", Some("bar"))),
+          List(ScalarNestedNested("foo", Some("bar"))),
+          Map("foo" -> ScalarNestedNested("foo", Some("bar")))
         )
       )
     )
@@ -337,7 +397,9 @@ class SdkScalaTypeTest {
         ScalarNested(
           "foo",
           Some("bar"),
-          Some(ScalarNestedNested("foo", Some("bar")))
+          Some(ScalarNestedNested("foo", Some("bar"))),
+          List(ScalarNestedNested("foo", Some("bar"))),
+          Map("foo" -> ScalarNestedNested("foo", Some("bar")))
         )
       )
     ).asJava

diff --git a/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala b/flytekit-scala_2.13/src/main/scala/org/flyte/flytekitscala/SdkLiteralTypes.scala
@@ -308,6 +308,19 @@ object SdkLiteralTypes {
           value.asInstanceOf[Double].toLong
         } else if (tpe =:= typeOf[Float]) {
           value.asInstanceOf[Double].toFloat
+        } else if (tpe <:< typeOf[List[Any]]) {
+          value
+            .asInstanceOf[List[Any]]
+            .map(v => {
+              valueToParamValue(v, tpe.typeArgs.head)
+            })
+        } else if (tpe <:< typeOf[Map[String, Any]]) {
+          value
+            .asInstanceOf[Map[String, Any]]
+            .mapValues(v => {
+              valueToParamValue(v, tpe.typeArgs(1))
+            })
+            .toMap
         } else if (tpe <:< typeOf[Option[Any]]) { // this has to be before Product check because Option is a Product
           if (value == None) { // None is used to represent Struct.Value.Kind.NULL_VALUE when converting struct to map
             None