add more test

Netflix · Apr 9, 2024 · a2a3ca5 · a2a3ca5
1 parent 3a909f4
commit a2a3ca5
Show file tree

Hide file tree

Showing 2 changed files with 119 additions and 0 deletions.
diff --git a/...rc/test/groovy/com/netflix/metacat/connector/hive/converters/HiveTypeConverterSpec.groovy b/...rc/test/groovy/com/netflix/metacat/connector/hive/converters/HiveTypeConverterSpec.groovy
@@ -119,6 +119,11 @@ class HiveTypeConverterSpec extends Specification {
             "struct<prediction_date:int,lower_confidence_amt:double,upper_confidence_amt:double,model_short_name:string>",
             "struct<prediction_date:int,lower_confidence_amt:int,upper_confidence_amt:int,model_short_name:string>",
             "struct<prediction_date:int,prediction_source:string>",
+
+            // Nested Type with UpperCase
+            'array<struct<date:string,countryCodes:array<string>,source:string>>',
+            "struct<Field3:struct<Nested_Field1:bigint,Nested_Field2:bigint>>",
+            "struct<Field1:bigint,Field2:bigint,field3:struct<NESTED_Field1:bigint,NesteD_Field2:bigint>>"
         ]
     }
 
@@ -218,6 +223,10 @@ class HiveTypeConverterSpec extends Specification {
         "struct<prediction_date:int,lower_confidence_amt:double,upper_confidence_amt:double,model_short_name:string>" || "struct<prediction_date:int,lower_confidence_amt:double,upper_confidence_amt:double,model_short_name:string>"
         "struct<prediction_date:int,lower_confidence_amt:int,upper_confidence_amt:int,model_short_name:string>" || "struct<prediction_date:int,lower_confidence_amt:int,upper_confidence_amt:int,model_short_name:string>"
         "struct<prediction_date:int,prediction_source:string>" || "struct<prediction_date:int,prediction_source:string>"
+
+        'array<struct<field2:decimal(38 ),countryCodes:array<string>,source:string>>' || 'array<struct<field2:decimal(38),countryCodes:array<string>,source:string>>'
+        "struct<Field3:struct<Nested_Field1:bigint,Nested_Field2:bigint,Nested_FIELD3:decimal( 38, 9)>>" || "struct<Field3:struct<Nested_Field1:bigint,Nested_Field2:bigint,Nested_FIELD3:decimal(38,9)>>"
+        "struct<Field1:decimal (38,9 ),Field2:bigint,field3:struct<NESTED_Field1:decimal ( 38,9 ),NesteD_Field2:bigint>>" || "struct<Field1:decimal(38,9),Field2:bigint,field3:struct<NESTED_Field1:decimal(38,9),NesteD_Field2:bigint>>"
     }
 
     @Unroll
@@ -233,4 +242,16 @@ class HiveTypeConverterSpec extends Specification {
         ]
     }
 
+    @Unroll
+    def 'case reserve fieldName Fidelity'(String typeString, String expectedString) {
+        expect:
+        def result = converter.fromMetacatTypeToJson(converter.toMetacatType(typeString)).toString()
+
+        assert result == expectedString
+
+        where:
+        typeString | expectedString
+        "struct<Field1:bigint,Field2:bigint,field3:struct<nested_Field1:bigint,nested_Field2:bigint>>" | """{"type":"row","fields":[{"name":"Field1","type":"bigint"},{"name":"Field2","type":"bigint"},{"name":"field3","type":{"type":"row","fields":[{"name":"nested_Field1","type":"bigint"},{"name":"nested_Field2","type":"bigint"}]}}]}"""
+        "array<struct<date:string,countryCodes:array<string>,source:string>>" | """{"type":"array","elementType":{"type":"row","fields":[{"name":"date","type":"string"},{"name":"countryCodes","type":{"type":"array","elementType":"string"}},{"name":"source","type":"string"}]}}"""
+    }
 }
diff --git a/...at-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/...at-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy
@@ -166,6 +166,104 @@ class MetacatSmokeSpec extends Specification {
         thrown(MetacatNotSupportedException)
     }
 
+    @Unroll
+    def "Test create/get iceberg table with nested fields with upper case"() {
+        given:
+        def catalogName = 'embedded-fast-hive-metastore'
+        def databaseName = 'iceberg_db'
+        def tableName = 'iceberg_table_with_upper_case_nested_fields'
+        def uri = isLocalEnv ? String.format('file:/tmp/data/') : null
+        def tableDto = new TableDto(
+            name: QualifiedName.ofTable(catalogName, databaseName, tableName),
+            serde: new StorageDto(
+                owner: 'metacat-test',
+                inputFormat: 'org.apache.hadoop.mapred.TextInputFormat',
+                outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
+                serializationLib: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe',
+                parameters: [
+                    'serialization.format': '1'
+                ],
+                uri: uri
+            ),
+            definitionMetadata: null,
+            dataMetadata: null,
+            fields: [
+                new FieldDto(
+                    comment: "DGS",
+                    name: "movieId",
+                    pos: 0,
+                    type: "int",
+                    partition_key: false,
+                    source_type: "int",
+                    jsonType: "int",
+                    isNullable: false,
+                    size: null,
+                    defaultValue: null,
+                    isSortKey: null,
+                    isIndexKey: null
+                ),
+                new FieldDto(
+                    comment: null,
+                    name: "allPlannedLaunchDates",
+                    pos: 1,
+                    partition_key: false,
+                    source_type: "list<struct<28: date: optional string (The date is in UTC timezone and is null for movies with no planned launch dates. \nDGS:, 29: countryCodes: optional list<string> (Country codes for this launch date \nDGS: , 30: source: optional string (The source is INVALID when the movie has neither a valid first window start date, nor a Gravity forecast date. \nDGS:)>>",
+                    type: "{(date: chararray,countryCodes: {(array_element: chararray)},source: chararray)}",
+                    jsonType: [
+                        type: "array",
+                        elementType: [
+                            type: "row",
+                            fields: [
+                                [
+                                    name: "date",
+                                    type: "chararray"
+                                ],
+                                [
+                                    name: "countryCodes",
+                                    type: [
+                                        type: "array",
+                                        elementType: "chararray"
+                                    ]
+                                ],
+                                [
+                                    name: "source",
+                                    type: "chararray"
+                                ]
+                            ]
+                        ]
+                    ],
+                    isNullable: true,
+                    size: null,
+                    defaultValue: null,
+                    isSortKey: null,
+                    isIndexKey: null
+                )
+            ]
+        )
+
+        def metadataLocation = String.format('/tmp/data/metadata/00002-2d6c1951-31d5-4bea-8edd-e35746b172f3.metadata.json')
+
+        def metadata = [table_type: 'ICEBERG', metadata_location: metadataLocation]
+        tableDto.setMetadata(metadata)
+        when:
+        try {api.createDatabase(catalogName, databaseName, new DatabaseCreateRequestDto())
+        } catch (Exception ignored) {
+        }
+        api.createTable(catalogName, databaseName, tableName, tableDto)
+        def tableDTO = api.getTable(catalogName, databaseName, tableName, true, true, true)
+
+        then:
+        noExceptionThrown()
+        tableDTO.metadata.get("metadata_location").equals(metadataLocation)
+        tableDTO.getFields().size() == 2
+        def nestedFieldsNames = tableDTO.getFields().find { it.pos == 1 }.jsonType.elementType.fields*.name
+        def expectedNames = ["date", "countrycodes", "source"]
+        assert nestedFieldsNames == expectedNames : "The nested fields names differ from the expected. They are: $nestedFieldsNames"
+
+        cleanup:
+        api.deleteTable(catalogName, databaseName, tableName)
+    }
+
     @Unroll
     def "Test create database for #catalogName/#databaseName"() {
         given: