From c701c0816dedf2d545ee9024e8b4333ec5a61513 Mon Sep 17 00:00:00 2001 From: Jay Chia Date: Tue, 5 Sep 2023 10:43:57 -0700 Subject: [PATCH 1/2] Parse PrimitiveLogicalType::Unknown as Arrow Null DataType --- src/io/parquet/read/schema/convert.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/io/parquet/read/schema/convert.rs b/src/io/parquet/read/schema/convert.rs index 821d5107649..564b9f98a4d 100644 --- a/src/io/parquet/read/schema/convert.rs +++ b/src/io/parquet/read/schema/convert.rs @@ -170,6 +170,12 @@ fn from_fixed_len_byte_array( /// Maps a [`PhysicalType`] with optional metadata to a [`DataType`] fn to_primitive_type_inner(primitive_type: &PrimitiveType) -> DataType { + // Unknown type refers to values that should always be treated as Null + // See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null + if matches!(primitive_type.logical_type, Some(PrimitiveLogicalType::Unknown)) { + return DataType::Null; + } + match primitive_type.physical_type { PhysicalType::Boolean => DataType::Boolean, PhysicalType::Int32 => { From 7c7d7028c70177828a1e37ccd2a73c3a49c984b6 Mon Sep 17 00:00:00 2001 From: Jay Chia Date: Tue, 5 Sep 2023 11:26:43 -0700 Subject: [PATCH 2/2] lints --- src/io/parquet/read/schema/convert.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/io/parquet/read/schema/convert.rs b/src/io/parquet/read/schema/convert.rs index 564b9f98a4d..ad7fe415348 100644 --- a/src/io/parquet/read/schema/convert.rs +++ b/src/io/parquet/read/schema/convert.rs @@ -172,7 +172,10 @@ fn from_fixed_len_byte_array( fn to_primitive_type_inner(primitive_type: &PrimitiveType) -> DataType { // Unknown type refers to values that should always be treated as Null // See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null - if matches!(primitive_type.logical_type, Some(PrimitiveLogicalType::Unknown)) { + if matches!( + primitive_type.logical_type, + Some(PrimitiveLogicalType::Unknown) + ) { return DataType::Null; }