diff --git a/README.md b/README.md index 0de4044..a303038 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ The Arrow ecosystem provides many ways to convert between Arrow and other popula ## Design -Types that implements the `ArrowField`, `ArrowSerialize` and `ArrowDeserialize` traits can be converted to/from Arrow via the `try_into_arrow` and the `try_into_collection` methods. +Types that implements the `ArrowField`, `ArrowFieldSerialize` and `ArrowDeserialize` traits can be converted to/from Arrow via the `try_into_arrow` and the `try_into_collection` methods. -The `ArrowField` implementation for a type defines the Arrow schema. The `ArrowSerialize` and `ArrowDeserialize` implementations provide the conversion logic via arrow2's data structures. +The `ArrowField` implementation for a type defines the Arrow schema. The `ArrowFieldSerialize` and `ArrowDeserialize` implementations provide the conversion logic via arrow2's data structures. ## Features @@ -79,11 +79,11 @@ fn test_simple_roundtrip() { ### Similarities with Serde -The design is inspired by serde. The `ArrowSerialize` and `ArrowDeserialize` are analogs of serde's `Serialize` and `Deserialize` respectively. +The design is inspired by serde. The `ArrowFieldSerialize` and `ArrowDeserialize` are analogs of serde's `Serialize` and `Deserialize` respectively. However unlike serde's traits provide an exhaustive and flexible mapping to the serde data model, arrow2_convert's traits provide a much more narrower mapping to arrow2's data structures. -Specifically, the `ArrowSerialize` trait provides the logic to serialize a type to the corresponding `arrow2::array::MutableArray`. The `ArrowDeserialize` trait deserializes a type from the corresponding `arrow2::array::ArrowArray`. +Specifically, the `ArrowFieldSerialize` trait provides the logic to serialize a type to the corresponding `arrow2::array::MutableArray`. The `ArrowDeserialize` trait deserializes a type from the corresponding `arrow2::array::ArrowArray`. ### Workarounds @@ -92,9 +92,9 @@ Features such as partial implementation specialization and generic associated ty For example custom types need to explicitly enable Vec serialization via the `arrow_enable_vec_for_type` macro on the primitive type. This is needed since Vec is a special type in Arrow, but without implementation specialization there's no way to special-case it. -Availability of generaic associated types would simplify the implementation for large and fixed types, since a generic MutableArray can be defined. Ideally for code reusability, we wouldn’t have to reimplement `ArrowSerialize` and `ArrowDeserialize` for large and fixed size types since the primitive types are the same. However, this requires the trait functions to take a generic bounded mutable array as an argument instead of a single array type. This requires the `ArrowSerialize` and `ArrowDeserialize` implementations to be able to specify the bounds as part of the associated type, which is not possible without generic associated types. +Availability of generaic associated types would simplify the implementation for large and fixed types, since a generic MutableArray can be defined. Ideally for code reusability, we wouldn’t have to reimplement `ArrowFieldSerialize` and `ArrowDeserialize` for large and fixed size types since the primitive types are the same. However, this requires the trait functions to take a generic bounded mutable array as an argument instead of a single array type. This requires the `ArrowFieldSerialize` and `ArrowDeserialize` implementations to be able to specify the bounds as part of the associated type, which is not possible without generic associated types. -As a result, we’re forced to sacrifice code reusability and introduce a little bit of complexity by providing separate `ArrowSerialize` and `ArrowDeserialize` implementations for large and fixed size types via placeholder structures. This also requires introducing the `Type` associated type to `ArrowField` so that the arrow type can be overriden via a macro field attribute without affecting the actual type. +As a result, we’re forced to sacrifice code reusability and introduce a little bit of complexity by providing separate `ArrowFieldSerialize` and `ArrowDeserialize` implementations for large and fixed size types via placeholder structures. This also requires introducing the `Type` associated type to `ArrowField` so that the arrow type can be overriden via a macro field attribute without affecting the actual type. ## License diff --git a/arrow2_convert/src/deserialize.rs b/arrow2_convert/src/deserialize.rs index f991242..5f8461e 100644 --- a/arrow2_convert/src/deserialize.rs +++ b/arrow2_convert/src/deserialize.rs @@ -6,15 +6,15 @@ use chrono::{NaiveDate, NaiveDateTime}; use crate::field::*; /// Implemented by [`ArrowField`] that can be deserialized from arrow -pub trait ArrowDeserialize: ArrowField + Sized +pub trait ArrowFieldDeserialize: ArrowField + Sized where - Self::ArrayType: ArrowArray, + Self::ArrayType: ArrowArrayIterator, for<'a> &'a Self::ArrayType: IntoIterator, { type ArrayType; /// Deserialize this field from arrow - fn arrow_deserialize( + fn arrow_field_deserialize( v: <&Self::ArrayType as IntoIterator>::Item, ) -> Option<::Type>; @@ -25,10 +25,10 @@ where // Ideally we would be able to capture the optional field of the iterator via // something like for<'a> &'a T::ArrayType: IntoIterator>, // However, the E parameter seems to confuse the borrow checker if it's a reference. - fn arrow_deserialize_internal( + fn arrow_field_deserialize_internal( v: <&Self::ArrayType as IntoIterator>::Item, ) -> ::Type { - Self::arrow_deserialize(v).unwrap() + Self::arrow_field_deserialize(v).unwrap() } } @@ -38,7 +38,7 @@ where /// /// The derive macro generates implementations for typed struct arrays. #[doc(hidden)] -pub trait ArrowArray +pub trait ArrowArrayIterator where for<'a> &'a Self: IntoIterator, { @@ -51,11 +51,11 @@ where // Macro to facilitate implementation for numeric types and numeric arrays. macro_rules! impl_arrow_deserialize_primitive { ($physical_type:ty, $logical_type:ident) => { - impl ArrowDeserialize for $physical_type { + impl ArrowFieldDeserialize for $physical_type { type ArrayType = PrimitiveArray<$physical_type>; #[inline] - fn arrow_deserialize<'a>(v: Option<&$physical_type>) -> Option { + fn arrow_field_deserialize<'a>(v: Option<&$physical_type>) -> Option { v.map(|t| *t) } } @@ -66,7 +66,7 @@ macro_rules! impl_arrow_deserialize_primitive { macro_rules! impl_arrow_array { ($array:ty) => { - impl ArrowArray for $array { + impl ArrowArrayIterator for $array { type BaseArrayType = Self; fn iter_from_array_ref(b: &dyn Array) -> <&Self as IntoIterator>::IntoIter { @@ -80,26 +80,26 @@ macro_rules! impl_arrow_array { } // blanket implementation for optional fields -impl ArrowDeserialize for Option +impl ArrowFieldDeserialize for Option where - T: ArrowDeserialize, - T::ArrayType: 'static + ArrowArray, + T: ArrowFieldDeserialize, + T::ArrayType: 'static + ArrowArrayIterator, for<'a> &'a T::ArrayType: IntoIterator, { - type ArrayType = ::ArrayType; + type ArrayType = ::ArrayType; #[inline] - fn arrow_deserialize( + fn arrow_field_deserialize( v: <&Self::ArrayType as IntoIterator>::Item, ) -> Option<::Type> { - Some(Self::arrow_deserialize_internal(v)) + Some(Self::arrow_field_deserialize_internal(v)) } #[inline] - fn arrow_deserialize_internal( + fn arrow_field_deserialize_internal( v: <&Self::ArrayType as IntoIterator>::Item, ) -> ::Type { - ::arrow_deserialize(v) + ::arrow_field_deserialize(v) } } @@ -114,74 +114,74 @@ impl_arrow_deserialize_primitive!(i64, Int64); impl_arrow_deserialize_primitive!(f32, Float32); impl_arrow_deserialize_primitive!(f64, Float64); -impl ArrowDeserialize for String { +impl ArrowFieldDeserialize for String { type ArrayType = Utf8Array; #[inline] - fn arrow_deserialize(v: Option<&str>) -> Option { + fn arrow_field_deserialize(v: Option<&str>) -> Option { v.map(|t| t.to_string()) } } -impl ArrowDeserialize for LargeString { +impl ArrowFieldDeserialize for LargeString { type ArrayType = Utf8Array; #[inline] - fn arrow_deserialize(v: Option<&str>) -> Option { + fn arrow_field_deserialize(v: Option<&str>) -> Option { v.map(|t| t.to_string()) } } -impl ArrowDeserialize for bool { +impl ArrowFieldDeserialize for bool { type ArrayType = BooleanArray; #[inline] - fn arrow_deserialize(v: Option) -> Option { + fn arrow_field_deserialize(v: Option) -> Option { v } } -impl ArrowDeserialize for NaiveDateTime { +impl ArrowFieldDeserialize for NaiveDateTime { type ArrayType = PrimitiveArray; #[inline] - fn arrow_deserialize(v: Option<&i64>) -> Option { + fn arrow_field_deserialize(v: Option<&i64>) -> Option { v.map(|t| arrow2::temporal_conversions::timestamp_ns_to_datetime(*t)) } } -impl ArrowDeserialize for NaiveDate { +impl ArrowFieldDeserialize for NaiveDate { type ArrayType = PrimitiveArray; #[inline] - fn arrow_deserialize(v: Option<&i32>) -> Option { + fn arrow_field_deserialize(v: Option<&i32>) -> Option { v.map(|t| arrow2::temporal_conversions::date32_to_date(*t)) } } -impl ArrowDeserialize for Vec { +impl ArrowFieldDeserialize for Vec { type ArrayType = BinaryArray; #[inline] - fn arrow_deserialize(v: Option<&[u8]>) -> Option { + fn arrow_field_deserialize(v: Option<&[u8]>) -> Option { v.map(|t| t.to_vec()) } } -impl ArrowDeserialize for LargeBinary { +impl ArrowFieldDeserialize for LargeBinary { type ArrayType = BinaryArray; #[inline] - fn arrow_deserialize(v: Option<&[u8]>) -> Option> { + fn arrow_field_deserialize(v: Option<&[u8]>) -> Option> { v.map(|t| t.to_vec()) } } -impl ArrowDeserialize for FixedSizeBinary { +impl ArrowFieldDeserialize for FixedSizeBinary { type ArrayType = FixedSizeBinaryArray; #[inline] - fn arrow_deserialize(v: Option<&[u8]>) -> Option> { + fn arrow_field_deserialize(v: Option<&[u8]>) -> Option> { v.map(|t| t.to_vec()) } } @@ -190,7 +190,7 @@ fn arrow_deserialize_vec_helper( v: Option>, ) -> Option< as ArrowField>::Type> where - T: ArrowDeserialize + ArrowEnableVecForType + 'static, + T: ArrowFieldDeserialize + ArrowEnableVecForType + 'static, for<'a> &'a T::ArrayType: IntoIterator, { use std::ops::Deref; @@ -205,41 +205,41 @@ where } // Blanket implementation for Vec -impl ArrowDeserialize for Vec +impl ArrowFieldDeserialize for Vec where - T: ArrowDeserialize + ArrowEnableVecForType + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, + T: ArrowFieldDeserialize + ArrowEnableVecForType + 'static, + ::ArrayType: 'static, + for<'b> &'b ::ArrayType: IntoIterator, { type ArrayType = ListArray; - fn arrow_deserialize(v: Option>) -> Option<::Type> { + fn arrow_field_deserialize(v: Option>) -> Option<::Type> { arrow_deserialize_vec_helper::(v) } } -impl ArrowDeserialize for LargeVec +impl ArrowFieldDeserialize for LargeVec where - T: ArrowDeserialize + ArrowEnableVecForType + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, + T: ArrowFieldDeserialize + ArrowEnableVecForType + 'static, + ::ArrayType: 'static, + for<'b> &'b ::ArrayType: IntoIterator, { type ArrayType = ListArray; - fn arrow_deserialize(v: Option>) -> Option<::Type> { + fn arrow_field_deserialize(v: Option>) -> Option<::Type> { arrow_deserialize_vec_helper::(v) } } -impl ArrowDeserialize for FixedSizeVec +impl ArrowFieldDeserialize for FixedSizeVec where - T: ArrowDeserialize + ArrowEnableVecForType + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, + T: ArrowFieldDeserialize + ArrowEnableVecForType + 'static, + ::ArrayType: 'static, + for<'b> &'b ::ArrayType: IntoIterator, { type ArrayType = FixedSizeListArray; - fn arrow_deserialize(v: Option>) -> Option<::Type> { + fn arrow_field_deserialize(v: Option>) -> Option<::Type> { arrow_deserialize_vec_helper::(v) } } @@ -263,8 +263,8 @@ where fn try_into_collection(self) -> arrow2::error::Result; fn try_into_collection_as_type(self) -> arrow2::error::Result where - ArrowType: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator; + ArrowType: ArrowFieldDeserialize + ArrowField + 'static, + for<'b> &'b ::ArrayType: IntoIterator; } /// Helper to return an iterator for elements from a [`arrow2::array::Array`]. @@ -272,12 +272,12 @@ fn arrow_array_deserialize_iterator_internal<'a, Element, Field>( b: &'a dyn arrow2::array::Array, ) -> arrow2::error::Result + 'a> where - Field: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, + Field: ArrowFieldDeserialize + ArrowField + 'static, + for<'b> &'b ::ArrayType: IntoIterator, { Ok( - <::ArrayType as ArrowArray>::iter_from_array_ref(b) - .map(::arrow_deserialize_internal), + <::ArrayType as ArrowArrayIterator>::iter_from_array_ref(b) + .map(::arrow_field_deserialize_internal), ) } @@ -286,8 +286,8 @@ pub fn arrow_array_deserialize_iterator_as_type<'a, Element, ArrowType>( ) -> arrow2::error::Result + 'a> where Element: 'static, - ArrowType: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, + ArrowType: ArrowFieldDeserialize + ArrowField + 'static, + for<'b> &'b ::ArrayType: IntoIterator, { if &::data_type() != arr.data_type() { Err(arrow2::error::Error::InvalidArgumentError( @@ -306,16 +306,16 @@ pub fn arrow_array_deserialize_iterator<'a, T>( arr: &'a dyn arrow2::array::Array, ) -> arrow2::error::Result + 'a> where - T: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, + T: ArrowFieldDeserialize + ArrowField + 'static, + for<'b> &'b ::ArrayType: IntoIterator, { arrow_array_deserialize_iterator_as_type::(arr) } impl<'a, Collection, Element, ArrowArray> TryIntoCollection for ArrowArray where - Element: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, + Element: ArrowFieldDeserialize + ArrowField + 'static, + for<'b> &'b ::ArrayType: IntoIterator, ArrowArray: std::borrow::Borrow, Collection: FromIterator, { @@ -325,8 +325,8 @@ where fn try_into_collection_as_type(self) -> arrow2::error::Result where - ArrowType: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, + ArrowType: ArrowFieldDeserialize + ArrowField + 'static, + for<'b> &'b ::ArrayType: IntoIterator, { Ok( arrow_array_deserialize_iterator_as_type::(self.borrow())? diff --git a/arrow2_convert/src/field.rs b/arrow2_convert/src/field.rs index 0301eb4..fe17731 100644 --- a/arrow2_convert/src/field.rs +++ b/arrow2_convert/src/field.rs @@ -12,7 +12,7 @@ use chrono::{NaiveDate, NaiveDateTime}; /// /// The trait simply requires defining the [`ArrowField::data_type`] /// -/// Serialize and Deserialize functionality requires implementing the [`crate::ArrowSerialize`] +/// Serialize and Deserialize functionality requires implementing the [`crate::ArrowFieldSerialize`] /// and the [`crate::ArrowDeserialize`] traits respectively. pub trait ArrowField { /// This should be `Self` except when implementing large offset and fixed placeholder types. diff --git a/arrow2_convert/src/serialize.rs b/arrow2_convert/src/serialize.rs index fd9c814..8661de2 100644 --- a/arrow2_convert/src/serialize.rs +++ b/arrow2_convert/src/serialize.rs @@ -7,10 +7,10 @@ use crate::field::*; /// Trait that is implemented by all types that are serializable to Arrow. /// /// Implementations are provided for all built-in arrow types as well as Vec, and Option -/// if T implements ArrowSerialize. +/// if T implements ArrowFieldSerialize. /// /// Note that Vec implementation needs to be enabled by the [`crate::arrow_enable_vec_for_type`] macro. -pub trait ArrowSerialize: ArrowField { +pub trait ArrowFieldSerialize: ArrowField { /// The [`arrow2::array::MutableArray`] that holds this value type MutableArrayType: ArrowMutableArray; @@ -18,7 +18,7 @@ pub trait ArrowSerialize: ArrowField { fn new_array() -> Self::MutableArrayType; /// Serialize this field to arrow - fn arrow_serialize( + fn arrow_field_serialize( v: &::Type, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()>; @@ -36,7 +36,7 @@ pub trait ArrowMutableArray: arrow2::array::MutableArray { // Macro to facilitate implementation of serializable traits for numeric types and numeric mutable arrays. macro_rules! impl_numeric_type { ($physical_type:ty, $logical_type:ident) => { - impl ArrowSerialize for $physical_type { + impl ArrowFieldSerialize for $physical_type { type MutableArrayType = MutablePrimitiveArray<$physical_type>; #[inline] @@ -45,7 +45,7 @@ macro_rules! impl_numeric_type { } #[inline] - fn arrow_serialize( + fn arrow_field_serialize( v: &Self, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { @@ -70,24 +70,24 @@ macro_rules! impl_mutable_array_body { } // blanket implementation for optional fields -impl ArrowSerialize for Option +impl ArrowFieldSerialize for Option where - T: ArrowSerialize, + T: ArrowFieldSerialize, { - type MutableArrayType = ::MutableArrayType; + type MutableArrayType = ::MutableArrayType; #[inline] fn new_array() -> Self::MutableArrayType { - ::new_array() + ::new_array() } #[inline] - fn arrow_serialize( + fn arrow_field_serialize( v: &::Type, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { match v.as_ref() { - Some(t) => ::arrow_serialize(t, array), + Some(t) => ::arrow_field_serialize(t, array), None => { array.push_null(); Ok(()) @@ -107,7 +107,7 @@ impl_numeric_type!(i64, Int64); impl_numeric_type!(f32, Float32); impl_numeric_type!(f64, Float64); -impl ArrowSerialize for String { +impl ArrowFieldSerialize for String { type MutableArrayType = MutableUtf8Array; #[inline] @@ -116,12 +116,12 @@ impl ArrowSerialize for String { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { array.try_push(Some(v)) } } -impl ArrowSerialize for LargeString { +impl ArrowFieldSerialize for LargeString { type MutableArrayType = MutableUtf8Array; #[inline] @@ -130,7 +130,7 @@ impl ArrowSerialize for LargeString { } #[inline] - fn arrow_serialize( + fn arrow_field_serialize( v: &String, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { @@ -138,7 +138,7 @@ impl ArrowSerialize for LargeString { } } -impl ArrowSerialize for bool { +impl ArrowFieldSerialize for bool { type MutableArrayType = MutableBooleanArray; #[inline] @@ -147,12 +147,12 @@ impl ArrowSerialize for bool { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { array.try_push(Some(*v)) } } -impl ArrowSerialize for NaiveDateTime { +impl ArrowFieldSerialize for NaiveDateTime { type MutableArrayType = MutablePrimitiveArray; #[inline] @@ -161,12 +161,12 @@ impl ArrowSerialize for NaiveDateTime { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { array.try_push(Some(v.timestamp_nanos())) } } -impl ArrowSerialize for NaiveDate { +impl ArrowFieldSerialize for NaiveDate { type MutableArrayType = MutablePrimitiveArray; #[inline] @@ -175,7 +175,7 @@ impl ArrowSerialize for NaiveDate { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { array.try_push(Some( chrono::Datelike::num_days_from_ce(v) - arrow2::temporal_conversions::EPOCH_DAYS_FROM_CE, @@ -183,7 +183,7 @@ impl ArrowSerialize for NaiveDate { } } -impl ArrowSerialize for Vec { +impl ArrowFieldSerialize for Vec { type MutableArrayType = MutableBinaryArray; #[inline] @@ -192,12 +192,12 @@ impl ArrowSerialize for Vec { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { array.try_push(Some(v)) } } -impl ArrowSerialize for LargeBinary { +impl ArrowFieldSerialize for LargeBinary { type MutableArrayType = MutableBinaryArray; #[inline] @@ -206,7 +206,7 @@ impl ArrowSerialize for LargeBinary { } #[inline] - fn arrow_serialize( + fn arrow_field_serialize( v: &Vec, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { @@ -214,7 +214,7 @@ impl ArrowSerialize for LargeBinary { } } -impl ArrowSerialize for FixedSizeBinary { +impl ArrowFieldSerialize for FixedSizeBinary { type MutableArrayType = MutableFixedSizeBinaryArray; #[inline] @@ -223,7 +223,7 @@ impl ArrowSerialize for FixedSizeBinary { } #[inline] - fn arrow_serialize( + fn arrow_field_serialize( v: &Vec, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { @@ -232,86 +232,86 @@ impl ArrowSerialize for FixedSizeBinary { } // Blanket implementation for Vec -impl ArrowSerialize for Vec +impl ArrowFieldSerialize for Vec where - T: ArrowSerialize + ArrowEnableVecForType + 'static, - ::MutableArrayType: Default, + T: ArrowFieldSerialize + ArrowEnableVecForType + 'static, + ::MutableArrayType: Default, { - type MutableArrayType = MutableListArray::MutableArrayType>; + type MutableArrayType = MutableListArray::MutableArrayType>; #[inline] fn new_array() -> Self::MutableArrayType { Self::MutableArrayType::new_with_field( - ::new_array(), + ::new_array(), "item", ::is_nullable(), ) } - fn arrow_serialize( + fn arrow_field_serialize( v: &::Type, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { let values = array.mut_values(); for i in v.iter() { - ::arrow_serialize(i, values)?; + ::arrow_field_serialize(i, values)?; } array.try_push_valid() } } -impl ArrowSerialize for LargeVec +impl ArrowFieldSerialize for LargeVec where - T: ArrowSerialize + ArrowEnableVecForType + 'static, - ::MutableArrayType: Default, + T: ArrowFieldSerialize + ArrowEnableVecForType + 'static, + ::MutableArrayType: Default, { - type MutableArrayType = MutableListArray::MutableArrayType>; + type MutableArrayType = MutableListArray::MutableArrayType>; #[inline] fn new_array() -> Self::MutableArrayType { Self::MutableArrayType::new_with_field( - ::new_array(), + ::new_array(), "item", ::is_nullable(), ) } - fn arrow_serialize( + fn arrow_field_serialize( v: &::Type, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { let values = array.mut_values(); for i in v.iter() { - ::arrow_serialize(i, values)?; + ::arrow_field_serialize(i, values)?; } array.try_push_valid() } } -impl ArrowSerialize for FixedSizeVec +impl ArrowFieldSerialize for FixedSizeVec where - T: ArrowSerialize + ArrowEnableVecForType + 'static, - ::MutableArrayType: Default, + T: ArrowFieldSerialize + ArrowEnableVecForType + 'static, + ::MutableArrayType: Default, { - type MutableArrayType = MutableFixedSizeListArray<::MutableArrayType>; + type MutableArrayType = MutableFixedSizeListArray<::MutableArrayType>; #[inline] fn new_array() -> Self::MutableArrayType { Self::MutableArrayType::new_with_field( - ::new_array(), + ::new_array(), "item", ::is_nullable(), SIZE, ) } - fn arrow_serialize( + fn arrow_field_serialize( v: &::Type, array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { let values = array.mut_values(); for i in v.iter() { - ::arrow_serialize(i, values)?; + ::arrow_field_serialize(i, values)?; } array.try_push_valid() } @@ -373,34 +373,34 @@ where } // internal helper method to extend a mutable array -fn arrow_serialize_extend_internal< +fn arrow_field_serialize_extend_internal< 'a, A: 'static, - T: ArrowSerialize + ArrowField + 'static, + T: ArrowFieldSerialize + ArrowField + 'static, I: IntoIterator, >( into_iter: I, - array: &mut ::MutableArrayType, + array: &mut ::MutableArrayType, ) -> arrow2::error::Result<()> { let iter = into_iter.into_iter(); array.reserve(iter.size_hint().0, 0); for i in iter { - ::arrow_serialize(i, array)?; + ::arrow_field_serialize(i, array)?; } Ok(()) } // internal helper method to serialize to a mutable array -fn arrow_serialize_internal< +fn arrow_field_serialize_internal< 'a, A: 'static, - T: ArrowSerialize + ArrowField + 'static, + T: ArrowFieldSerialize + ArrowField + 'static, I: IntoIterator, >( into_iter: I, -) -> arrow2::error::Result<::MutableArrayType> { - let mut arr = ::new_array(); - arrow_serialize_extend_internal::(into_iter, &mut arr)?; +) -> arrow2::error::Result<::MutableArrayType> { + let mut arr = ::new_array(); + arrow_field_serialize_extend_internal::(into_iter, &mut arr)?; Ok(arr) } @@ -413,39 +413,39 @@ where fn try_into_arrow(self) -> arrow2::error::Result; fn try_into_arrow_as_type(self) -> arrow2::error::Result where - ArrowType: ArrowSerialize + ArrowField + 'static; + ArrowType: ArrowFieldSerialize + ArrowField + 'static; } impl<'a, Element, Collection> TryIntoArrow<'a, Arc, Element> for Collection where - Element: ArrowSerialize + ArrowField + 'static, + Element: ArrowFieldSerialize + ArrowField + 'static, Collection: IntoIterator, { fn try_into_arrow(self) -> arrow2::error::Result> { - Ok(arrow_serialize_internal::(self)?.as_arc()) + Ok(arrow_field_serialize_internal::(self)?.as_arc()) } fn try_into_arrow_as_type(self) -> arrow2::error::Result> where - Field: ArrowSerialize + ArrowField + 'static, + Field: ArrowFieldSerialize + ArrowField + 'static, { - Ok(arrow_serialize_internal::(self)?.as_arc()) + Ok(arrow_field_serialize_internal::(self)?.as_arc()) } } impl<'a, Element, Collection> TryIntoArrow<'a, Box, Element> for Collection where - Element: ArrowSerialize + ArrowField + 'static, + Element: ArrowFieldSerialize + ArrowField + 'static, Collection: IntoIterator, { fn try_into_arrow(self) -> arrow2::error::Result> { - Ok(arrow_serialize_internal::(self)?.as_box()) + Ok(arrow_field_serialize_internal::(self)?.as_box()) } fn try_into_arrow_as_type(self) -> arrow2::error::Result> where - E: ArrowSerialize + ArrowField + 'static, + E: ArrowFieldSerialize + ArrowField + 'static, { - Ok(arrow_serialize_internal::(self)?.as_box()) + Ok(arrow_field_serialize_internal::(self)?.as_box()) } } diff --git a/arrow2_convert/tests/complex_example.rs b/arrow2_convert/tests/complex_example.rs index 844b9ad..673d5f9 100644 --- a/arrow2_convert/tests/complex_example.rs +++ b/arrow2_convert/tests/complex_example.rs @@ -77,7 +77,7 @@ pub struct CustomType(u64); /// To use with Arrow three traits need to be implemented: /// - ArrowField -/// - ArrowSerialize +/// - ArrowFieldSerialize /// - ArrowDeserialize impl arrow2_convert::field::ArrowField for CustomType { type Type = Self; @@ -92,7 +92,7 @@ impl arrow2_convert::field::ArrowField for CustomType { } } -impl arrow2_convert::serialize::ArrowSerialize for CustomType { +impl arrow2_convert::serialize::ArrowFieldSerialize for CustomType { type MutableArrayType = arrow2::array::MutablePrimitiveArray; #[inline] @@ -101,16 +101,16 @@ impl arrow2_convert::serialize::ArrowSerialize for CustomType { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { array.try_push(Some(v.0)) } } -impl arrow2_convert::deserialize::ArrowDeserialize for CustomType { +impl arrow2_convert::deserialize::ArrowFieldDeserialize for CustomType { type ArrayType = arrow2::array::PrimitiveArray; #[inline] - fn arrow_deserialize(v: Option<&u64>) -> Option { + fn arrow_field_deserialize(v: Option<&u64>) -> Option { v.map(|t| CustomType(*t)) } } diff --git a/arrow2_convert/tests/test_schema.rs b/arrow2_convert/tests/test_schema.rs index ebb1e75..8be9a52 100644 --- a/arrow2_convert/tests/test_schema.rs +++ b/arrow2_convert/tests/test_schema.rs @@ -85,7 +85,7 @@ fn test_schema_types() { } } - impl arrow2_convert::serialize::ArrowSerialize for CustomType { + impl arrow2_convert::serialize::ArrowFieldSerialize for CustomType { type MutableArrayType = arrow2::array::MutablePrimitiveArray; #[inline] @@ -94,7 +94,7 @@ fn test_schema_types() { } #[inline] - fn arrow_serialize( + fn arrow_field_serialize( _v: &Self, _array: &mut Self::MutableArrayType, ) -> arrow2::error::Result<()> { @@ -102,11 +102,11 @@ fn test_schema_types() { } } - impl arrow2_convert::deserialize::ArrowDeserialize for CustomType { + impl arrow2_convert::deserialize::ArrowFieldDeserialize for CustomType { type ArrayType = arrow2::array::PrimitiveArray; #[inline] - fn arrow_deserialize(_v: Option<&u64>) -> Option { + fn arrow_field_deserialize(_v: Option<&u64>) -> Option { unimplemented!(); } } diff --git a/arrow2_convert_derive/src/_struct.rs b/arrow2_convert_derive/src/_struct.rs index aa8ba7b..fa5a6a7 100644 --- a/arrow2_convert_derive/src/_struct.rs +++ b/arrow2_convert_derive/src/_struct.rs @@ -87,7 +87,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { let mutable_field_array_types = field_types .iter() - .map(|field_type| quote_spanned!( field_type.span() => <#field_type as arrow2_convert::serialize::ArrowSerialize>::MutableArrayType)) + .map(|field_type| quote_spanned!( field_type.span() => <#field_type as arrow2_convert::serialize::ArrowFieldSerialize>::MutableArrayType)) .collect::>(); let mut generated = quote!( @@ -127,7 +127,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { impl #mutable_array_name { pub fn new() -> Self { Self { - #(#field_names: <#field_types as arrow2_convert::serialize::ArrowSerialize>::new_array(),)* + #(#field_names: <#field_types as arrow2_convert::serialize::ArrowFieldSerialize>::new_array(),)* data_type: <#original_name as arrow2_convert::field::ArrowField>::data_type(), validity: None, } @@ -152,7 +152,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { if let Some(x) = self.validity.as_mut() { x.reserve(additional) } - #(<<#field_types as arrow2_convert::serialize::ArrowSerialize>::MutableArrayType as arrow2_convert::serialize::ArrowMutableArray>::reserve(&mut self.#field_names, additional, _additional_values);)* + #(<<#field_types as arrow2_convert::serialize::ArrowFieldSerialize>::MutableArrayType as arrow2_convert::serialize::ArrowMutableArray>::reserve(&mut self.#field_names, additional, _additional_values);)* } } @@ -165,7 +165,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { Some(i) => { let i = i.borrow(); #( - <#field_types as arrow2_convert::serialize::ArrowSerialize>::arrow_serialize(i.#field_names.borrow(), &mut self.#field_names)?; + <#field_types as arrow2_convert::serialize::ArrowFieldSerialize>::arrow_field_serialize(i.#field_names.borrow(), &mut self.#field_names)?; )*; match &mut self.validity { Some(validity) => validity.push(true), @@ -258,7 +258,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { } } - impl arrow2_convert::serialize::ArrowSerialize for #original_name { + impl arrow2_convert::serialize::ArrowFieldSerialize for #original_name { type MutableArrayType = #mutable_array_name; #[inline] @@ -266,7 +266,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { Self::MutableArrayType::default() } - fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { + fn arrow_field_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> { use arrow2::array::TryPush; array.try_push(Some(v)) } @@ -281,7 +281,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { array: Box } - impl arrow2_convert::deserialize::ArrowArray for #array_name + impl arrow2_convert::deserialize::ArrowArrayIterator for #array_name { type BaseArrayType = arrow2::array::StructArray; @@ -294,7 +294,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { // for now do a straight comp #iterator_name { #( - #field_names: <<#field_types as arrow2_convert::deserialize::ArrowDeserialize>::ArrayType as arrow2_convert::deserialize::ArrowArray>::iter_from_array_ref(values[#field_indices].deref()), + #field_names: <<#field_types as arrow2_convert::deserialize::ArrowFieldDeserialize>::ArrayType as arrow2_convert::deserialize::ArrowArrayIterator>::iter_from_array_ref(values[#field_indices].deref()), )* has_validity: validity.as_ref().is_some(), validity_iter: validity.as_ref().map(|x| x.iter()).unwrap_or_else(|| arrow2::bitmap::utils::BitmapIter::new(&[], 0, 0)) @@ -314,7 +314,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { #visibility struct #iterator_name<'a> { #( - #field_names: <&'a <#field_types as arrow2_convert::deserialize::ArrowDeserialize>::ArrayType as IntoIterator>::IntoIter, + #field_names: <&'a <#field_types as arrow2_convert::deserialize::ArrowFieldDeserialize>::ArrayType as IntoIterator>::IntoIter, )* validity_iter: arrow2::bitmap::utils::BitmapIter<'a>, has_validity: bool @@ -329,7 +329,7 @@ pub fn expand_derive(input: &Input) -> TokenStream { ) { Some(#original_name { - #(#field_names: <#field_types as arrow2_convert::deserialize::ArrowDeserialize>::arrow_deserialize_internal(#field_names),)* + #(#field_names: <#field_types as arrow2_convert::deserialize::ArrowFieldDeserialize>::arrow_field_deserialize_internal(#field_names),)* }) } else { @@ -356,10 +356,10 @@ pub fn expand_derive(input: &Input) -> TokenStream { } } - impl arrow2_convert::deserialize::ArrowDeserialize for #original_name { + impl arrow2_convert::deserialize::ArrowFieldDeserialize for #original_name { type ArrayType = #array_name; - fn arrow_deserialize<'a>(v: Option) -> Option { + fn arrow_field_deserialize<'a>(v: Option) -> Option { v } }