diff --git a/README.md b/README.md index 0221e34..690e64f 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,9 @@ Types (currently only structures) can be annotated with the `ArrowField` procedu - A typed iterator for deserialization - Implementations of the `ArrowField`, `ArrowSerialize`, and `ArrowDeserialize` traits. -Serialization can be performed by using the `arrow_serialize` method from the `IntoArrow` trait or by manually pushing elements to the generated `arrow2::array::MutableArray`. +Serialization can be performed by using the `arrow_serialize` method from the `TryIntoArrow` trait or by manually pushing elements to the generated `arrow2::array::MutableArray`. -Deserialization can be performed by using the `try_into_iter` method from the `TryIntoIterator` trait or by iterating through the iterator provided by `arrow_array_deserialize_iterator`. +Deserialization can be performed by using the `try_into_collection` method from the `TryIntoCollection` trait or by iterating through the iterator provided by `arrow_array_deserialize_iterator`. Both serialization and deserialization perform memory copies for the elements converted. For example, iterating through the deserialize iterator will copy memory from the arrow2 array, into the structure that the iterator returns. Deserialization can be more efficient by supporting structs with references. @@ -47,7 +47,7 @@ Please see the [complex_example.rs](./arrow2_convert/tests/complex_example.rs) f /// Simple example use arrow2::array::Array; -use arrow2_convert::{deserialize::TryIntoIter, serialize::IntoArrow, ArrowField}; +use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField}; #[derive(Debug, Clone, PartialEq, ArrowField)] pub struct Foo { @@ -63,16 +63,16 @@ fn test_simple_roundtrip() { Foo { name: "good bye".to_string() }, ]; - // serialize to an arrow array. into_arrow() is enabled by the IntoArrow trait - let arrow_array: Box = original_array.into_arrow().unwrap(); + // serialize to an arrow array. try_into_arrow() is enabled by the TryIntoArrow trait + let arrow_array: Box = original_array.try_into_arrow().unwrap(); // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc. // supported by `arrow2` let struct_array= arrow_array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.len(), 3); - // deserialize back to our original vector via TryIntoIter trait. - let round_trip_array: Vec = arrow_array.try_into_iter().unwrap(); + // deserialize back to our original vector via TryIntoCollection trait. + let round_trip_array: Vec = arrow_array.try_into_collection().unwrap(); assert_eq!(round_trip_array, original_array); } ``` diff --git a/arrow2_convert/src/deserialize.rs b/arrow2_convert/src/deserialize.rs index 227df82..fef386a 100644 --- a/arrow2_convert/src/deserialize.rs +++ b/arrow2_convert/src/deserialize.rs @@ -217,12 +217,12 @@ impl_arrow_array!(ListArray); impl_arrow_array!(ListArray); /// Top-level API to deserialize from Arrow -pub trait TryIntoIter +pub trait TryIntoCollection where Element: ArrowField, Collection: FromIterator { - fn try_into_iter(self) -> arrow2::error::Result; - fn try_into_iter_as_type(self) -> arrow2::error::Result + fn try_into_collection(self) -> arrow2::error::Result; + fn try_into_collection_as_type(self) -> arrow2::error::Result where ArrowType: ArrowDeserialize + ArrowField + 'static, for<'b> &'b ::ArrayType: IntoIterator; } @@ -269,17 +269,18 @@ where arrow_array_deserialize_iterator_as_type::(arr) } -impl<'a, Element, ArrowArray> TryIntoIter, Element> for ArrowArray +impl<'a, Collection, Element, ArrowArray> TryIntoCollection for ArrowArray where Element: ArrowDeserialize + ArrowField + 'static, for<'b> &'b ::ArrayType: IntoIterator, - ArrowArray: std::borrow::Borrow + ArrowArray: std::borrow::Borrow, + Collection: FromIterator { - fn try_into_iter(self) -> arrow2::error::Result> { + fn try_into_collection(self) -> arrow2::error::Result { Ok(arrow_array_deserialize_iterator::(self.borrow())?.collect()) } - fn try_into_iter_as_type(self) -> arrow2::error::Result> + fn try_into_collection_as_type(self) -> arrow2::error::Result where ArrowType: ArrowDeserialize + ArrowField + 'static, for<'b> &'b ::ArrayType: IntoIterator { diff --git a/arrow2_convert/src/serialize.rs b/arrow2_convert/src/serialize.rs index 5c0868e..6bc5fde 100644 --- a/arrow2_convert/src/serialize.rs +++ b/arrow2_convert/src/serialize.rs @@ -286,41 +286,41 @@ fn arrow_serialize_internal<'a, A: 'static, T: ArrowSerialize + ArrowField +pub trait TryIntoArrow<'a, ArrowArray, Element> where Self: IntoIterator, Element: 'static { - fn into_arrow(self) -> arrow2::error::Result; - fn into_arrow_as_type(self) -> arrow2::error::Result + fn try_into_arrow(self) -> arrow2::error::Result; + fn try_into_arrow_as_type(self) -> arrow2::error::Result where ArrowType: ArrowSerialize + ArrowField + 'static; } -impl<'a, Element, Collection> IntoArrow<'a, Arc, Element> for Collection +impl<'a, Element, Collection> TryIntoArrow<'a, Arc, Element> for Collection where Element: ArrowSerialize + ArrowField + 'static, Collection: IntoIterator, { - fn into_arrow(self) -> arrow2::error::Result> { + fn try_into_arrow(self) -> arrow2::error::Result> { Ok(arrow_serialize_internal::(self)?.as_arc()) } - fn into_arrow_as_type(self) -> arrow2::error::Result> + fn try_into_arrow_as_type(self) -> arrow2::error::Result> where Field: ArrowSerialize + ArrowField + 'static { Ok(arrow_serialize_internal::(self)?.as_arc()) } } -impl<'a, Element, Collection> IntoArrow<'a, Box, Element> for Collection +impl<'a, Element, Collection> TryIntoArrow<'a, Box, Element> for Collection where Element: ArrowSerialize + ArrowField + 'static, Collection: IntoIterator, { - fn into_arrow(self) -> arrow2::error::Result> { + fn try_into_arrow(self) -> arrow2::error::Result> { Ok(arrow_serialize_internal::(self)?.as_box()) } - fn into_arrow_as_type(self) -> arrow2::error::Result> + fn try_into_arrow_as_type(self) -> arrow2::error::Result> where E: ArrowSerialize + ArrowField + 'static { Ok(arrow_serialize_internal::(self)?.as_box()) diff --git a/arrow2_convert/tests/complex_example.rs b/arrow2_convert/tests/complex_example.rs index d52cf03..180026d 100644 --- a/arrow2_convert/tests/complex_example.rs +++ b/arrow2_convert/tests/complex_example.rs @@ -4,8 +4,8 @@ /// - Custom types use arrow2::array::*; -use arrow2_convert::deserialize::{arrow_array_deserialize_iterator, TryIntoIter}; -use arrow2_convert::serialize::IntoArrow; +use arrow2_convert::deserialize::{arrow_array_deserialize_iterator, TryIntoCollection}; +use arrow2_convert::serialize::TryIntoArrow; use arrow2_convert::ArrowField; use arrow2_convert::field::{LargeBinary, LargeString, LargeVec}; use std::borrow::Borrow; @@ -206,7 +206,7 @@ fn test_round_trip() -> arrow2::error::Result<()> { // serialize to an arrow array let original_array = [item1(), item2()]; - let array: Box = original_array.into_arrow()?; + let array: Box = original_array.try_into_arrow()?; let struct_array = array .as_any() .downcast_ref::() @@ -223,7 +223,7 @@ fn test_round_trip() -> arrow2::error::Result<()> { } // or can back to our original vector - let foo_array: Vec = array.try_into_iter()?; + let foo_array: Vec = array.try_into_collection()?; assert_eq!(foo_array, original_array); Ok(()) } diff --git a/arrow2_convert/tests/simple_example.rs b/arrow2_convert/tests/simple_example.rs index 62db4da..e145e1f 100644 --- a/arrow2_convert/tests/simple_example.rs +++ b/arrow2_convert/tests/simple_example.rs @@ -1,6 +1,6 @@ /// Simple example use arrow2::array::Array; -use arrow2_convert::{deserialize::TryIntoIter, serialize::IntoArrow, ArrowField}; +use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField}; #[derive(Debug, Clone, PartialEq, ArrowField)] pub struct Foo { @@ -22,8 +22,8 @@ fn test_simple_roundtrip() { }, ]; - // serialize to an arrow array. into_arrow() is enabled by the IntoArrow trait - let arrow_array: Box = original_array.into_arrow().unwrap(); + // serialize to an arrow array. try_into_arrow() is enabled by the TryIntoArrow trait + let arrow_array: Box = original_array.try_into_arrow().unwrap(); // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc. // supported by `arrow2` @@ -33,7 +33,7 @@ fn test_simple_roundtrip() { .unwrap(); assert_eq!(struct_array.len(), 3); - // deserialize back to our original vector via TryIntoIter trait. - let round_trip_array: Vec = arrow_array.try_into_iter().unwrap(); + // deserialize back to our original vector via TryIntoCollection trait. + let round_trip_array: Vec = arrow_array.try_into_collection().unwrap(); assert_eq!(round_trip_array, original_array); } diff --git a/arrow2_convert/tests/test_deserialize.rs b/arrow2_convert/tests/test_deserialize.rs index e08b623..6752c6d 100644 --- a/arrow2_convert/tests/test_deserialize.rs +++ b/arrow2_convert/tests/test_deserialize.rs @@ -19,7 +19,7 @@ fn test_deserialize_iterator() { let original_array = [S { a1: 1 }, S { a1: 100 }, S { a1: 1000 }]; - let b: Box = original_array.into_arrow().unwrap(); + let b: Box = original_array.try_into_arrow().unwrap(); let iter = arrow_array_deserialize_iterator::(b.borrow()).unwrap(); @@ -40,9 +40,9 @@ fn test_deserialize_schema_mismatch_error() { } let arr1 = vec![S1 { a: 1 }, S1 { a: 2 }]; - let arr1: Box = arr1.into_arrow().unwrap(); + let arr1: Box = arr1.try_into_arrow().unwrap(); - let result: Result> = arr1.try_into_iter(); + let result: Result> = arr1.try_into_collection(); assert!(result.is_err()); } @@ -59,8 +59,8 @@ fn test_deserialize_large_types_schema_mismatch_error() { } let arr1 = vec![S1 { a: "123".to_string() }, S1 { a: "333".to_string() }]; - let arr1: Box = arr1.into_arrow().unwrap(); + let arr1: Box = arr1.try_into_arrow().unwrap(); - let result: Result> = arr1.try_into_iter(); + let result: Result> = arr1.try_into_collection(); assert!(result.is_err()); } diff --git a/arrow2_convert/tests/test_round_trip.rs b/arrow2_convert/tests/test_round_trip.rs index 49078db..61f90dc 100644 --- a/arrow2_convert/tests/test_round_trip.rs +++ b/arrow2_convert/tests/test_round_trip.rs @@ -33,8 +33,8 @@ fn test_nested_optional_struct_array() { }, ]; - let b: Box = original_array.into_arrow().unwrap(); - let round_trip: Vec = b.try_into_iter().unwrap(); + let b: Box = original_array.try_into_arrow().unwrap(); + let round_trip: Vec = b.try_into_collection().unwrap(); assert_eq!(original_array, round_trip); } @@ -42,9 +42,9 @@ fn test_nested_optional_struct_array() { fn test_large_string() { let strs = vec!["1".to_string(), "2".to_string()]; - let b: Box = strs.into_arrow_as_type::().unwrap(); + let b: Box = strs.try_into_arrow_as_type::().unwrap(); assert_eq!(b.data_type(), &DataType::LargeUtf8); - let round_trip = b.try_into_iter_as_type::().unwrap(); + let round_trip: Vec = b.try_into_collection_as_type::().unwrap(); assert_eq!(round_trip, strs); } @@ -52,14 +52,14 @@ fn test_large_string() fn test_large_string_nested() { let strs = [vec!["1".to_string(), "2".to_string()]]; - let b: Box = strs.into_arrow_as_type::>().unwrap(); + let b: Box = strs.try_into_arrow_as_type::>().unwrap(); assert_eq!(b.data_type(), &DataType::List(Box::new(Field::new( "item", DataType::LargeUtf8, false )))); - let round_trip = b.try_into_iter_as_type::>().unwrap(); + let round_trip: Vec> = b.try_into_collection_as_type::>().unwrap(); assert_eq!(round_trip, strs); } @@ -67,9 +67,9 @@ fn test_large_string_nested() fn test_large_binary() { let strs = [b"abc".to_vec()]; - let b: Box = strs.into_arrow_as_type::().unwrap(); + let b: Box = strs.try_into_arrow_as_type::().unwrap(); assert_eq!(b.data_type(), &DataType::LargeBinary); - let round_trip = b.try_into_iter_as_type::().unwrap(); + let round_trip: Vec> = b.try_into_collection_as_type::().unwrap(); assert_eq!(round_trip, strs); } @@ -77,14 +77,14 @@ fn test_large_binary() fn test_large_binary_nested() { let strs = [vec![b"abc".to_vec(), b"abd".to_vec()]]; - let b: Box = strs.into_arrow_as_type::>().unwrap(); + let b: Box = strs.try_into_arrow_as_type::>().unwrap(); assert_eq!(b.data_type(), &DataType::List(Box::new(Field::new( "item", DataType::LargeBinary, false )))); - let round_trip = b.try_into_iter_as_type::>().unwrap(); + let round_trip: Vec>> = b.try_into_collection_as_type::>().unwrap(); assert_eq!(round_trip, strs); } @@ -92,14 +92,14 @@ fn test_large_binary_nested() fn test_large_vec() { let ints = vec![vec![1, 2, 3]]; - let b: Box = ints.into_arrow_as_type::>().unwrap(); + let b: Box = ints.try_into_arrow_as_type::>().unwrap(); assert_eq!(b.data_type(), &DataType::LargeList(Box::new(Field::new( "item", DataType::Int32, false )))); - let round_trip = b.try_into_iter_as_type::>().unwrap(); + let round_trip: Vec> = b.try_into_collection_as_type::>().unwrap(); assert_eq!(round_trip, ints); } @@ -107,13 +107,13 @@ fn test_large_vec() fn test_large_vec_nested() { let strs = [vec![b"abc".to_vec(), b"abd".to_vec()]]; - let b: Box = strs.into_arrow_as_type::>().unwrap(); + let b: Box = strs.try_into_arrow_as_type::>().unwrap(); assert_eq!(b.data_type(), &DataType::LargeList(Box::new(Field::new( "item", DataType::LargeBinary, false )))); - let round_trip = b.try_into_iter_as_type::>().unwrap(); + let round_trip: Vec>> = b.try_into_collection_as_type::>().unwrap(); assert_eq!(round_trip, strs); }