Skip to content

Commit

Permalink
Rename top traits for consistency (#28)
Browse files Browse the repository at this point in the history
  • Loading branch information
ncpenke authored Mar 3, 2022
1 parent 5c19ce8 commit 3532105
Show file tree
Hide file tree
Showing 7 changed files with 52 additions and 51 deletions.
14 changes: 7 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ Types (currently only structures) can be annotated with the `ArrowField` procedu
- A typed iterator for deserialization
- Implementations of the `ArrowField`, `ArrowSerialize`, and `ArrowDeserialize` traits.

Serialization can be performed by using the `arrow_serialize` method from the `IntoArrow` trait or by manually pushing elements to the generated `arrow2::array::MutableArray`.
Serialization can be performed by using the `arrow_serialize` method from the `TryIntoArrow` trait or by manually pushing elements to the generated `arrow2::array::MutableArray`.

Deserialization can be performed by using the `try_into_iter` method from the `TryIntoIterator` trait or by iterating through the iterator provided by `arrow_array_deserialize_iterator`.
Deserialization can be performed by using the `try_into_collection` method from the `TryIntoCollection` trait or by iterating through the iterator provided by `arrow_array_deserialize_iterator`.

Both serialization and deserialization perform memory copies for the elements converted. For example, iterating through the deserialize iterator will copy memory from the arrow2 array, into the structure that the iterator returns. Deserialization can be more efficient by supporting structs with references.

Expand All @@ -47,7 +47,7 @@ Please see the [complex_example.rs](./arrow2_convert/tests/complex_example.rs) f
/// Simple example

use arrow2::array::Array;
use arrow2_convert::{deserialize::TryIntoIter, serialize::IntoArrow, ArrowField};
use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField};

#[derive(Debug, Clone, PartialEq, ArrowField)]
pub struct Foo {
Expand All @@ -63,16 +63,16 @@ fn test_simple_roundtrip() {
Foo { name: "good bye".to_string() },
];

// serialize to an arrow array. into_arrow() is enabled by the IntoArrow trait
let arrow_array: Box<dyn Array> = original_array.into_arrow().unwrap();
// serialize to an arrow array. try_into_arrow() is enabled by the TryIntoArrow trait
let arrow_array: Box<dyn Array> = original_array.try_into_arrow().unwrap();

// which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc.
// supported by `arrow2`
let struct_array= arrow_array.as_any().downcast_ref::<arrow2::array::StructArray>().unwrap();
assert_eq!(struct_array.len(), 3);

// deserialize back to our original vector via TryIntoIter trait.
let round_trip_array: Vec<Foo> = arrow_array.try_into_iter().unwrap();
// deserialize back to our original vector via TryIntoCollection trait.
let round_trip_array: Vec<Foo> = arrow_array.try_into_collection().unwrap();
assert_eq!(round_trip_array, original_array);
}
```
Expand Down
15 changes: 8 additions & 7 deletions arrow2_convert/src/deserialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -217,12 +217,12 @@ impl_arrow_array!(ListArray<i32>);
impl_arrow_array!(ListArray<i64>);

/// Top-level API to deserialize from Arrow
pub trait TryIntoIter<Collection, Element>
pub trait TryIntoCollection<Collection, Element>
where Element: ArrowField,
Collection: FromIterator<Element>
{
fn try_into_iter(self) -> arrow2::error::Result<Collection>;
fn try_into_iter_as_type<ArrowType>(self) -> arrow2::error::Result<Collection>
fn try_into_collection(self) -> arrow2::error::Result<Collection>;
fn try_into_collection_as_type<ArrowType>(self) -> arrow2::error::Result<Collection>
where ArrowType: ArrowDeserialize + ArrowField<Type = Element> + 'static,
for<'b> &'b <ArrowType as ArrowDeserialize>::ArrayType: IntoIterator;
}
Expand Down Expand Up @@ -269,17 +269,18 @@ where
arrow_array_deserialize_iterator_as_type::<T, T>(arr)
}

impl<'a, Element, ArrowArray> TryIntoIter<Vec<Element>, Element> for ArrowArray
impl<'a, Collection, Element, ArrowArray> TryIntoCollection<Collection, Element> for ArrowArray
where
Element: ArrowDeserialize + ArrowField<Type = Element> + 'static,
for<'b> &'b <Element as ArrowDeserialize>::ArrayType: IntoIterator,
ArrowArray: std::borrow::Borrow<dyn Array>
ArrowArray: std::borrow::Borrow<dyn Array>,
Collection: FromIterator<Element>
{
fn try_into_iter(self) -> arrow2::error::Result<Vec<Element>> {
fn try_into_collection(self) -> arrow2::error::Result<Collection> {
Ok(arrow_array_deserialize_iterator::<Element>(self.borrow())?.collect())
}

fn try_into_iter_as_type<ArrowType>(self) -> arrow2::error::Result<Vec<Element>>
fn try_into_collection_as_type<ArrowType>(self) -> arrow2::error::Result<Collection>
where ArrowType: ArrowDeserialize + ArrowField<Type = Element> + 'static,
for<'b> &'b <ArrowType as ArrowDeserialize>::ArrayType: IntoIterator
{
Expand Down
18 changes: 9 additions & 9 deletions arrow2_convert/src/serialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,41 +286,41 @@ fn arrow_serialize_internal<'a, A: 'static, T: ArrowSerialize + ArrowField<Type
}

/// Top-level API to serialize to Arrow
pub trait IntoArrow<'a, ArrowArray, Element>
pub trait TryIntoArrow<'a, ArrowArray, Element>
where Self: IntoIterator<Item = &'a Element>,
Element: 'static
{
fn into_arrow(self) -> arrow2::error::Result<ArrowArray>;
fn into_arrow_as_type<ArrowType>(self) -> arrow2::error::Result<ArrowArray>
fn try_into_arrow(self) -> arrow2::error::Result<ArrowArray>;
fn try_into_arrow_as_type<ArrowType>(self) -> arrow2::error::Result<ArrowArray>
where ArrowType: ArrowSerialize + ArrowField<Type = Element> + 'static;
}

impl<'a, Element, Collection> IntoArrow<'a, Arc<dyn Array>, Element> for Collection
impl<'a, Element, Collection> TryIntoArrow<'a, Arc<dyn Array>, Element> for Collection
where
Element: ArrowSerialize + ArrowField<Type = Element> + 'static,
Collection: IntoIterator<Item = &'a Element>,
{
fn into_arrow(self) -> arrow2::error::Result<Arc<dyn Array>> {
fn try_into_arrow(self) -> arrow2::error::Result<Arc<dyn Array>> {
Ok(arrow_serialize_internal::<Element, Element, Collection>(self)?.as_arc())
}

fn into_arrow_as_type<Field>(self) -> arrow2::error::Result<Arc<dyn Array>>
fn try_into_arrow_as_type<Field>(self) -> arrow2::error::Result<Arc<dyn Array>>
where Field: ArrowSerialize + ArrowField<Type = Element> + 'static
{
Ok(arrow_serialize_internal::<Element, Field, Collection>(self)?.as_arc())
}
}

impl<'a, Element, Collection> IntoArrow<'a, Box<dyn Array>, Element> for Collection
impl<'a, Element, Collection> TryIntoArrow<'a, Box<dyn Array>, Element> for Collection
where
Element: ArrowSerialize + ArrowField<Type = Element> + 'static,
Collection: IntoIterator<Item = &'a Element>,
{
fn into_arrow(self) -> arrow2::error::Result<Box<dyn Array>> {
fn try_into_arrow(self) -> arrow2::error::Result<Box<dyn Array>> {
Ok(arrow_serialize_internal::<Element, Element, Collection>(self)?.as_box())
}

fn into_arrow_as_type<E>(self) -> arrow2::error::Result<Box<dyn Array>>
fn try_into_arrow_as_type<E>(self) -> arrow2::error::Result<Box<dyn Array>>
where E: ArrowSerialize + ArrowField<Type = Element> + 'static
{
Ok(arrow_serialize_internal::<Element, E, Collection>(self)?.as_box())
Expand Down
8 changes: 4 additions & 4 deletions arrow2_convert/tests/complex_example.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
/// - Custom types
use arrow2::array::*;
use arrow2_convert::deserialize::{arrow_array_deserialize_iterator, TryIntoIter};
use arrow2_convert::serialize::IntoArrow;
use arrow2_convert::deserialize::{arrow_array_deserialize_iterator, TryIntoCollection};
use arrow2_convert::serialize::TryIntoArrow;
use arrow2_convert::ArrowField;
use arrow2_convert::field::{LargeBinary, LargeString, LargeVec};
use std::borrow::Borrow;
Expand Down Expand Up @@ -206,7 +206,7 @@ fn test_round_trip() -> arrow2::error::Result<()> {
// serialize to an arrow array
let original_array = [item1(), item2()];

let array: Box<dyn Array> = original_array.into_arrow()?;
let array: Box<dyn Array> = original_array.try_into_arrow()?;
let struct_array = array
.as_any()
.downcast_ref::<arrow2::array::StructArray>()
Expand All @@ -223,7 +223,7 @@ fn test_round_trip() -> arrow2::error::Result<()> {
}

// or can back to our original vector
let foo_array: Vec<Root> = array.try_into_iter()?;
let foo_array: Vec<Root> = array.try_into_collection()?;
assert_eq!(foo_array, original_array);
Ok(())
}
10 changes: 5 additions & 5 deletions arrow2_convert/tests/simple_example.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/// Simple example
use arrow2::array::Array;
use arrow2_convert::{deserialize::TryIntoIter, serialize::IntoArrow, ArrowField};
use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField};

#[derive(Debug, Clone, PartialEq, ArrowField)]
pub struct Foo {
Expand All @@ -22,8 +22,8 @@ fn test_simple_roundtrip() {
},
];

// serialize to an arrow array. into_arrow() is enabled by the IntoArrow trait
let arrow_array: Box<dyn Array> = original_array.into_arrow().unwrap();
// serialize to an arrow array. try_into_arrow() is enabled by the TryIntoArrow trait
let arrow_array: Box<dyn Array> = original_array.try_into_arrow().unwrap();

// which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc.
// supported by `arrow2`
Expand All @@ -33,7 +33,7 @@ fn test_simple_roundtrip() {
.unwrap();
assert_eq!(struct_array.len(), 3);

// deserialize back to our original vector via TryIntoIter trait.
let round_trip_array: Vec<Foo> = arrow_array.try_into_iter().unwrap();
// deserialize back to our original vector via TryIntoCollection trait.
let round_trip_array: Vec<Foo> = arrow_array.try_into_collection().unwrap();
assert_eq!(round_trip_array, original_array);
}
10 changes: 5 additions & 5 deletions arrow2_convert/tests/test_deserialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ fn test_deserialize_iterator() {

let original_array = [S { a1: 1 }, S { a1: 100 }, S { a1: 1000 }];

let b: Box<dyn Array> = original_array.into_arrow().unwrap();
let b: Box<dyn Array> = original_array.try_into_arrow().unwrap();

let iter = arrow_array_deserialize_iterator::<S>(b.borrow()).unwrap();

Expand All @@ -40,9 +40,9 @@ fn test_deserialize_schema_mismatch_error() {
}

let arr1 = vec![S1 { a: 1 }, S1 { a: 2 }];
let arr1: Box<dyn Array> = arr1.into_arrow().unwrap();
let arr1: Box<dyn Array> = arr1.try_into_arrow().unwrap();

let result: Result<Vec<S2>> = arr1.try_into_iter();
let result: Result<Vec<S2>> = arr1.try_into_collection();
assert!(result.is_err());
}

Expand All @@ -59,8 +59,8 @@ fn test_deserialize_large_types_schema_mismatch_error() {
}

let arr1 = vec![S1 { a: "123".to_string() }, S1 { a: "333".to_string() }];
let arr1: Box<dyn Array> = arr1.into_arrow().unwrap();
let arr1: Box<dyn Array> = arr1.try_into_arrow().unwrap();

let result: Result<Vec<S2>> = arr1.try_into_iter();
let result: Result<Vec<S2>> = arr1.try_into_collection();
assert!(result.is_err());
}
28 changes: 14 additions & 14 deletions arrow2_convert/tests/test_round_trip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,87 +33,87 @@ fn test_nested_optional_struct_array() {
},
];

let b: Box<dyn Array> = original_array.into_arrow().unwrap();
let round_trip: Vec<Top> = b.try_into_iter().unwrap();
let b: Box<dyn Array> = original_array.try_into_arrow().unwrap();
let round_trip: Vec<Top> = b.try_into_collection().unwrap();
assert_eq!(original_array, round_trip);
}

#[test]
fn test_large_string()
{
let strs = vec!["1".to_string(), "2".to_string()];
let b: Box<dyn Array> = strs.into_arrow_as_type::<LargeString>().unwrap();
let b: Box<dyn Array> = strs.try_into_arrow_as_type::<LargeString>().unwrap();
assert_eq!(b.data_type(), &DataType::LargeUtf8);
let round_trip = b.try_into_iter_as_type::<LargeString>().unwrap();
let round_trip: Vec<String> = b.try_into_collection_as_type::<LargeString>().unwrap();
assert_eq!(round_trip, strs);
}

#[test]
fn test_large_string_nested()
{
let strs = [vec!["1".to_string(), "2".to_string()]];
let b: Box<dyn Array> = strs.into_arrow_as_type::<Vec<LargeString>>().unwrap();
let b: Box<dyn Array> = strs.try_into_arrow_as_type::<Vec<LargeString>>().unwrap();
assert_eq!(b.data_type(),
&DataType::List(Box::new(Field::new(
"item",
DataType::LargeUtf8,
false
))));
let round_trip = b.try_into_iter_as_type::<Vec<LargeString>>().unwrap();
let round_trip: Vec<Vec<String>> = b.try_into_collection_as_type::<Vec<LargeString>>().unwrap();
assert_eq!(round_trip, strs);
}

#[test]
fn test_large_binary()
{
let strs = [b"abc".to_vec()];
let b: Box<dyn Array> = strs.into_arrow_as_type::<LargeBinary>().unwrap();
let b: Box<dyn Array> = strs.try_into_arrow_as_type::<LargeBinary>().unwrap();
assert_eq!(b.data_type(), &DataType::LargeBinary);
let round_trip = b.try_into_iter_as_type::<LargeBinary>().unwrap();
let round_trip: Vec<Vec<u8>> = b.try_into_collection_as_type::<LargeBinary>().unwrap();
assert_eq!(round_trip, strs);
}

#[test]
fn test_large_binary_nested()
{
let strs = [vec![b"abc".to_vec(), b"abd".to_vec()]];
let b: Box<dyn Array> = strs.into_arrow_as_type::<Vec<LargeBinary>>().unwrap();
let b: Box<dyn Array> = strs.try_into_arrow_as_type::<Vec<LargeBinary>>().unwrap();
assert_eq!(b.data_type(),
&DataType::List(Box::new(Field::new(
"item",
DataType::LargeBinary,
false
))));
let round_trip = b.try_into_iter_as_type::<Vec<LargeBinary>>().unwrap();
let round_trip: Vec<Vec<Vec<u8>>> = b.try_into_collection_as_type::<Vec<LargeBinary>>().unwrap();
assert_eq!(round_trip, strs);
}

#[test]
fn test_large_vec()
{
let ints = vec![vec![1, 2, 3]];
let b: Box<dyn Array> = ints.into_arrow_as_type::<LargeVec<i32>>().unwrap();
let b: Box<dyn Array> = ints.try_into_arrow_as_type::<LargeVec<i32>>().unwrap();
assert_eq!(b.data_type(),
&DataType::LargeList(Box::new(Field::new(
"item",
DataType::Int32,
false
))));
let round_trip = b.try_into_iter_as_type::<LargeVec<i32>>().unwrap();
let round_trip: Vec<Vec<i32>> = b.try_into_collection_as_type::<LargeVec<i32>>().unwrap();
assert_eq!(round_trip, ints);
}

#[test]
fn test_large_vec_nested()
{
let strs = [vec![b"abc".to_vec(), b"abd".to_vec()]];
let b: Box<dyn Array> = strs.into_arrow_as_type::<LargeVec<LargeBinary>>().unwrap();
let b: Box<dyn Array> = strs.try_into_arrow_as_type::<LargeVec<LargeBinary>>().unwrap();
assert_eq!(b.data_type(),
&DataType::LargeList(Box::new(Field::new(
"item",
DataType::LargeBinary,
false
))));
let round_trip = b.try_into_iter_as_type::<LargeVec<LargeBinary>>().unwrap();
let round_trip: Vec<Vec<Vec<u8>>> = b.try_into_collection_as_type::<LargeVec<LargeBinary>>().unwrap();
assert_eq!(round_trip, strs);
}

0 comments on commit 3532105

Please sign in to comment.