Update to latest arrow2 + new datatypes

Cargo.toml

@@ -39,6 +39,8 @@ either = "1.8"
 
 [workspace.dependencies.arrow]
 package = "arrow2"
+git = "https://github.com/rerun-io/arrow2"
+branch = "cmc/arc_datatype"
 # git = "https://github.com/jorgecarleitao/arrow2"
 # git = "https://github.com/ritchie46/arrow2"
 # rev = "f258a3e06ac408aebe7a7a497694729dc65a5e46"

polars/polars-arrow/src/conversion.rs

@@ -6,7 +6,7 @@ use arrow::types::NativeType;
 use crate::prelude::*;
 
 pub fn chunk_to_struct(chunk: Chunk<ArrayRef>, fields: Vec<Field>) -> StructArray {
-    let dtype = DataType::Struct(fields);
+    let dtype = DataType::Struct(std::sync::Arc::new(fields));
     StructArray::new(dtype, chunk.into_arrays(), None)
 }
 

polars/polars-core/src/chunked_array/logical/categorical/from.rs

@@ -12,7 +12,7 @@ impl From<&CategoricalChunked> for DictionaryArray<u32> {
         let map = &**ca.get_rev_map();
         let dtype = ArrowDataType::Dictionary(
             IntegerType::UInt32,
-            Box::new(ArrowDataType::LargeUtf8),
+            Arc::new(ArrowDataType::LargeUtf8),
             false,
         );
         match map {
@@ -47,7 +47,7 @@ impl From<&CategoricalChunked> for DictionaryArray<i64> {
         let map = &**ca.get_rev_map();
         let dtype = ArrowDataType::Dictionary(
             IntegerType::UInt32,
-            Box::new(ArrowDataType::LargeUtf8),
+            Arc::new(ArrowDataType::LargeUtf8),
             false,
         );
         match map {

polars/polars-core/src/chunked_array/logical/struct_/mod.rs

@@ -40,7 +40,11 @@ fn fields_to_struct_array(fields: &[Series]) -> (ArrayRef, Vec<Series>) {
     // we determine fields from arrays as there might be object arrays
     // where the dtype is bound to that single array
     let new_fields = arrays_to_fields(&field_arrays, &fields);
-    let arr = StructArray::new(ArrowDataType::Struct(new_fields), field_arrays, None);
+    let arr = StructArray::new(
+        ArrowDataType::Struct(Arc::new(new_fields)),
+        field_arrays,
+        None,
+    );
     (Box::new(arr), fields)
 }
 
@@ -116,7 +120,7 @@ impl StructChunked {
             // where the dtype is bound to that single array
             let new_fields = arrays_to_fields(&field_arrays, &self.fields);
             let arr = Box::new(StructArray::new(
-                ArrowDataType::Struct(new_fields),
+                ArrowDataType::Struct(Arc::new(new_fields)),
                 field_arrays,
                 None,
             )) as ArrayRef;

polars/polars-core/src/chunked_array/ops/full.rs

@@ -104,7 +104,7 @@ impl ChunkFullNull for ListChunked {
 impl ListChunked {
     pub fn full_null_with_dtype(name: &str, length: usize, inner_dtype: &DataType) -> ListChunked {
         let arr = new_null_array(
-            ArrowDataType::LargeList(Box::new(ArrowField::new(
+            ArrowDataType::LargeList(Arc::new(ArrowField::new(
                 "item",
                 inner_dtype.to_arrow(),
                 true,

polars/polars-core/src/datatypes/any_value.rs

@@ -793,7 +793,7 @@ mod test {
                 DataType::Datetime(TimeUnit::Milliseconds, None),
             ),
             (
-                ArrowDataType::Timestamp(ArrowTimeUnit::Second, Some("".to_string())),
+                ArrowDataType::Timestamp(ArrowTimeUnit::Second, Some(Arc::new("".to_string()))),
                 DataType::Datetime(TimeUnit::Milliseconds, Some("".to_string())),
             ),
             (ArrowDataType::LargeUtf8, DataType::Utf8),
@@ -827,15 +827,15 @@ mod test {
             ),
             (ArrowDataType::Time32(ArrowTimeUnit::Second), DataType::Time),
             (
-                ArrowDataType::List(Box::new(ArrowField::new(
+                ArrowDataType::List(Arc::new(ArrowField::new(
                     "item",
                     ArrowDataType::Float64,
                     true,
                 ))),
                 DataType::List(DataType::Float64.into()),
             ),
             (
-                ArrowDataType::LargeList(Box::new(ArrowField::new(
+                ArrowDataType::LargeList(Arc::new(ArrowField::new(
                     "item",
                     ArrowDataType::Float64,
                     true,

polars/polars-core/src/datatypes/dtype.rs

@@ -216,10 +216,12 @@ impl DataType {
             Utf8 => ArrowDataType::LargeUtf8,
             Binary => ArrowDataType::LargeBinary,
             Date => ArrowDataType::Date32,
-            Datetime(unit, tz) => ArrowDataType::Timestamp(unit.to_arrow(), tz.clone()),
+            Datetime(unit, tz) => {
+                ArrowDataType::Timestamp(unit.to_arrow(), tz.clone().map(Arc::new))
+            }
             Duration(unit) => ArrowDataType::Duration(unit.to_arrow()),
             Time => ArrowDataType::Time64(ArrowTimeUnit::Nanosecond),
-            List(dt) => ArrowDataType::LargeList(Box::new(arrow::datatypes::Field::new(
+            List(dt) => ArrowDataType::LargeList(Arc::new(arrow::datatypes::Field::new(
                 "item",
                 dt.to_arrow(),
                 true,
@@ -230,13 +232,13 @@ impl DataType {
             #[cfg(feature = "dtype-categorical")]
             Categorical(_) => ArrowDataType::Dictionary(
                 IntegerType::UInt32,
-                Box::new(ArrowDataType::LargeUtf8),
+                Arc::new(ArrowDataType::LargeUtf8),
                 false,
             ),
             #[cfg(feature = "dtype-struct")]
             Struct(fields) => {
                 let fields = fields.iter().map(|fld| fld.to_arrow()).collect();
-                ArrowDataType::Struct(fields)
+                ArrowDataType::Struct(Arc::new(fields))
             }
             Unknown => unreachable!(),
         }

polars/polars-core/src/datatypes/field.rs

@@ -127,7 +127,7 @@ impl From<&ArrowDataType> for DataType {
             ArrowDataType::LargeList(f) => DataType::List(Box::new(f.data_type().into())),
             ArrowDataType::List(f) => DataType::List(Box::new(f.data_type().into())),
             ArrowDataType::Date32 => DataType::Date,
-            ArrowDataType::Timestamp(tu, tz) => DataType::Datetime(tu.into(), tz.clone()),
+            ArrowDataType::Timestamp(tu, tz) => DataType::Datetime(tu.into(), tz.as_ref().map(|tz| tz.to_string())),
             ArrowDataType::Duration(tu) => DataType::Duration(tu.into()),
             ArrowDataType::Date64 => DataType::Datetime(TimeUnit::Milliseconds, None),
             ArrowDataType::LargeUtf8 | ArrowDataType::Utf8 => DataType::Utf8,

polars/polars-core/src/series/from.rs

@@ -154,13 +154,13 @@ impl Series {
             #[cfg(feature = "dtype-datetime")]
             ArrowDataType::Timestamp(tu, tz) => {
                 let mut tz = tz.clone();
-                if tz.as_deref() == Some("") {
+                if tz.as_ref().map(|tz| tz.as_str()) == Some("") {
                     tz = None;
                 }
                 // we still drop timezone for now
                 let chunks = cast_chunks(&chunks, &DataType::Int64, false).unwrap();
                 let s = Int64Chunked::from_chunks(name, chunks)
-                    .into_datetime(tu.into(), tz)
+                    .into_datetime(tu.into(), tz.map(|tz| tz.to_string()))
                     .into_series();
                 Ok(match tu {
                     ArrowTimeUnit::Second => &s * MILLISECONDS,
@@ -461,7 +461,7 @@ fn convert_inner_types(arr: &ArrayRef) -> ArrayRef {
                 .map(|(arr, field)| ArrowField::new(&field.name, arr.data_type().clone(), true))
                 .collect();
             Box::new(StructArray::new(
-                ArrowDataType::Struct(fields),
+                ArrowDataType::Struct(Arc::new(fields)),
                 values,
                 arr.validity().cloned(),
             ))