From c925537c48ffa2bd927dd33f30af462483fea657 Mon Sep 17 00:00:00 2001 From: Stefan Holderbach Date: Wed, 11 Oct 2023 21:28:18 +0200 Subject: [PATCH] Update `polars` to `0.33` (#10672) # Description Open question: Undocumented behavior for the new argument `ambiguous` to the `as_datetime` methods. I cheated by passing a default (assuming empty string). This appears like an API primarily serving the python impl: https://pola-rs.github.io/polars/py-polars/html/reference/expressions/api/polars.Expr.str.to_datetime.html#polars-expr-str-to-datetime # User-Facing Changes Only dependent on breaking changes to the behavior of polars. # Tests + Formatting No observed changes to tests Manually checked `dfr as-datetime`, doesn't seem to panic. --- Cargo.lock | 91 ++++++++++--------- crates/nu-cmd-dataframe/Cargo.toml | 6 +- .../src/dataframe/eager/sql_context.rs | 2 +- .../src/dataframe/lazy/groupby.rs | 2 +- .../src/dataframe/series/date/as_datetime.rs | 4 +- .../src/dataframe/series/masks/is_in.rs | 8 +- 6 files changed, 59 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57d7dbf5f..43681fa04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,9 +208,9 @@ dependencies = [ [[package]] name = "arrow2" -version = "0.17.4" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59c468daea140b747d781a1da9f7db5f0a8e6636d4af20cc539e43d05b0604fa" +checksum = "963fef509b757bcbbf9e5ffa23bcb345614d99f4f6f531f97417b27b8604d389" dependencies = [ "ahash", "arrow-format", @@ -226,13 +226,14 @@ dependencies = [ "futures", "getrandom", "hash_hasher", + "hashbrown 0.14.1", "lexical-core", "lz4", "multiversion", "num-traits", "parquet2", "regex", - "regex-syntax 0.6.29", + "regex-syntax", "rustc_version", "simdutf8", "streaming-iterator", @@ -904,6 +905,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -3684,9 +3695,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1362d4a136c0ebacb40d88a37ba361738b222fd8a2ee9340a3d8642f698c52b" +checksum = "3030de163b9ff2c9dac9a12dcb9be25cc0f2bc7c8e7cd2e4b2592ebed458ce6a" dependencies = [ "getrandom", "polars-core", @@ -3700,9 +3711,9 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f967c901fa5da4ca7f64e813d1268488ba97e9b3004cefc579ff851c197a1138" +checksum = "35cd38a64fb389fd990e4efd433a36331c995c981d353bfef83b5de4d87f1828" dependencies = [ "arrow2", "hashbrown 0.14.1", @@ -3716,9 +3727,9 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b24f92fc5b167f668ff85ab9607dfa72e2c09664cacef59297ee8601dee60126" +checksum = "08367c014c07fa8f141680e024f926cab3a1fe839605a8fcf2223647eb45ca71" dependencies = [ "ahash", "arrow2", @@ -3748,9 +3759,9 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d09c3a7337e53b38c37b57999038440fa39c6801b9ba48afaecd8e16f7ac0a" +checksum = "9b20a09651a299979354945819dc2ce017964b80b916954e9d2ce39002a5f949" dependencies = [ "arrow2", "regex", @@ -3759,17 +3770,16 @@ dependencies = [ [[package]] name = "polars-io" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92cab0df9f2a35702fa5aec99edfaabf9ae8e9cdd0acf69e143ad2d132f34f9c" +checksum = "88cf4a89c18a90ac20dfbcdfd19ab50ad4ac5a76fc7bb775d3c28bb738cf1f34" dependencies = [ "ahash", "arrow2", - "async-trait", "bytes", "chrono", "fast-float", - "futures", + "flate2", "home", "lexical", "lexical-core", @@ -3789,14 +3799,13 @@ dependencies = [ "serde_json", "simd-json", "simdutf8", - "tokio", ] [[package]] name = "polars-json" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e58094557cf6446808c7348dcb797885db61815857f6ea02924b35505566e94" +checksum = "d6d5666176d681706aef5a06a57597c83391948b3d958f9fbe9b4cf016527eb8" dependencies = [ "ahash", "arrow2", @@ -3812,9 +3821,9 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c33762ec2a55e01c9f8776b34db86257c70a0a3b3929bd4eb91a52aacf61456" +checksum = "5110eab438848c981cc5f541fbc5b21bb263fd707000b4715233074fb2630fcf" dependencies = [ "ahash", "bitflags 2.4.0", @@ -3836,9 +3845,9 @@ dependencies = [ [[package]] name = "polars-ops" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825575c96302d2daedfc205a0062180033c92c55bcd6aafc4e109d4d8849ed0" +checksum = "7740d7bc4c2ca08044f9ef599638e116fdd7d687e80d1974b698e390c6ce4252" dependencies = [ "argminmax", "arrow2", @@ -3848,6 +3857,7 @@ dependencies = [ "polars-arrow", "polars-core", "polars-utils", + "regex", "serde", "smartstring", "version_check", @@ -3855,10 +3865,12 @@ dependencies = [ [[package]] name = "polars-pipe" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f2bc9a12da9ed043fb0cb51dbcb87b365e4845b7ab6399d7a81e838460c6974" +checksum = "1f30c5e77c5594ddc958a46fe2e021da2feba9c94e767e1d798bd82ac5a33c3b" dependencies = [ + "crossbeam-channel", + "crossbeam-queue", "enum_dispatch", "hashbrown 0.14.1", "num-traits", @@ -3876,9 +3888,9 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb67b014f0295e8e9dbb84404a91d666d477b3bc248a2ed51bc442833b16da35" +checksum = "678cbeb730e29e50f0f8d844102d15454fc6113a74c667eab046c0e4a4322a9e" dependencies = [ "ahash", "arrow2", @@ -3899,9 +3911,9 @@ dependencies = [ [[package]] name = "polars-row" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f54c1956027bf6301948fb4f2837cf6d6b638d8dd1edf3aaeaa19906a986be" +checksum = "c52ef8885b9d13f848839594fbab21ad79fc63f7e11c19cdc2cfe9bb03c313ac" dependencies = [ "arrow2", "polars-error", @@ -3910,9 +3922,9 @@ dependencies = [ [[package]] name = "polars-sql" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfcb15cf8eebd25ea1724109d0153817cd484c6326290585f0736b4e7fcf2f4" +checksum = "4d716855267e3516f722287f68cf10e650e33f7197df83a79e680602471456fc" dependencies = [ "polars-arrow", "polars-core", @@ -3925,9 +3937,9 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f42d2632f5971c9575041d33cbcfb1f996900c40bbf58bc6eb0a0c5efbecea" +checksum = "a2eb75a24f11b55a400b52dc19a2a3e949aaaa46a911f99496de4485b1127063" dependencies = [ "arrow2", "atoi", @@ -3945,11 +3957,12 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.32.1" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c326708a370d71dc6e11a8f4bbc10a8479e1c314dc048ba73543b815cd0bf339" +checksum = "2a4a5e743509096322cad39104d56e329fe2748483a3354a0f0c354724f3cef6" dependencies = [ "ahash", + "bytemuck", "hashbrown 0.14.1", "num-traits", "once_cell", @@ -4269,7 +4282,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.7.5", + "regex-syntax", ] [[package]] @@ -4280,15 +4293,9 @@ checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.7.5" diff --git a/crates/nu-cmd-dataframe/Cargo.toml b/crates/nu-cmd-dataframe/Cargo.toml index 6d9f9ea30..ffd801e61 100644 --- a/crates/nu-cmd-dataframe/Cargo.toml +++ b/crates/nu-cmd-dataframe/Cargo.toml @@ -24,7 +24,7 @@ indexmap = { version = "2.0" } num = { version = "0.4", optional = true } serde = { version = "1.0", features = ["derive"] } sqlparser = { version = "0.36.1", optional = true } -polars-io = { version = "0.32", features = ["avro"], optional = true } +polars-io = { version = "0.33", features = ["avro"], optional = true } [dependencies.polars] features = [ @@ -38,7 +38,7 @@ features = [ "dtype-categorical", "dtype-datetime", "dtype-struct", - "dynamic_groupby", + "dynamic_group_by", "ipc", "is_in", "json", @@ -54,7 +54,7 @@ features = [ "to_dummies", ] optional = true -version = "0.32" +version = "0.33" [features] dataframe = ["num", "polars", "polars-io", "sqlparser"] diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs index ac2e2c50a..42ab272c2 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs @@ -147,7 +147,7 @@ impl SQLContext { .enumerate() .map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len()))) .unzip(); - let agg_df = df.groupby(group_by).agg(agg_projection); + let agg_df = df.group_by(group_by).agg(agg_projection); let mut final_proj_pos = groupby_pos .into_iter() .chain(agg_proj_pos) diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs index 048c62130..8f8d293f7 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs @@ -131,7 +131,7 @@ impl Command for ToLazyGroupBy { let group_by = NuLazyGroupBy { schema: lazy.schema.clone(), from_eager: lazy.from_eager, - group_by: Some(lazy.into_polars().groupby(&expressions)), + group_by: Some(lazy.into_polars().group_by(&expressions)), }; Ok(PipelineData::Value(group_by.into_value(call.head), None)) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs index 8a31b1c83..bf13c51f7 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs @@ -148,7 +148,7 @@ fn command( TimeUnit::Nanoseconds, false, None, - None, + &Default::default(), ) } else { casted.as_datetime( @@ -157,7 +157,7 @@ fn command( false, false, None, - None, + &Default::default(), ) }; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs index f9a0dab39..8dbe36612 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs @@ -6,7 +6,7 @@ use nu_protocol::{ engine::{Command, EngineState, Stack}, Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value, }; -use polars::prelude::IntoSeries; +use polars::prelude::{is_in, IntoSeries}; #[derive(Clone)] pub struct IsIn; @@ -71,16 +71,14 @@ fn command( call: &Call, input: PipelineData, ) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; + let df = NuDataFrame::try_from_pipeline(input, call.head)?.as_series(call.head)?; let other_value: Value = call.req(engine_state, stack, 0)?; let other_span = other_value.span(); let other_df = NuDataFrame::try_from_value(other_value)?; let other = other_df.as_series(other_span)?; - let mut res = df - .as_series(call.head)? - .is_in(&other) + let mut res = is_in(&df, &other) .map_err(|e| { ShellError::GenericError( "Error finding in other".into(),