Update to Rust Polars 0.48

stijnherfst · stijnherfst · commit 43fcc96a6c23 · 2025-05-20T14:58:14.000+02:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,18 +9,18 @@ members = [
 ]
 
 [workspace.dependencies]
-polars = { version = "0.46.0", default-features = false }
-polars-core = { version = "0.46.0", default-features = false }
-polars-arrow = { version = "0.46.0", default-features = false }
-polars-ffi = { version = "0.46.0", default-features = false }
-polars-plan = { version = "0.46.0", default-features = false }
-polars-lazy = { version = "0.46.0", default-features = false }
-polars-python = {version = "0.46.0", default-features = false }
-polars-utils = {version = "0.46.0", default-features = false}
+polars = { version = "0.48.0", default-features = false }
+polars-core = { version = "0.48.0", default-features = false }
+polars-arrow = { version = "0.48.0", default-features = false }
+polars-ffi = { version = "0.48.0", default-features = false }
+polars-plan = { version = "0.48.0", default-features = false }
+polars-lazy = { version = "0.48.0", default-features = false }
+polars-python = {version = "0.48.0", default-features = false }
+polars-utils = {version = "0.48.0", default-features = false}
 
 [workspace.dependencies.arrow]
 package = "polars-arrow"
-version = "0.46.0"
+version = "0.48.0"
 path = "../polars/crates/polars-arrow"
 default-features = false
 
diff --git a/example/derive_expression/expression_lib/Cargo.toml b/example/derive_expression/expression_lib/Cargo.toml
@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
 [dependencies]
 polars = { workspace = true, features = ["fmt", "dtype-date", "timezones"], default-features = false }
 polars-arrow = { workspace = true, default-features = false }
-pyo3 = { version = "0.23", features = ["abi3-py38"] }
+pyo3 = { version = "0.24.2", features = ["abi3-py38"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive"] }
 rayon = "1.7.0"
 serde = { version = "1", features = ["derive"] }
diff --git a/example/derive_expression/expression_lib/pyproject.toml b/example/derive_expression/expression_lib/pyproject.toml
@@ -4,6 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "expression_lib"
+version = "0.0.1"
 requires-python = ">=3.8"
 classifiers = [
   "Programming Language :: Rust",
diff --git a/example/derive_expression/expression_lib/src/distances.rs b/example/derive_expression/expression_lib/src/distances.rs
@@ -25,6 +25,7 @@ fn jacc_helper<T: NativeType + Hash + Eq>(a: &PrimitiveArray<T>, b: &PrimitiveAr
     s3_len as f64 / (s1.len() + s2.len() - s3_len) as f64
 }
 
+#[allow(unexpected_cfgs)]
 pub(super) fn naive_jaccard_sim(a: &ListChunked, b: &ListChunked) -> PolarsResult<Float64Chunked> {
     polars_ensure!(
         a.inner_dtype() == b.inner_dtype(),
diff --git a/example/derive_expression/expression_lib/src/expressions.rs b/example/derive_expression/expression_lib/src/expressions.rs
@@ -193,7 +193,10 @@ struct TimeZone {
 
 fn convert_timezone(input_fields: &[Field], kwargs: TimeZone) -> PolarsResult<Field> {
     FieldsMapper::new(input_fields).try_map_dtype(|dtype| match dtype {
-        DataType::Datetime(tu, _) => Ok(DataType::Datetime(*tu, Some(kwargs.tz.into()))),
+        DataType::Datetime(tu, _) => Ok(DataType::Datetime(
+            *tu,
+            datatypes::TimeZone::opt_try_new(Some(kwargs.tz))?,
+        )),
         _ => polars_bail!(ComputeError: "expected datetime"),
     })
 }
@@ -206,6 +209,11 @@ fn change_time_zone(input: &[Series], kwargs: TimeZone) -> PolarsResult<Series>
     let ca = input.datetime()?;
 
     let mut out = ca.clone();
-    out.set_time_zone(kwargs.tz.into())?;
+
+    let Some(timezone) = datatypes::TimeZone::opt_try_new(Some(kwargs.tz))? else {
+        polars_bail!(ComputeError: "expected timezone")
+    };
+
+    out.set_time_zone(timezone)?;
     Ok(out.into_series())
 }
diff --git a/example/extend_polars_python_dispatch/extend_polars/Cargo.toml b/example/extend_polars_python_dispatch/extend_polars/Cargo.toml
@@ -9,9 +9,9 @@ name = "extend_polars"
 crate-type = ["cdylib"]
 
 [dependencies]
-polars = { workspace = true, features = ["fmt"] }
+polars = { workspace = true, features = ["fmt", "object"] }
 polars-core = { workspace = true }
 polars-lazy = { workspace = true }
-pyo3 = { version = "0.23", features = ["extension-module"] }
+pyo3 = { version = "0.24.2", features = ["extension-module"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["lazy"] }
 rayon = "1.10"
diff --git a/example/io_plugin/io_plugin/Cargo.toml b/example/io_plugin/io_plugin/Cargo.toml
@@ -9,8 +9,8 @@ name = "io_plugin"
 crate-type = ["cdylib"]
 
 [dependencies]
-polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy"], default-features = false }
+polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy", "object", "strings"], default-features = false }
 polars-arrow = { workspace = true, default-features = false }
-pyo3 = { version = "0.23", features = ["abi3-py38"] }
+pyo3 = { version = "0.24.2", features = ["abi3-py38"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive", "lazy"] }
 rand = { version = "0.8.5", features = [] }
diff --git a/example/io_plugin/io_plugin/io_plugin/__init__.py b/example/io_plugin/io_plugin/io_plugin/__init__.py
@@ -42,4 +42,4 @@ def source_generator(
 
     # create src again to compute the schema
     src = RandomSource(samplers, 0, 0)
-    return register_io_source(callable=source_generator, schema=src.schema())
+    return register_io_source(io_source=source_generator, schema=src.schema())
diff --git a/example/io_plugin/io_plugin/pyproject.toml b/example/io_plugin/io_plugin/pyproject.toml
@@ -4,6 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "io_plugin"
+version = "0.0.1"
 requires-python = ">=3.10"
 classifiers = [
   "Programming Language :: Rust",
diff --git a/example/io_plugin/io_plugin/src/samplers.rs b/example/io_plugin/io_plugin/src/samplers.rs
@@ -52,7 +52,7 @@ where
     }
 
     fn dtype(&self) -> DataType {
-        T::PolarsType::get_dtype()
+        T::PolarsType::get_static_dtype()
     }
 
     fn next_n(&mut self, n: usize) -> Series {
diff --git a/pyo3-polars-derive/Cargo.toml b/pyo3-polars-derive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-polars-derive"
-version = "0.14.0"
+version = "0.15.0"
 edition = "2021"
 license = "MIT"
 readme = "README.md"
@@ -27,3 +27,4 @@ syn = { version = "2", features = ["full", "extra-traits"] }
 
 [dev-dependencies]
 trybuild = { version = "1", features = ["diff"] }
+pyo3-polars = { version = "0.21.0", path = "../pyo3-polars", features = ["derive"] }
diff --git a/pyo3-polars-derive/tests/01.rs b/pyo3-polars-derive/tests/01.rs
@@ -1,17 +1,19 @@
 use polars_core::error::PolarsResult;
-use polars_core::prelude::{Field, Series};
+use polars_core::prelude::*;
 use polars_plan::dsl::FieldsMapper;
 use pyo3_polars_derive::polars_expr;
 
 fn horizontal_product_output(input_fields: &[Field]) -> PolarsResult<Field> {
     FieldsMapper::new(input_fields).map_to_supertype()
 }
 
-#[polars_expr(type_func=horizontal_product_output)]
-fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult<Series> {
+#[polars_expr(output_type_func=horizontal_product_output)]
+fn horizontal_product(series: &[Series], kwargs: Option<&str>) -> PolarsResult<Series> {
+    let _ = kwargs;
+
     let mut acc = series[0].clone();
     for s in &series[1..] {
-        acc = &acc * s
+        acc = (&acc * s)?
     }
     Ok(acc)
 }
diff --git a/pyo3-polars-derive/tests/02.rs b/pyo3-polars-derive/tests/02.rs
@@ -1,12 +1,14 @@
 use polars_core::error::PolarsResult;
-use polars_core::prelude::Series;
+use polars_core::prelude::*;
 use pyo3_polars_derive::polars_expr;
 
 #[polars_expr(output_type=Int32)]
-fn horizontal_product(series: &[Series], _kwargs: Option<&str>) -> PolarsResult<Series> {
+fn horizontal_product(series: &[Series], kwargs: Option<&str>) -> PolarsResult<Series> {
+    let _ = kwargs;
+
     let mut acc = series[0].clone();
     for s in &series[1..] {
-        acc = &acc * s
+        acc = (&acc * s)?
     }
     Ok(acc)
 }
diff --git a/pyo3-polars/Cargo.toml b/pyo3-polars/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-polars"
-version = "0.20.0"
+version = "0.21.0"
 edition = "2021"
 license = "MIT"
 readme = "../README.md"
@@ -19,8 +19,8 @@ polars-ffi = { workspace = true, optional = true }
 polars-lazy = { workspace = true, optional = true }
 polars-plan = { workspace = true, optional = true }
 polars-utils = {workspace = true, features = ["serde"], optional = true }
-pyo3 = "0.23"
-pyo3-polars-derive = { version = "0.14.0", path = "../pyo3-polars-derive", optional = true }
+pyo3 = "0.24.2"
+pyo3-polars-derive = { version = "0.15.0", path = "../pyo3-polars-derive", optional = true }
 serde = { version = "1", optional = true }
 serde-pickle = { version = "1", optional = true }
 thiserror = "1"
diff --git a/pyo3-polars/src/error.rs b/pyo3-polars/src/error.rs
@@ -2,7 +2,9 @@ use std::fmt::{Debug, Formatter};
 
 use polars::prelude::PolarsError;
 use pyo3::create_exception;
-use pyo3::exceptions::{PyException, PyIOError, PyIndexError, PyRuntimeError, PyValueError};
+use pyo3::exceptions::{
+    PyAssertionError, PyException, PyIOError, PyIndexError, PyRuntimeError, PyValueError,
+};
 use pyo3::prelude::*;
 use thiserror::Error;
 
@@ -16,8 +18,9 @@ pub enum PyPolarsErr {
 
 impl std::convert::From<PyPolarsErr> for PyErr {
     fn from(err: PyPolarsErr) -> PyErr {
-        fn convert(err: &PolarsError) -> PyErr {
+        fn convert(err: PolarsError) -> PyErr {
             match err {
+                PolarsError::AssertionError(error) => PyAssertionError::new_err(error.to_string()),
                 PolarsError::ComputeError(err) => ComputeError::new_err(err.to_string()),
                 PolarsError::NoData(err) => NoDataError::new_err(err.to_string()),
                 PolarsError::ShapeMismatch(err) => ShapeError::new_err(err.to_string()),
@@ -38,14 +41,16 @@ impl std::convert::From<PyPolarsErr> for PyErr {
                 }
                 PolarsError::SQLInterface(err) => SQLInterface::new_err(err.to_string()),
                 PolarsError::SQLSyntax(err) => SQLSyntax::new_err(err.to_string()),
-                PolarsError::Context { error, .. } => convert(error),
+                PolarsError::Context { error, .. } => convert(*error),
+                #[cfg(feature = "lazy")]
+                PolarsError::Python { error } => error.0,
             }
         }
 
         use PyPolarsErr::*;
-        match &err {
+        match err {
             Polars(err) => convert(err),
-            _ => PyRuntimeError::new_err(format!("{:?}", &err)),
+            _ => PyRuntimeError::new_err(format!("{err:?}")),
         }
     }
 }
@@ -60,6 +65,7 @@ impl Debug for PyPolarsErr {
     }
 }
 
+create_exception!(exceptions, AssertionError, PyException);
 create_exception!(exceptions, ColumnNotFound, PyException);
 create_exception!(exceptions, SchemaFieldNotFound, PyException);
 create_exception!(exceptions, StructFieldNotFound, PyException);
diff --git a/pyo3-polars/src/types.rs b/pyo3-polars/src/types.rs
@@ -11,9 +11,9 @@ use polars_core::utils::materialize_dyn_int;
 #[cfg(feature = "lazy")]
 use polars_lazy::frame::LazyFrame;
 #[cfg(feature = "lazy")]
-use polars_plan::dsl::Expr;
+use polars_plan::dsl::DslPlan;
 #[cfg(feature = "lazy")]
-use polars_plan::plans::DslPlan;
+use polars_plan::dsl::Expr;
 #[cfg(feature = "lazy")]
 use polars_utils::pl_serialize;
 use pyo3::exceptions::{PyTypeError, PyValueError};
@@ -23,7 +23,10 @@ use pyo3::prelude::*;
 use pyo3::pybacked::PyBackedStr;
 #[cfg(feature = "dtype-struct")]
 use pyo3::types::PyList;
-use pyo3::types::{PyBytes, PyDict, PyString};
+use pyo3::types::{PyDict, PyString};
+
+#[cfg(feature = "lazy")]
+use pyo3::types::PyBytes;
 
 #[cfg(feature = "dtype-categorical")]
 pub(crate) fn get_series(obj: &Bound<'_, PyAny>) -> PyResult<Series> {
@@ -218,7 +221,7 @@ impl<'a> FromPyObject<'a> for PyLazyFrame {
         let b = b.as_bytes();
 
         let lp: DslPlan = pl_serialize::SerializeOptions::default()
-            .deserialize_from_reader(&*b)
+            .deserialize_from_reader::<DslPlan, &[u8], false>(&*b)
             .map_err(
             |e| PyPolarsErr::Other(
                 format!("Error when deserializing LazyFrame. This may be due to mismatched polars versions. {}", e)
@@ -235,7 +238,7 @@ impl<'a> FromPyObject<'a> for PyExpr {
         let s = ob.call_method0("__getstate__")?.extract::<Vec<u8>>()?;
 
         let e: Expr = pl_serialize::SerializeOptions::default()
-            .deserialize_from_reader(&*s)
+            .deserialize_from_reader::<Expr, &[u8], false>(&*s)
             .map_err(
             |e| PyPolarsErr::Other(
                 format!("Error when deserializing 'Expr'. This may be due to mismatched polars versions. {}", e)
@@ -356,7 +359,7 @@ impl<'py> IntoPyObject<'py> for PyLazyFrame {
         let instance = cls.call_method1(intern!(py, "__new__"), (&cls,)).unwrap();
 
         let buf = pl_serialize::SerializeOptions::default()
-            .serialize_to_bytes(&self.0.logical_plan)
+            .serialize_to_bytes::<DslPlan, false>(&self.0.logical_plan)
             .unwrap();
         instance.call_method1("__setstate__", (&buf,))?;
         Ok(instance)
@@ -375,7 +378,7 @@ impl<'py> IntoPyObject<'py> for PyExpr {
         let instance = cls.call_method1(intern!(py, "__new__"), (&cls,))?;
 
         let buf = pl_serialize::SerializeOptions::default()
-            .serialize_to_bytes(&self.0)
+            .serialize_to_bytes::<Expr, false>(&self.0)
             .unwrap();
 
         instance
@@ -660,7 +663,7 @@ impl<'py> FromPyObject<'py> for PyDataType {
                 let time_unit = time_unit.extract::<PyTimeUnit>()?.0;
                 let time_zone = ob.getattr(intern!(py, "time_zone")).unwrap();
                 let time_zone: Option<String> = time_zone.extract()?;
-                DataType::Datetime(time_unit, time_zone.map(PlSmallStr::from))
+                DataType::Datetime(time_unit, TimeZone::opt_try_new(time_zone).unwrap())
             },
             "Duration" => {
                 let time_unit = ob.getattr(intern!(py, "time_unit")).unwrap();
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2025-01-05"
+channel = "nightly-2025-04-19"

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ fn jacc_helper<T: NativeType + Hash + Eq>(a: &PrimitiveArray<T>, b: &PrimitiveAr`
`25`	`25`	`s3_len as f64 / (s1.len() + s2.len() - s3_len) as f64`
`26`	`26`	`}`
`27`	`27`
	`28`	`+#[allow(unexpected_cfgs)]`
`28`	`29`	`pub(super) fn naive_jaccard_sim(a: &ListChunked, b: &ListChunked) -> PolarsResult<Float64Chunked> {`
`29`	`30`	`polars_ensure!(`
`30`	`31`	`a.inner_dtype() == b.inner_dtype(),`
Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ where`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`fn dtype(&self) -> DataType {`
`55`		`- T::PolarsType::get_dtype()`
	`55`	`+ T::PolarsType::get_static_dtype()`
`56`	`56`	`}`
`57`	`57`
`58`	`58`	`fn next_n(&mut self, n: usize) -> Series {`