8000 Upgrade to DataFusion 33.0.0-rc1 (#530) · llama90/arrow-datafusion-python@e4942b8 · GitHub
[go: up one dir, main page]
Skip to content

Commit e4942b8

Browse files
andygroveandygrove
authored
Upgrade to DataFusion 33.0.0-rc1 (apache#530)
* fix issues * fix TODO * clippy * use df 33.0.0-rc1
1 parent da6c183 commit e4942b8

File tree

9 files changed

+299
-309
lines changed

9 files changed

+299
-309
lines changed

‎Cargo.lock

Lines changed: 274 additions & 288 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
[package]
1919
name = "datafusion-python"
20-
version = "32.0.0"
20+
version = "33.0.0"
2121
homepage = "https://github.com/apache/arrow-datafusion-python"
2222
repository = "https://github.com/apache/arrow-datafusion-python"
2323
authors = ["Apache Arrow <dev@arrow.apache.org>"]
@@ -36,15 +36,15 @@ substrait = ["dep:datafusion-substrait"]
3636
[dependencies]
3737
tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
3838
rand = "0.8"
39-
pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] }
40-
datafusion = { version = "32.0.0", features = ["pyarrow", "avro"] }
41-
datafusion-common = { version = "32.0.0", features = ["pyarrow"] }
42-
datafusion-expr = { version = "32.0.0" }
43-
datafusion-optimizer = { version = "32.0.0" }
44-
datafusion-sql = { version = "32.0.0" }
45-
datafusion-substrait = { version = "32.0.0", optional = true }
46-
prost = "0.11"
47-
prost-types = "0.11"
39+
pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] }
40+
datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", features = ["pyarrow", "avro"] }
41+
datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", features = ["pyarrow"] }
42+
datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" }
43+
datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" }
44+
datafusion-sql = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" }
45+
datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", optional = true }
46+
prost = "0.12"
47+
prost-types = "0.12"
4848
uuid = { version = "1.3", features = ["v4"] }
4949
mimalloc = { version = "0.1", optional = true, default-features = false }
5050
async-trait = "0.1"

‎src/common/data_type.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ impl DataTypeMap {
288288
ScalarValue::IntervalMonthDayNano(..) => {
289289
Ok(DataType::Interval(IntervalUnit::MonthDayNano))
290290
}
291-
ScalarValue::List(_val, field_ref) => Ok(DataType::List(field_ref.to_owned())),
291+
ScalarValue::List(arr) => Ok(arr.data_type().to_owned()),
292292
ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())),
293293
ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)),
294294
ScalarValue::Fixedsizelist(_, field_ref, size) => {

‎src/dataset_exec.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,13 +133,14 @@ impl DatasetExec {
133133
.downcast()
134134
.map_err(PyErr::from)?;
135135

136+
let projected_statistics = Statistics::new_unknown(&schema);
136137
Ok(DatasetExec {
137138
dataset: dataset.into(),
138139
schema,
139140
fragments: fragments.into(),
140141
columns,
141142
filter_expr,
142-
projected_statistics: Default::default(),
143+
projected_statistics,
143144
})
144145
}
145146
}
@@ -236,8 +237,8 @@ impl ExecutionPlan for DatasetExec {
236237
})
237238
}
238239

239-
fn statistics(&self) -> Statistics {
240-
self.projected_statistics.clone()
240+
fn statistics(&self) -> DFResult<Statistics> {
241+
Ok(self.projected_statistics.clone())
241242
}
242243
}
243244

‎src/expr.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ impl PyExpr {
332332
ScalarValue::Binary(v) => v.clone().into_py(py),
333333
ScalarValue::FixedSizeBinary(_, _) => todo!(),
334334
ScalarValue::LargeBinary(v) => v.clone().into_py(py),
335-
ScalarValue::List(_, _) => todo!(),
335+
ScalarValue::List(_) => todo!(),
336336
ScalarValue::Date32(v) => v.into_py(py),
337337
ScalarValue::Date64(v) => v.into_py(py),
338338
ScalarValue::Time32Second(v) => v.into_py(py),

‎src/expr/window.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,10 @@ fn not_window_function_err(expr: Expr) -> PyErr {
166166

167167
#[pymethods]
168168
impl PyWindowFrame {
169-
#[new(unit, start_bound, end_bound)]
170-
pub fn new(units: &str, start_bound: Option<u64>, end_bound: Option<u64>) -> PyResult<Self> {
171-
let units = units.to_ascii_lowercase();
169+
#[new]
170+
#[pyo3(signature=(unit, start_bound, end_bound))]
171+
pub fn new(unit: &str, start_bound: Option<u64>, end_bound: Option<u64>) -> PyResult<Self> {
172+
let units = unit.to_ascii_lowercase();
172173
let units = match units.as_str() {
173174
"rows" => WindowFrameUnits::Rows,
174175
"range" => WindowFrameUnits::Range,

‎src/substrait.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ impl PySubstraitConsumer {
134134
plan: PyPlan,
135135
py: Python,
136136
) -> PyResult<PyLogicalPlan> {
137-
let result = consumer::from_substrait_plan(&mut ctx.ctx, &plan.plan);
137+
let result = consumer::from_substrait_plan(&ctx.ctx, &plan.plan);
138138
let logical_plan = wait_for_future(py, result).map_err(DataFusionError::from)?;
139139
Ok(PyLogicalPlan::new(logical_plan))
140140
}

‎src/udaf.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,8 @@ pub struct PyAggregateUDF {
143143

144144
#[pymethods]
145145
impl PyAggregateUDF {
146-
#[new(name, accumulator, input_type, return_type, state_type, volatility)]
146+
#[new]
147+
#[pyo3(signature=(name, accumulator, input_type, return_type, state_type, volatility))]
147148
fn new(
148149
name: &str,
149150
accumulator: PyObject,

‎src/udf.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ pub struct PyScalarUDF {
6868

6969
#[pymethods]
7070
impl PyScalarUDF {
71-
#[new(name, func, input_types, return_type, volatility)]
71+
#[new]
72+
#[pyo3(signature=(name, func, input_types, return_type, volatility))]
7273
fn new(
7374
name: &str,
7475
func: PyObject,

0 commit comments

Comments
 (0)
0