chore: adopt new datafusion crate

Signed-off-by: R. Tyler Croy <[email protected]>
Signed-off-by: Rush <[email protected]>

Cargo.toml

@@ -45,15 +45,15 @@ object_store = { version = "0.11" }
 parquet = { version = "53" }
 
 # datafusion
-datafusion = { version = "41" }
-datafusion-expr = { version = "41" }
-datafusion-common = { version = "41" }
-datafusion-proto = { version = "41" }
-datafusion-sql = { version = "41" }
-datafusion-physical-expr = { version = "41" }
-datafusion-physical-plan = { version = "41" }
-datafusion-functions = { version = "41" }
-datafusion-functions-aggregate = { version = "41" }
+datafusion = { version = "42" }
+datafusion-expr = { version = "42" }
+datafusion-common = { version = "42" }
+datafusion-proto = { version = "42" }
+datafusion-sql = { version = "42" }
+datafusion-physical-expr = { version = "42" }
+datafusion-physical-plan = { version = "42" }
+datafusion-functions = { version = "42" }
+datafusion-functions-aggregate = { version = "42" }
 
 # serde
 serde = { version = "1.0.194", features = ["derive"] }

crates/core/src/delta_datafusion/mod.rs

@@ -21,6 +21,7 @@
 //! ```
 
 use std::any::Any;
+use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
 use std::fmt::{self, Debug};
 use std::sync::Arc;
@@ -707,7 +708,7 @@ impl TableProvider for DeltaTable {
         None
     }
 
-    fn get_logical_plan(&self) -> Option<&LogicalPlan> {
+    fn get_logical_plan(&self) -> Option<Cow<'_, LogicalPlan>> {
         None
     }
 
@@ -796,7 +797,7 @@ impl TableProvider for DeltaTableProvider {
         None
     }
 
-    fn get_logical_plan(&self) -> Option<&LogicalPlan> {
+    fn get_logical_plan(&self) -> Option<Cow<'_, LogicalPlan>> {
         None
     }
 

crates/core/src/kernel/scalars.rs

@@ -1,5 +1,5 @@
 //! Auxiliary methods for dealing with kernel scalars
-use std::{cmp::Ordering, fmt::Debug};
+use std::cmp::Ordering;
 
 use arrow_array::Array;
 use arrow_schema::TimeUnit;
@@ -270,6 +270,7 @@ impl ScalarExt for Scalar {
             Self::Binary(val) => Value::String(create_escaped_binary_string(val.as_slice())),
             Self::Null(_) => Value::Null,
             Self::Struct(_) => unimplemented!(),
+            Self::Array(_) => unimplemented!(),
         }
     }
 }

crates/core/src/writer/stats.rs

@@ -613,7 +613,7 @@ mod tests {
                 Some($value),
                 Some($value),
                 None,
-                0,
+                Some(0),
                 false,
             ))
         };

crates/core/tests/command_merge.rs

@@ -173,6 +173,7 @@ async fn test_merge_different_range() {
     let (_table_ref1, _metrics) = merge(table_ref1, df1, expr.clone()).await.unwrap();
     let result = merge(table_ref2, df2, expr).await;
 
+    println!("{result:#?}");
     assert!(result.is_ok());
 }
 

crates/hdfs/Cargo.toml

@@ -13,7 +13,7 @@ rust-version.workspace = true
 
 [dependencies]
 deltalake-core = { version = "0.21.0", path = "../core" }
-hdfs-native-object-store = "0.11"
+hdfs-native-object-store = "0.12"
 
 # workspace dependecies
 object_store = { workspace = true }