Merge branch 'main' into typos

Author: sebastiantia

acceptance/tests/other.rs

@@ -25,7 +25,7 @@ async fn test_read_last_checkpoint() {
     let url = url::Url::from_directory_path(path).unwrap();
 
     let store = Arc::new(LocalFileSystem::new());
-    let prefix = Path::from(url.path());
+    let prefix = Path::from_url_path(url.path()).unwrap();
     let storage = ObjectStoreStorageHandler::new(store, prefix);
     let cp = read_last_checkpoint(&storage, &url).await.unwrap().unwrap();
     assert_eq!(cp.version, 2);

ffi/examples/read-table/arrow.c

@@ -122,7 +122,7 @@ static ExclusiveEngineData* apply_transform(
     context->read_schema, // input schema
     context->arrow_context->cur_transform,
     context->logical_schema); // output schema
-  ExternResultHandleExclusiveEngineData transformed_res = evaluate(
+  ExternResultHandleExclusiveEngineData transformed_res = evaluate_expression(
     context->engine,
     &data,
     evaluator);

ffi/examples/visit-expression/expression.h

@@ -336,7 +336,7 @@ uintptr_t make_field_list(void* data, uintptr_t reserve) {
   return id;
 }
 
-ExpressionItemList construct_predicate(SharedExpression* predicate) {
+ExpressionItemList construct_expression(SharedExpression* expression) {
   ExpressionBuilder data = { 0 };
   EngineExpressionVisitor visitor = {
     .data = &data,
@@ -378,7 +378,54 @@ ExpressionItemList construct_predicate(SharedExpression* predicate) {
     .visit_column = visit_expr_column,
     .visit_struct_expr = visit_expr_struct_expr,
   };
-  uintptr_t top_level_id = visit_expression(&predicate, &visitor);
+  uintptr_t top_level_id = visit_expression(&expression, &visitor);
+  ExpressionItemList top_level_expr = data.lists[top_level_id];
+  free(data.lists);
+  return top_level_expr;
+}
+
+ExpressionItemList construct_predicate(SharedPredicate* predicate) {
+  ExpressionBuilder data = { 0 };
+  EngineExpressionVisitor visitor = {
+    .data = &data,
+    .make_field_list = make_field_list,
+    .visit_literal_int = visit_expr_int_literal,
+    .visit_literal_long = visit_expr_long_literal,
+    .visit_literal_short = visit_expr_short_literal,
+    .visit_literal_byte = visit_expr_byte_literal,
+    .visit_literal_float = visit_expr_float_literal,
+    .visit_literal_double = visit_expr_double_literal,
+    .visit_literal_bool = visit_expr_boolean_literal,
+    .visit_literal_timestamp = visit_expr_timestamp_literal,
+    .visit_literal_timestamp_ntz = visit_expr_timestamp_ntz_literal,
+    .visit_literal_date = visit_expr_date_literal,
+    .visit_literal_binary = visit_expr_binary_literal,
+    .visit_literal_null = visit_expr_null_literal,
+    .visit_literal_decimal = visit_expr_decimal_literal,
+    .visit_literal_string = visit_expr_string_literal,
+    .visit_literal_struct = visit_expr_struct_literal,
+    .visit_literal_array = visit_expr_array_literal,
+    .visit_and = visit_expr_and,
+    .visit_or = visit_expr_or,
+    .visit_not = visit_expr_not,
+    .visit_is_null = visit_expr_is_null,
+    .visit_lt = visit_expr_lt,
+    .visit_le = visit_expr_le,
+    .visit_gt = visit_expr_gt,
+    .visit_ge = visit_expr_ge,
+    .visit_eq = visit_expr_eq,
+    .visit_ne = visit_expr_ne,
+    .visit_distinct = visit_expr_distinct,
+    .visit_in = visit_expr_in,
+    .visit_not_in = visit_expr_not_in,
+    .visit_add = visit_expr_add,
+    .visit_minus = visit_expr_minus,
+    .visit_multiply = visit_expr_multiply,
+    .visit_divide = visit_expr_divide,
+    .visit_column = visit_expr_column,
+    .visit_struct_expr = visit_expr_struct_expr,
+  };
+  uintptr_t top_level_id = visit_predicate(&predicate, &visitor);
   ExpressionItemList top_level_expr = data.lists[top_level_id];
   free(data.lists);
   return top_level_expr;

ffi/examples/visit-expression/visit_expression.c

@@ -3,10 +3,16 @@
 #include "expression_print.h"
 
 int main() {
-  SharedExpression* pred = get_testing_kernel_expression();
-  ExpressionItemList expr = construct_predicate(pred);
-  print_expression(expr);
-  free_expression_list(expr);
+  SharedExpression* expr = get_testing_kernel_expression();
+  ExpressionItemList expr_list = construct_expression(expr);
+  print_expression(expr_list);
+  free_expression_list(expr_list);
+  free_kernel_expression(expr);
+
+  SharedPredicate* pred = get_testing_kernel_predicate();
+  ExpressionItemList pred_list = construct_predicate(pred);
+  print_expression(pred_list);
+  free_expression_list(pred_list);
   free_kernel_predicate(pred);
   return 0;
 }

ffi/src/engine_funcs.rs

@@ -185,7 +185,7 @@ fn new_expression_evaluator_impl(
 /// Caller is responsible for passing a valid handle.
 #[no_mangle]
 pub unsafe extern "C" fn free_expression_evaluator(evaluator: Handle<SharedExpressionEvaluator>) {
-    debug!("engine released evaluator");
+    debug!("engine released expression evaluator");
     evaluator.drop_handle();
 }
 
@@ -194,19 +194,19 @@ pub unsafe extern "C" fn free_expression_evaluator(evaluator: Handle<SharedExpre
 /// # Safety
 /// Caller is responsible for calling with a valid `Engine`, `ExclusiveEngineData`, and `Evaluator`
 #[no_mangle]
-pub unsafe extern "C" fn evaluate(
+pub unsafe extern "C" fn evaluate_expression(
     engine: Handle<SharedExternEngine>,
     batch: &mut Handle<ExclusiveEngineData>,
     evaluator: Handle<SharedExpressionEvaluator>,
 ) -> ExternResult<Handle<ExclusiveEngineData>> {
     let engine = unsafe { engine.clone_as_arc() };
     let batch = unsafe { batch.as_mut() };
     let evaluator = unsafe { evaluator.clone_as_arc() };
-    let res = evaluate_impl(batch, evaluator.as_ref());
+    let res = evaluate_expression_impl(batch, evaluator.as_ref());
     res.into_extern_result(&engine.as_ref())
 }
 
-fn evaluate_impl(
+fn evaluate_expression_impl(
     batch: &dyn EngineData,
     evaluator: &dyn ExpressionEvaluator,
 ) -> DeltaResult<Handle<ExclusiveEngineData>> {
@@ -224,7 +224,7 @@ mod tests {
     use std::sync::Arc;
 
     #[test]
-    fn test_new_evaluator() {
+    fn test_new_expression_evaluator() {
         let engine = get_default_engine();
         let in_schema = Arc::new(StructType::new(vec![StructField::new(
             "a",

ffi/src/expressions/engine.rs

@@ -6,14 +6,19 @@ use crate::{
     AllocateErrorFn, EngineIterator, ExternResult, IntoExternResult, KernelStringSlice,
     ReferenceSet, TryFromStringSlice,
 };
-use delta_kernel::{
-    expressions::{BinaryOperator, ColumnName, Expression, UnaryOperator},
-    DeltaResult,
+use delta_kernel::expressions::{
+    BinaryExpressionOp, BinaryPredicateOp, ColumnName, Expression, Predicate, UnaryPredicateOp,
 };
+use delta_kernel::DeltaResult;
+
+pub(crate) enum ExpressionOrPredicate {
+    Expression(Expression),
+    Predicate(Predicate),
+}
 
 #[derive(Default)]
 pub struct KernelExpressionVisitorState {
-    inflight_ids: ReferenceSet<Expression>,
+    inflight_ids: ReferenceSet<ExpressionOrPredicate>,
 }
 
 /// A predicate that can be used to skip data when scanning.
@@ -36,95 +41,163 @@ pub struct EnginePredicate {
 }
 
 fn wrap_expression(state: &mut KernelExpressionVisitorState, expr: impl Into<Expression>) -> usize {
-    state.inflight_ids.insert(expr.into())
+    let expr = ExpressionOrPredicate::Expression(expr.into());
+    state.inflight_ids.insert(expr)
+}
+
+fn wrap_predicate(state: &mut KernelExpressionVisitorState, pred: impl Into<Predicate>) -> usize {
+    let pred = ExpressionOrPredicate::Predicate(pred.into());
+    state.inflight_ids.insert(pred)
 }
 
 pub(crate) fn unwrap_kernel_expression(
     state: &mut KernelExpressionVisitorState,
     exprid: usize,
 ) -> Option<Expression> {
-    state.inflight_ids.take(exprid)
+    match state.inflight_ids.take(exprid)? {
+        ExpressionOrPredicate::Expression(expr) => Some(expr),
+        ExpressionOrPredicate::Predicate(pred) => Some(Expression::predicate(pred)),
+    }
+}
+
+pub(crate) fn unwrap_kernel_predicate(
+    state: &mut KernelExpressionVisitorState,
+    predid: usize,
+) -> Option<Predicate> {
+    match state.inflight_ids.take(predid)? {
+        ExpressionOrPredicate::Expression(expr) => Some(Predicate::from_expr(expr)),
+        ExpressionOrPredicate::Predicate(pred) => Some(pred),
+    }
 }
 
 fn visit_expression_binary(
     state: &mut KernelExpressionVisitorState,
-    op: BinaryOperator,
+    op: BinaryExpressionOp,
     a: usize,
     b: usize,
 ) -> usize {
-    let left = unwrap_kernel_expression(state, a);
-    let right = unwrap_kernel_expression(state, b);
-    match left.zip(right) {
-        Some((left, right)) => wrap_expression(state, Expression::binary(op, left, right)),
-        None => 0, // invalid child => invalid node
+    let a = unwrap_kernel_expression(state, a);
+    let b = unwrap_kernel_expression(state, b);
+    match (a, b) {
+        (Some(a), Some(b)) => wrap_expression(state, Expression::binary(op, a, b)),
+        _ => 0, // invalid child => invalid node
     }
 }
 
-fn visit_expression_unary(
+fn visit_predicate_binary(
     state: &mut KernelExpressionVisitorState,
-    op: UnaryOperator,
+    op: BinaryPredicateOp,
+    a: usize,
+    b: usize,
+) -> usize {
+    let a = unwrap_kernel_expression(state, a);
+    let b = unwrap_kernel_expression(state, b);
+    match (a, b) {
+        (Some(a), Some(b)) => wrap_predicate(state, Predicate::binary(op, a, b)),
+        _ => 0, // invalid child => invalid node
+    }
+}
+
+fn visit_predicate_unary(
+    state: &mut KernelExpressionVisitorState,
+    op: UnaryPredicateOp,
     inner_expr: usize,
 ) -> usize {
-    unwrap_kernel_expression(state, inner_expr).map_or(0, |expr| {
-        wrap_expression(state, Expression::unary(op, expr))
-    })
+    unwrap_kernel_expression(state, inner_expr)
+        .map_or(0, |expr| wrap_predicate(state, Predicate::unary(op, expr)))
 }
 
 // The EngineIterator is not thread safe, not reentrant, not owned by callee, not freed by callee.
 #[no_mangle]
-pub extern "C" fn visit_expression_and(
+pub extern "C" fn visit_predicate_and(
     state: &mut KernelExpressionVisitorState,
     children: &mut EngineIterator,
 ) -> usize {
-    let result = Expression::and_from(
-        children.flat_map(|child| unwrap_kernel_expression(state, child as usize)),
+    let result = Predicate::and_from(
+        children.flat_map(|child| unwrap_kernel_predicate(state, child as usize)),
     );
-    wrap_expression(state, result)
+    wrap_predicate(state, result)
+}
+
+#[no_mangle]
+pub extern "C" fn visit_expression_plus(
+    state: &mut KernelExpressionVisitorState,
+    a: usize,
+    b: usize,
+) -> usize {
+    visit_expression_binary(state, BinaryExpressionOp::Plus, a, b)
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_lt(
+pub extern "C" fn visit_expression_minus(
     state: &mut KernelExpressionVisitorState,
     a: usize,
     b: usize,
 ) -> usize {
-    visit_expression_binary(state, BinaryOperator::LessThan, a, b)
+    visit_expression_binary(state, BinaryExpressionOp::Minus, a, b)
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_le(
+pub extern "C" fn visit_expression_multiply(
     state: &mut KernelExpressionVisitorState,
     a: usize,
     b: usize,
 ) -> usize {
-    visit_expression_binary(state, BinaryOperator::LessThanOrEqual, a, b)
+    visit_expression_binary(state, BinaryExpressionOp::Multiply, a, b)
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_gt(
+pub extern "C" fn visit_expression_divide(
     state: &mut KernelExpressionVisitorState,
     a: usize,
     b: usize,
 ) -> usize {
-    visit_expression_binary(state, BinaryOperator::GreaterThan, a, b)
+    visit_expression_binary(state, BinaryExpressionOp::Divide, a, b)
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_ge(
+pub extern "C" fn visit_predicate_lt(
     state: &mut KernelExpressionVisitorState,
     a: usize,
     b: usize,
 ) -> usize {
-    visit_expression_binary(state, BinaryOperator::GreaterThanOrEqual, a, b)
+    visit_predicate_binary(state, BinaryPredicateOp::LessThan, a, b)
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_eq(
+pub extern "C" fn visit_predicate_le(
     state: &mut KernelExpressionVisitorState,
     a: usize,
     b: usize,
 ) -> usize {
-    visit_expression_binary(state, BinaryOperator::Equal, a, b)
+    visit_predicate_binary(state, BinaryPredicateOp::LessThanOrEqual, a, b)
+}
+
+#[no_mangle]
+pub extern "C" fn visit_predicate_gt(
+    state: &mut KernelExpressionVisitorState,
+    a: usize,
+    b: usize,
+) -> usize {
+    visit_predicate_binary(state, BinaryPredicateOp::GreaterThan, a, b)
+}
+
+#[no_mangle]
+pub extern "C" fn visit_predicate_ge(
+    state: &mut KernelExpressionVisitorState,
+    a: usize,
+    b: usize,
+) -> usize {
+    visit_predicate_binary(state, BinaryPredicateOp::GreaterThanOrEqual, a, b)
+}
+
+#[no_mangle]
+pub extern "C" fn visit_predicate_eq(
+    state: &mut KernelExpressionVisitorState,
+    a: usize,
+    b: usize,
+) -> usize {
+    visit_predicate_binary(state, BinaryPredicateOp::Equal, a, b)
 }
 
 /// # Safety
@@ -148,19 +221,20 @@ fn visit_expression_column_impl(
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_not(
+pub extern "C" fn visit_predicate_not(
     state: &mut KernelExpressionVisitorState,
-    inner_expr: usize,
+    inner_pred: usize,
 ) -> usize {
-    visit_expression_unary(state, UnaryOperator::Not, inner_expr)
+    unwrap_kernel_predicate(state, inner_pred)
+        .map_or(0, |pred| wrap_predicate(state, Predicate::not(pred)))
 }
 
 #[no_mangle]
-pub extern "C" fn visit_expression_is_null(
+pub extern "C" fn visit_predicate_is_null(
     state: &mut KernelExpressionVisitorState,
     inner_expr: usize,
 ) -> usize {
-    visit_expression_unary(state, UnaryOperator::IsNull, inner_expr)
+    visit_predicate_unary(state, UnaryPredicateOp::IsNull, inner_expr)
 }
 
 /// # Safety