Fix

Author: 16pierre

datafusion/datasource-parquet/src/opener.rs

@@ -284,6 +284,7 @@ impl FileOpener for ParquetOpener {
 
         let reverse_row_groups = self.reverse_row_groups;
         let preserve_order = self.preserve_order;
+        let pruning_predicate_config = PruningPredicateConfig { max_in_list: self.pruning_max_inlist_limit };
 
         Ok(Box::pin(async move {
             #[cfg(feature = "parquet_encryption")]
@@ -330,6 +331,7 @@ impl FileOpener for ParquetOpener {
                         &logical_file_schema,
                         &partitioned_file,
                         predicate_creation_errors.clone(),
+                        pruning_predicate_config.clone(),
                     )
                 });
 
@@ -426,14 +428,11 @@ impl FileOpener for ParquetOpener {
                 .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;
 
             // Build predicates for this specific file
-            let pruning_config = PruningPredicateConfig {
-                max_in_list: self.pruning_max_inlist_limit,
-            };
             let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(
                 predicate.as_ref(),
                 &physical_file_schema,
                 &predicate_creation_errors,
-                &pruning_config,
+                &pruning_predicate_config,
             );
 
             // The page index is not stored inline in the parquet footer so the
@@ -971,6 +970,7 @@ pub(crate) fn build_pruning_predicates(
         Arc::clone(predicate),
         file_schema,
         predicate_creation_errors,
+        config,
     );
     let page_pruning_predicate =
         build_page_pruning_predicate(predicate, file_schema, config);

datafusion/datasource-parquet/src/source.rs

@@ -641,7 +641,7 @@ impl FileSource for ParquetSource {
                 // Instead we use the logical schema of the file (the table schema without partition columns).
                 if let Some(predicate) = &self.predicate {
                     let predicate_creation_errors = Count::new();
-                    let pruning_config = PruningPredicateConfig {
+                    let pruning_predicate_config = PruningPredicateConfig {
                         max_in_list: self
                             .table_parquet_options
                             .global
@@ -651,7 +651,7 @@ impl FileSource for ParquetSource {
                         Some(predicate),
                         self.table_schema.table_schema(),
                         &predicate_creation_errors,
-                        &pruning_config,
+                        &pruning_predicate_config,
                     ) {
                         let mut guarantees = pruning_predicate
                             .literal_guarantees()

datafusion/pruning/src/file_pruner.rs

@@ -26,7 +26,7 @@ use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, snapshot_gene
 use datafusion_physical_plan::metrics::Count;
 use log::debug;
 
-use crate::build_pruning_predicate;
+use crate::{build_pruning_predicate, PruningPredicateConfig};
 
 /// Prune based on file-level statistics.
 ///
@@ -40,6 +40,7 @@ pub struct FilePruner {
     file_schema: SchemaRef,
     file_stats_pruning: PrunableStatistics,
     predicate_creation_errors: Count,
+    pruning_predicate_config: PruningPredicateConfig
 }
 
 impl FilePruner {
@@ -54,12 +55,14 @@ impl FilePruner {
         _partition_fields: Vec<FieldRef>,
         partitioned_file: PartitionedFile,
         predicate_creation_errors: Count,
+        pruning_predicate_config: PruningPredicateConfig,
     ) -> Result<Self> {
         Self::try_new(
             predicate,
             logical_file_schema,
             &partitioned_file,
             predicate_creation_errors,
+            pruning_predicate_config
         )
         .ok_or_else(|| {
             internal_datafusion_err!(
@@ -76,6 +79,7 @@ impl FilePruner {
         file_schema: &SchemaRef,
         partitioned_file: &PartitionedFile,
         predicate_creation_errors: Count,
+        pruning_predicate_config: PruningPredicateConfig,
     ) -> Option<Self> {
         let file_stats = partitioned_file.statistics.as_ref()?;
         let file_stats_pruning =
@@ -86,6 +90,7 @@ impl FilePruner {
             file_schema: Arc::clone(file_schema),
             file_stats_pruning,
             predicate_creation_errors,
+            pruning_predicate_config
         })
     }
 
@@ -110,6 +115,7 @@ impl FilePruner {
             Arc::clone(&self.predicate),
             &self.file_schema,
             &self.predicate_creation_errors,
+            &self.pruning_predicate_config,
         );
         let Some(pruning_predicate) = pruning_predicate else {
             return Ok(false);

datafusion/pruning/src/pruning_predicate.rs

@@ -399,11 +399,12 @@ pub fn build_pruning_predicate(
     predicate: Arc<dyn PhysicalExpr>,
     file_schema: &SchemaRef,
     predicate_creation_errors: &Count,
+    config: &PruningPredicateConfig,
 ) -> Option<Arc<PruningPredicate>> {
     match PruningPredicate::try_new(
         predicate,
         Arc::clone(file_schema),
-        &PruningPredicateConfig::default(),
+        config,
     ) {
         Ok(pruning_predicate) => {
             if !pruning_predicate.always_true() {