Skip to content

Commit 8a32ebb

Browse files
committed
Address PR feedback.
1 parent 7adc770 commit 8a32ebb

File tree

1 file changed

+13
-5
lines changed

1 file changed

+13
-5
lines changed

crates/iceberg/src/arrow/caching_delete_file_loader.rs

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -224,14 +224,22 @@ impl CachingDeleteFileLoader {
224224
let (sender, receiver) = channel();
225225
del_filter.insert_equality_delete(&task.file_path, receiver);
226226

227-
// Equality deletes intentionally have partial schemas. Schema evolution would add
228-
// NULL values for missing REQUIRED columns, causing Arrow validation to fail.
229-
Ok(DeleteFileContext::FreshEqDel {
230-
batch_stream: basic_delete_file_loader
227+
// Per the Iceberg spec, evolve schema for equality deletes but only for the
228+
// equality_ids columns, not all table columns.
229+
let equality_ids_vec = task.equality_ids.clone().unwrap();
230+
let evolved_stream = BasicDeleteFileLoader::evolve_schema(
231+
basic_delete_file_loader
231232
.parquet_to_batch_stream(&task.file_path)
232233
.await?,
234+
schema,
235+
&equality_ids_vec,
236+
)
237+
.await?;
238+
239+
Ok(DeleteFileContext::FreshEqDel {
240+
batch_stream: evolved_stream,
233241
sender,
234-
equality_ids: HashSet::from_iter(task.equality_ids.clone().unwrap()),
242+
equality_ids: HashSet::from_iter(equality_ids_vec),
235243
})
236244
}
237245

0 commit comments

Comments
 (0)