fix: list log files with checkpoint when version is None (#312)

Previously, if read_last_checkpoint returned a valid _last_checkpoint hint but the version requested was None (i.e. requesting the lastest version) we would still list_log_files without listing from the checkpoint hint. This change will detect when we have a valid _last_checkpoint and request the latest version and instead list_log_files_with_checkpoint --------- Co-authored-by: Mervyn Zhang <[email protected]> Co-authored-by: Zach Schuermann <[email protected]>
delta-io · Aug 14, 2024 · e4c07dd · e4c07dd
1 parent b2bb39a
commit e4c07dd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/kernel/src/snapshot.rs b/kernel/src/snapshot.rs
@@ -149,6 +149,9 @@ impl Snapshot {
         // List relevant files from log
         let (mut commit_files, checkpoint_files) =
             match (read_last_checkpoint(fs_client.as_ref(), &log_url)?, version) {
+                (Some(cp), None) => {
+                    list_log_files_with_checkpoint(&cp, fs_client.as_ref(), &log_url)?
+                }
                 (Some(cp), Some(version)) if cp.version >= version => {
                     list_log_files_with_checkpoint(&cp, fs_client.as_ref(), &log_url)?
                 }

diff --git a/kernel/tests/golden_tables.rs b/kernel/tests/golden_tables.rs
@@ -408,8 +408,8 @@ golden_test!("time-travel-start", latest_snapshot_test);
 golden_test!("time-travel-start-start20", latest_snapshot_test);
 golden_test!("time-travel-start-start20-start40", latest_snapshot_test);
 
-golden_test!("v2-checkpoint-json", latest_snapshot_test); // passing without v2 checkpoint support
-golden_test!("v2-checkpoint-parquet", latest_snapshot_test); // passing without v2 checkpoint support
+skip_test!("v2-checkpoint-json": "v2 checkpoint not supported");
+skip_test!("v2-checkpoint-parquet": "v2 checkpoint not supported");
 
 // BUG:
 // - AddFile: 'file:/some/unqualified/absolute/path'