Hack on it until the tests compile

Author: alamb

Diff for: kernel/src/arrow.rs

@@ -1,14 +1,10 @@
 //! This module exists to help re-export the version of arrow used by default-engine and other
 //! parts of kernel that need arrow
 
-
 #[cfg(all(feature = "arrow_55"))]
 pub use arrow_55::*;
 
 // if nothing is enabled but we need arrow because of some other feature flag, default to lowest
 // supported version
-#[cfg(all(
-    feature = "need_arrow",
-    not(feature = "arrow_55")
-))]
+#[cfg(all(feature = "need_arrow", not(feature = "arrow_55")))]
 compile_error!("Requested a feature that needs arrow without enabling arrow. Please enable the `arrow_53` or `arrow_54` feature");

Diff for: kernel/src/engine/default/filesystem.rs

@@ -69,7 +69,7 @@ impl<E: TaskExecutor> FileSystemClient for ObjectStoreFileSystemClient<E> {
                             .send(Ok(FileMeta {
                                 location,
                                 last_modified: meta.last_modified.timestamp_millis(),
-                                size: meta.size,
+                                size: meta.size as usize,
                             }))
                             .ok();
                     }
@@ -129,6 +129,8 @@ impl<E: TaskExecutor> FileSystemClient for ObjectStoreFileSystemClient<E> {
                             }
                             _ => {
                                 if let Some(rng) = range {
+                                    // convert to u64 for new API
+                                    let rng = rng.start as u64..rng.end as u64;
                                     Ok(store.get_range(&path, rng).await?)
                                 } else {
                                     let result = store.get(&path).await?;

Diff for: kernel/src/engine/default/json.rs

@@ -446,7 +446,7 @@ mod tests {
             &self,
             prefix: Option<&Path>,
             offset: &Path,
-        ) -> BoxStream<'_, Result<ObjectMeta>> {
+        ) -> BoxStream<'static, Result<ObjectMeta>> {
             self.inner.list_with_offset(prefix, offset)
         }
 
@@ -539,7 +539,7 @@ mod tests {
         let files = &[FileMeta {
             location: url.clone(),
             last_modified: meta.last_modified.timestamp_millis(),
-            size: meta.size,
+            size: meta.size as usize,
         }];
 
         let handler = DefaultJsonHandler::new(store, Arc::new(TokioBackgroundExecutor::new()));
@@ -688,7 +688,7 @@ mod tests {
                         FileMeta {
                             location: url,
                             last_modified: meta.last_modified.timestamp_millis(),
-                            size: meta.size,
+                            size: meta.size as usize,
                         }
                     }
                 })

Diff for: kernel/src/engine/default/parquet.rs

@@ -255,8 +255,8 @@ impl FileOpener for ParquetOpener {
         Ok(Box::pin(async move {
             // TODO avoid IO by converting passed file meta to ObjectMeta
             let meta = store.head(&path).await?;
-            let mut reader = ParquetObjectReader::new(store, meta.location)
-                .with_file_size(meta.size);
+            let mut reader =
+                ParquetObjectReader::new(store, meta.location).with_file_size(meta.size);
             let metadata = ArrowReaderMetadata::load_async(&mut reader, Default::default()).await?;
             let parquet_schema = metadata.schema();
             let (indices, requested_ordering) =
@@ -393,7 +393,8 @@ mod tests {
         let location = Path::from(url.path());
         let meta = store.head(&location).await.unwrap();
 
-        let reader = ParquetObjectReader::new(store.clone(), meta.clone());
+        let reader =
+            ParquetObjectReader::new(store.clone(), meta.location).with_file_size(meta.size);
         let physical_schema = ParquetRecordBatchStreamBuilder::new(reader)
             .await
             .unwrap()
@@ -403,7 +404,7 @@ mod tests {
         let files = &[FileMeta {
             location: url.clone(),
             last_modified: meta.last_modified.timestamp(),
-            size: meta.size,
+            size: meta.size as usize,
         }];
 
         let handler = DefaultParquetHandler::new(store, Arc::new(TokioBackgroundExecutor::new()));
@@ -511,13 +512,14 @@ mod tests {
 
         let filename = location.path().split('/').last().unwrap();
         assert_eq!(&expected_location.join(filename).unwrap(), location);
-        assert_eq!(expected_size, size);
+        assert_eq!(expected_size, size as u64);
         assert!(now - last_modified < 10_000);
 
         // check we can read back
         let path = Path::from(location.path());
         let meta = store.head(&path).await.unwrap();
-        let reader = ParquetObjectReader::new(store.clone(), meta.clone());
+        let reader =
+            ParquetObjectReader::new(store.clone(), meta.location).with_file_size(meta.size);
         let physical_schema = ParquetRecordBatchStreamBuilder::new(reader)
             .await
             .unwrap()

Diff for: kernel/src/engine/default/storage.rs

@@ -43,7 +43,3 @@ where
      */
     todo!("Need to update hdfs object store");
 }
-
-
-
-

Diff for: kernel/src/parquet.rs

@@ -6,8 +6,5 @@ pub use parquet_55::*;
 
 // if nothing is enabled but we need arrow because of some other feature flag, default to lowest
 // supported version
-#[cfg(all(
-    feature = "need_arrow",
-    not(feature = "arrow_55"),
-))]
+#[cfg(all(feature = "need_arrow", not(feature = "arrow_55"),))]
 compile_error!("Requested a feature that needs arrow without enabling arrow. Please enable the `arrow_53` or `arrow_54` feature");