Hack on it until the tests compile

Author: alamb

kernel/src/arrow.rs

@@ -1,14 +1,10 @@
 //! This module exists to help re-export the version of arrow used by default-engine and other
 //! parts of kernel that need arrow
 
-
 #[cfg(all(feature = "arrow_55"))]
 pub use arrow_55::*;
 
 // if nothing is enabled but we need arrow because of some other feature flag, default to lowest
 // supported version
-#[cfg(all(
-    feature = "need_arrow",
-    not(feature = "arrow_55")
-))]
+#[cfg(all(feature = "need_arrow", not(feature = "arrow_55")))]
 compile_error!("Requested a feature that needs arrow without enabling arrow. Please enable the `arrow_53` or `arrow_54` feature");

kernel/src/engine/default/filesystem.rs

@@ -79,7 +79,7 @@ impl<E: TaskExecutor> StorageHandler for ObjectStoreStorageHandler<E> {
                             .send(Ok(FileMeta {
                                 location,
                                 last_modified: meta.last_modified.timestamp_millis(),
-                                size: meta.size,
+                                size: meta.size as usize,
                             }))
                             .ok();
                     }
@@ -136,6 +136,7 @@ impl<E: TaskExecutor> StorageHandler for ObjectStoreStorageHandler<E> {
                             // have to annotate type here or rustc can't figure it out
                             Ok::<bytes::Bytes, Error>(reqwest::get(url).await?.bytes().await?)
                         } else if let Some(rng) = range {
+                            let rng = rng.start as u64..rng.end as u64;
                             Ok(store.get_range(&path, rng).await?)
                         } else {
                             let result = store.get(&path).await?;

kernel/src/engine/default/json.rs

@@ -447,7 +447,7 @@ mod tests {
             &self,
             prefix: Option<&Path>,
             offset: &Path,
-        ) -> BoxStream<'_, Result<ObjectMeta>> {
+        ) -> BoxStream<'static, Result<ObjectMeta>> {
             self.inner.list_with_offset(prefix, offset)
         }
 
@@ -532,7 +532,7 @@ mod tests {
         let files = &[FileMeta {
             location: url.clone(),
             last_modified: meta.last_modified.timestamp_millis(),
-            size: meta.size,
+            size: meta.size as usize,
         }];
 
         let handler = DefaultJsonHandler::new(store, Arc::new(TokioBackgroundExecutor::new()));
@@ -681,7 +681,7 @@ mod tests {
                         FileMeta {
                             location: url,
                             last_modified: meta.last_modified.timestamp_millis(),
-                            size: meta.size,
+                            size: meta.size as usize,
                         }
                     }
                 })

kernel/src/engine/default/parquet.rs

@@ -257,8 +257,8 @@ impl FileOpener for ParquetOpener {
         Ok(Box::pin(async move {
             // TODO avoid IO by converting passed file meta to ObjectMeta
             let meta = store.head(&path).await?;
-            let mut reader = ParquetObjectReader::new(store, meta.location)
-                .with_file_size(meta.size);
+            let mut reader =
+                ParquetObjectReader::new(store, meta.location).with_file_size(meta.size);
             let metadata = ArrowReaderMetadata::load_async(&mut reader, Default::default()).await?;
             let parquet_schema = metadata.schema();
             let (indices, requested_ordering) =
@@ -395,7 +395,8 @@ mod tests {
         let location = Path::from(url.path());
         let meta = store.head(&location).await.unwrap();
 
-        let reader = ParquetObjectReader::new(store.clone(), meta.clone());
+        let reader =
+            ParquetObjectReader::new(store.clone(), meta.location).with_file_size(meta.size);
         let physical_schema = ParquetRecordBatchStreamBuilder::new(reader)
             .await
             .unwrap()
@@ -405,7 +406,7 @@ mod tests {
         let files = &[FileMeta {
             location: url.clone(),
             last_modified: meta.last_modified.timestamp(),
-            size: meta.size,
+            size: meta.size as usize,
         }];
 
         let handler = DefaultParquetHandler::new(store, Arc::new(TokioBackgroundExecutor::new()));
@@ -513,13 +514,14 @@ mod tests {
 
         let filename = location.path().split('/').next_back().unwrap();
         assert_eq!(&expected_location.join(filename).unwrap(), location);
-        assert_eq!(expected_size, size);
+        assert_eq!(expected_size, size as u64);
         assert!(now - last_modified < 10_000);
 
         // check we can read back
         let path = Path::from(location.path());
         let meta = store.head(&path).await.unwrap();
-        let reader = ParquetObjectReader::new(store.clone(), meta.clone());
+        let reader =
+            ParquetObjectReader::new(store.clone(), meta.location).with_file_size(meta.size);
         let physical_schema = ParquetRecordBatchStreamBuilder::new(reader)
             .await
             .unwrap()

kernel/src/engine/default/storage.rs

@@ -43,7 +43,3 @@ where
      */
     todo!("Need to update hdfs object store");
 }
-
-
-
-

kernel/src/parquet.rs

@@ -6,8 +6,5 @@ pub use parquet_55::*;
 
 // if nothing is enabled but we need arrow because of some other feature flag, default to lowest
 // supported version
-#[cfg(all(
-    feature = "need_arrow",
-    not(feature = "arrow_55"),
-))]
+#[cfg(all(feature = "need_arrow", not(feature = "arrow_55"),))]
 compile_error!("Requested a feature that needs arrow without enabling arrow. Please enable the `arrow_53` or `arrow_54` feature");