[cli] ingest all queue fix (#20)

* fix ingest-all processing on gz, add test

* patch order of queue check

---------

Co-authored-by: Raffa Tempo <[email protected]>
Co-authored-by: Lucietta MacUrchin <[email protected]>
Co-authored-by: Caty Cony <[email protected]>

Author: 4 people

src/load.rs

@@ -44,29 +44,29 @@ pub async fn ingest_all(
 
     // This manifest may be for a .gz file, we should handle here as well
     for (_p, m) in archive_map.0.iter() {
-        info!("checking if we need to decompress");
-        let (new_unzip_path, temp) = unzip_temp::maybe_handle_gz(&m.archive_dir)?;
-        let mut better_man = ManifestInfo::new(&new_unzip_path);
-        better_man.set_info()?;
-
         println!(
             "\nProcessing: {:?} with archive: {}",
-            better_man.contents,
-            better_man.archive_dir.display()
+            m.contents,
+            m.archive_dir.display()
         );
 
-        let complete = queue::are_all_completed(pool, &better_man.archive_id).await?;
+        let complete = queue::are_all_completed(pool, &m.archive_id).await?;
 
         if !complete {
+            info!("checking if we need to decompress");
+            let (new_unzip_path, temp) = unzip_temp::maybe_handle_gz(&m.archive_dir)?;
+            let mut better_man = ManifestInfo::new(&new_unzip_path);
+            better_man.set_info()?;
+
             let batch_tx_return = try_load_one_archive(&better_man, pool, batch_size).await?;
             println!("SUCCESS: {}", batch_tx_return);
+            drop(temp);
         } else {
             info!(
                 "archive complete (or not in queue): {}",
-                better_man.archive_dir.display()
+                m.archive_dir.display()
             );
         }
-        drop(temp);
     }
 
     Ok(())

src/unzip_temp.rs

@@ -125,9 +125,8 @@ pub fn maybe_handle_gz(archive_path: &Path) -> Result<(PathBuf, Option<TempPath>
     // maybe stuff isn't unzipped yet
     let pattern = format!("{}/*.*.gz", archive_path.display());
     if glob(&pattern)?.count() > 0 {
-        let mut temp_dir = TempPath::new();
+        let temp_dir = TempPath::new();
         temp_dir.create_as_dir()?;
-        temp_dir.persist();
 
         // need to preserve the parent dir name in temp, since the manifest files reference it.
         let dir_name = archive_path.file_name().unwrap().to_str().unwrap();