Add some progress logs

digitalmethodsinitiative · Oct 23, 2024 · 25f9ffd · 25f9ffd
1 parent aad7d57
commit 25f9ffd
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/processors/filtering/unique_images.py b/processors/filtering/unique_images.py
@@ -104,11 +104,15 @@ def process(self):
         processed = 0
         staging_area = self.dataset.get_staging_area()
 
+        self.dataset.update_progress("Processing images and looking for duplicates")
         for image_file in self.iterate_archive_contents(self.source_file):
             if self.interrupted:
                 raise ProcessorInterruptedException("Interrupted while filtering for unique images")
 
             self.dataset.update_progress(processed / self.source_dataset.num_rows)
+            if processed % 100 == 0:
+                self.dataset.update_progress(f"Processed {processed:,} of {self.source_dataset.num_rows:,} images, "
+                                             f"found {dupes:,} duplicate(s)")
             processed += 1
 
             if image_file.name == ".metadata.json":