Reduce uploaders (#290)

Lightning-AI · Aug 1, 2024 · b2974f8 · b2974f8
1 parent b542461
commit b2974f8
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 2 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -3,3 +3,4 @@ filelock
 numpy
 boto3
 requests
+s5cmd
diff --git a/src/litdata/__about__.py b/src/litdata/__about__.py
@@ -14,7 +14,7 @@
 
 import time
 
-__version__ = "0.2.19"
+__version__ = "0.2.20"
 __author__ = "Lightning AI et al."
 __author_email__ = "[email protected]"
 __license__ = "Apache-2.0"

diff --git a/src/litdata/processing/data_processor.py b/src/litdata/processing/data_processor.py
@@ -910,7 +910,7 @@ def __init__(
 
         self.num_workers = num_workers or (1 if fast_dev_run else (os.cpu_count() or 1) * 4)
         self.num_downloaders = num_downloaders or 2
-        self.num_uploaders = num_uploaders or 5
+        self.num_uploaders = num_uploaders or 1
         self.delete_cached_files = delete_cached_files
         self.fast_dev_run = _get_fast_dev_run() if fast_dev_run is None else fast_dev_run
         self.workers: Any = []
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,4 @@ filelock @@
     numpy
     boto3
     requests
+    s5cmd