fix: increase csv field_size_limit to accommodate large fields

openfoodfacts · Oct 8, 2024 · 94be4d3 · 94be4d3
1 parent 6faadc1
commit 94be4d3
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/openfoodfacts/dataset.py b/openfoodfacts/dataset.py
@@ -14,6 +14,10 @@
 
 logger = get_logger(__name__)
 
+# Increase field_size to accommodate large fields.
+# sys.maxsize will overflow on windows so using max 32-bit integer instead.
+csv.field_size_limit(pow(2, 31) - 1)
+
 
 DEFAULT_CACHE_DIR = Path("~/.cache/openfoodfacts/datasets").expanduser()
 DATASET_FILE_NAMES = {