fix: Nonetypes causing error in preprocessing

Author: dannyjameswilliams

elysia/preprocessing/collection.py

@@ -188,7 +188,11 @@ async def _evaluate_field_statistics(
         # For text, we want to evaluate the length of the text in tokens (use spacy)
         lengths = []
         for obj in sample_objects:
-            if property in obj and isinstance(obj[property], str):
+            if (
+                property in obj
+                and isinstance(obj[property], str)
+                and obj[property] is not None
+            ):
                 lengths.append(len(nlp(obj[property])))
 
         if len(lengths) == 0:
@@ -228,7 +232,11 @@ async def _evaluate_field_statistics(
 
     # List (lengths)
     elif properties[property].endswith("[]"):
-        lengths = [len(obj[property]) for obj in sample_objects]
+        lengths = [
+            len(obj[property])
+            for obj in sample_objects
+            if isinstance(obj[property], list)
+        ]
 
         if len(lengths) == 0:
             out["range"] = None

elysia/util/client.py

@@ -101,8 +101,6 @@ def __init__(
         ```
         """
 
-        self.logger = logger
-
         if client_timeout is None:
             self.client_timeout = datetime.timedelta(
                 minutes=int(os.getenv("CLIENT_TIMEOUT", 3))
@@ -117,6 +115,11 @@ def __init__(
         else:
             self.settings = settings
 
+        if logger is None:
+            self.logger = self.settings.logger
+        else:
+            self.logger = logger
+
         # Set the weaviate url and api key
         if wcd_url is None:
             self.wcd_url = self.settings.WCD_URL
@@ -217,7 +220,16 @@ def __init__(
             return
 
         # Start sync client
-        self.client = self.get_client()
+        try:
+            self.client = self.get_client()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    "Error initialising Weaviate client. Please check your Weaviate configuration is set correctly (WCD_URL, WCD_API_KEY, WEAVIATE_IS_LOCAL, LOCAL_WEAVIATE_PORT, LOCAL_WEAVIATE_GRPC_PORT)."
+                )
+                self.logger.error(f"Full Weaviate connection error message: {e}")
+            self.is_client = False
+            return
         self.sync_restart_event.set()
 
     def _get_local_host_and_port(self) -> tuple[str, int]: