- added tokenizer fallback (required for gpt-4.1 at the moment)

Author: markhofm

src/hierarchical.py

@@ -160,7 +160,10 @@ def __init__(
                 try:
                     self.tokenizer = Tokenizer.from_pretrained(tokenizer)
                 except:
-                    raise ValueError("Invalid tokenizer or model name")
+                    logger.warning(
+                        f"Tokenizer {tokenizer} not found, using o200k_base tokenizer instead."
+                    )
+                    self.tokenizer = tiktoken.get_encoding('o200k_base')
         else:
             self.tokenizer = tokenizer
         self.chunker = chunker