File tree Expand file tree Collapse file tree 1 file changed +4
-2
lines changed Expand file tree Collapse file tree 1 file changed +4
-2
lines changed Original file line number Diff line number Diff line change @@ -62,7 +62,7 @@ def __init__(
6262 # however the requests can be sent as a string if the API doesn't support token inputs.
6363 # use tokenized_requests=False
6464 tokenizer_backend : Optional [
65- Literal ["tiktoken" , "huggingface" , None ]
65+ Literal ["tiktoken" , "huggingface" , " None" , "none" ]
6666 ] = "huggingface" ,
6767 truncate : bool = False ,
6868 # number of concurrent requests. More useful if not batching
@@ -116,7 +116,9 @@ def __init__(
116116 "Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1."
117117 )
118118 self ._concurrent = int (num_concurrent )
119- self .tokenizer_backend = tokenizer_backend
119+ self .tokenizer_backend = (
120+ None if tokenizer_backend in ("None" , "none" ) else tokenizer_backend
121+ )
120122 self .add_bos_token = add_bos_token
121123 self .custom_prefix_token_id = custom_prefix_token_id
122124 self .tokenized_requests = tokenized_requests
You can’t perform that action at this time.
0 commit comments