Hint Generator type

Author: LoganDark

tokenizer/rwkv_tokenizer.py

@@ -217,6 +217,7 @@ def printTokens(self, tokens):
 # Tokenizer #4 (fast) https://github.com/LoganDark
 ########################################################################################################
 
+from typing import Generator
 from ast import literal_eval
 
 class FastTokenizer:
@@ -255,7 +256,7 @@ def next_token(self, src: bytes) -> int:
                 break
         return last_token
 
-    def encode_bytes(self, src: bytes) -> list[int]:
+    def encode_bytes(self, src: bytes) -> Generator[int, None, None]:
         start, stop = 0, len(src)
         while start < stop:
             last_token, last = None, self.root
@@ -274,7 +275,7 @@ def encode_bytes(self, src: bytes) -> list[int]:
     def decode_bytes(self, tokens: list[int]) -> bytes:
         return b''.join(map(self.tok2val.__getitem__, tokens))
 
-    def encode(self, src: str) -> list[int]:
+    def encode(self, src: str) -> Generator[int, None, None]:
         return self.encode_bytes(src.encode('utf-8'))
 
     def decode(self, tokens: list[int]) -> str: