@@ -26,40 +26,40 @@ class TokenizerSpec(ABC):
2626
2727 @abstractmethod
2828 def text_to_tokens (self , text ):
29- """ Converts text into a list of tokens. """
29+ """Converts text into a list of tokens."""
3030 pass
3131
3232 @abstractmethod
3333 def tokens_to_text (self , tokens ):
34- """ Converts a list of tokens back into text. """
34+ """Converts a list of tokens back into text."""
3535 pass
3636
3737 @abstractmethod
3838 def tokens_to_ids (self , tokens ):
39- """ Converts a list of tokens to their corresponding IDs. """
39+ """Converts a list of tokens to their corresponding IDs."""
4040 pass
4141
4242 @abstractmethod
4343 def ids_to_tokens (self , ids ):
44- """ Converts a list of token IDs back to tokens. """
44+ """Converts a list of token IDs back to tokens."""
4545 pass
4646
4747 @abstractmethod
4848 def text_to_ids (self , text ):
49- """ Converts text directly to token IDs. """
49+ """Converts text directly to token IDs."""
5050 pass
5151
5252 @abstractmethod
5353 def ids_to_text (self , ids ):
54- """ Converts token IDs back to text. """
54+ """Converts token IDs back to text."""
5555 pass
5656
5757 def add_special_tokens (self , special_tokens : List [str ]):
58- """ Adds special tokens (eos, pad, cls...) to vocab. """
58+ """Adds special tokens (eos, pad, cls...) to vocab."""
5959 raise NotImplementedError ("To be implemented" )
6060
6161 def apply_chat_template (self , * args , ** kwargs ):
62- """ Appies chat template and tokenizes results """
62+ """Appies chat template and tokenizes results"""
6363 raise NotImplementedError ("To be implemented" )
6464
6565 @property
0 commit comments