Skip to content

Commit 7fbd20c

Browse files
committed
Apply isort and black reformatting
Signed-off-by: akoumpa <[email protected]>
1 parent c601e4e commit 7fbd20c

File tree

2 files changed

+9
-9
lines changed

2 files changed

+9
-9
lines changed

nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ def text_to_ids(self, text):
313313
return ids
314314

315315
def apply_chat_template(self, *args, **kwargs):
316-
""" Appies chat template and tokenizes results """
316+
"""Appies chat template and tokenizes results"""
317317
return self.tokenizer.apply_chat_template(*args, **kwargs)
318318

319319
def ids_to_text(self, ids, remove_special_tokens=True):

nemo/collections/common/tokenizers/tokenizer_spec.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,40 +26,40 @@ class TokenizerSpec(ABC):
2626

2727
@abstractmethod
2828
def text_to_tokens(self, text):
29-
""" Converts text into a list of tokens. """
29+
"""Converts text into a list of tokens."""
3030
pass
3131

3232
@abstractmethod
3333
def tokens_to_text(self, tokens):
34-
""" Converts a list of tokens back into text. """
34+
"""Converts a list of tokens back into text."""
3535
pass
3636

3737
@abstractmethod
3838
def tokens_to_ids(self, tokens):
39-
""" Converts a list of tokens to their corresponding IDs. """
39+
"""Converts a list of tokens to their corresponding IDs."""
4040
pass
4141

4242
@abstractmethod
4343
def ids_to_tokens(self, ids):
44-
""" Converts a list of token IDs back to tokens. """
44+
"""Converts a list of token IDs back to tokens."""
4545
pass
4646

4747
@abstractmethod
4848
def text_to_ids(self, text):
49-
""" Converts text directly to token IDs. """
49+
"""Converts text directly to token IDs."""
5050
pass
5151

5252
@abstractmethod
5353
def ids_to_text(self, ids):
54-
""" Converts token IDs back to text. """
54+
"""Converts token IDs back to text."""
5555
pass
5656

5757
def add_special_tokens(self, special_tokens: List[str]):
58-
""" Adds special tokens (eos, pad, cls...) to vocab. """
58+
"""Adds special tokens (eos, pad, cls...) to vocab."""
5959
raise NotImplementedError("To be implemented")
6060

6161
def apply_chat_template(self, *args, **kwargs):
62-
""" Appies chat template and tokenizes results """
62+
"""Appies chat template and tokenizes results"""
6363
raise NotImplementedError("To be implemented")
6464

6565
@property

0 commit comments

Comments
 (0)