Apply isort and black reformatting

Signed-off-by: akoumpa <[email protected]>

Author: akoumpa

nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py

@@ -313,7 +313,7 @@ def text_to_ids(self, text):
         return ids
 
     def apply_chat_template(self, *args, **kwargs):
-        """ Appies chat template and tokenizes results """
+        """Appies chat template and tokenizes results"""
         return self.tokenizer.apply_chat_template(*args, **kwargs)
 
     def ids_to_text(self, ids, remove_special_tokens=True):

nemo/collections/common/tokenizers/tokenizer_spec.py

@@ -26,40 +26,40 @@ class TokenizerSpec(ABC):
 
     @abstractmethod
     def text_to_tokens(self, text):
-        """  Converts text into a list of tokens. """
+        """Converts text into a list of tokens."""
         pass
 
     @abstractmethod
     def tokens_to_text(self, tokens):
-        """ Converts a list of tokens back into text. """
+        """Converts a list of tokens back into text."""
         pass
 
     @abstractmethod
     def tokens_to_ids(self, tokens):
-        """ Converts a list of tokens to their corresponding IDs. """
+        """Converts a list of tokens to their corresponding IDs."""
         pass
 
     @abstractmethod
     def ids_to_tokens(self, ids):
-        """ Converts a list of token IDs back to tokens. """
+        """Converts a list of token IDs back to tokens."""
         pass
 
     @abstractmethod
     def text_to_ids(self, text):
-        """ Converts text directly to token IDs. """
+        """Converts text directly to token IDs."""
         pass
 
     @abstractmethod
     def ids_to_text(self, ids):
-        """ Converts token IDs back to text. """
+        """Converts token IDs back to text."""
         pass
 
     def add_special_tokens(self, special_tokens: List[str]):
-        """ Adds special tokens (eos, pad, cls...) to vocab. """
+        """Adds special tokens (eos, pad, cls...) to vocab."""
         raise NotImplementedError("To be implemented")
 
     def apply_chat_template(self, *args, **kwargs):
-        """ Appies chat template and tokenizes results """
+        """Appies chat template and tokenizes results"""
         raise NotImplementedError("To be implemented")
 
     @property

tests/collections/common/test_apply_chat_template.py

@@ -1,35 +1,26 @@
 import pytest
-from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
 from transfomers import AutoTokenizer
 
+from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
+
+
 def test_chat_template():
     path = "/home/TestData/akoumparouli/tokenizer_with_chat_template/"
     tokenizers = [get_tokenizer(path), AutoTokenizer.from_pretrained(path)]
     prompt = "Give me a short introduction to pytest."
-    messages = [
-        {"role": "system", "content": "You are a helpful CI assistant."},
-        {"role": "user", "content": prompt}
+    messages = [{"role": "system", "content": "You are a helpful CI assistant."}, {"role": "user", "content": prompt}]
+    texts = [
+        tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) for tokenizer in tokenizers
     ]
-    texts = [tokenizer.apply_chat_template(
-        messages,
-        tokenize=False,
-        add_generation_prompt=True
-    ) for tokenizer in tokenizers]
     assert texts[0] == texts[1]
 
+
 def test_throws_chat_template():
     path = "/home/TestData/akoumparouli/tokenizer_without_chat_template/"
     tokenizer = get_tokenizer(path)
     prompt = "Give me a short introduction to pytest."
-    messages = [
-        {"role": "system", "content": "You are a helpful CI assistant."},
-        {"role": "user", "content": prompt}
-    ]
+    messages = [{"role": "system", "content": "You are a helpful CI assistant."}, {"role": "user", "content": prompt}]
     try:
-        tokenizer.apply_chat_template(
-            messages,
-            tokenize=False,
-            add_generation_prompt=True
-        )
+        tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     except ValueError as e:
-        assert 'Cannot use chat template functions because tokenizer.chat_template is not set' in str(e)
+        assert 'Cannot use chat template functions because tokenizer.chat_template is not set' in str(e)