add apply_chat_template method

Signed-off-by: Alexandros Koumparoulis <[email protected]>

Author: akoumpa

Diff for: nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py

@@ -312,6 +312,10 @@ def text_to_ids(self, text):
         ids = self.tokens_to_ids(tokens)
         return ids
 
+    def apply_chat_template(self, *args, **kwargs):
+        """ Appies chat template and tokenizes results """
+        return self.tokenizer.apply_chat_template(*args, **kwargs)
+
     def ids_to_text(self, ids, remove_special_tokens=True):
         """
         Converts token IDs back to text.

Diff for: nemo/collections/common/tokenizers/tokenizer_spec.py

@@ -51,6 +51,10 @@ def ids_to_text(self, ids):
     def add_special_tokens(self, special_tokens: List[str]):
         raise NotImplementedError("To be implemented")
 
+    def apply_chat_template(self, *args, **kwargs):
+        """ Appies chat template and tokenizes results """
+        raise NotImplementedError("To be implemented")
+
     @property
     def name(self):
         return type(self).__name__