From 703ff88d9a4f653325304884988e77f6a29a8b9f Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Mon, 13 Jan 2025 17:37:12 +0100 Subject: [PATCH] fix copies and style --- src/transformers/convert_slow_tokenizer.py | 22 ++++++++++++++----- .../models/helium/configuration_helium.py | 1 - .../models/helium/modular_helium.py | 6 +---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 6905634e271199..0c0b86beb181e7 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -1487,7 +1487,7 @@ def tokenizer(self, proto): ] ) tokenizer.add_tokens([AddedToken("\n", normalized=False, special=False)]) - tokenizer.enable_padding(pad_token ="", pad_id=3) + tokenizer.enable_padding(pad_token="", pad_id=3) return tokenizer def vocab(self, proto): @@ -1513,15 +1513,27 @@ def decoder(self, replacement, add_prefix_space): return decoders.Sequence(sequence) def normalizer(self, proto): - return normalizers.Sequence([normalizers.Prepend(" "), normalizers.Replace(r" ","▁")]) #, normalizers.Replace("\n", "<0x0A>" )]) + return normalizers.Sequence( + [normalizers.Prepend(" "), normalizers.Replace(r" ", "▁")] + ) # , normalizers.Replace("\n", "<0x0A>" )]) def pre_tokenizer(self, replacement, add_prefix_space): - return pre_tokenizers.Sequence([pre_tokenizers.Split('\n', "contiguous")]) #(prepend_scheme="always", split=True)]) + return pre_tokenizers.Sequence( + [pre_tokenizers.Split("\n", "contiguous")] + ) # (prepend_scheme="always", split=True)]) def post_processor(self): return processors.TemplateProcessing( - single=["", "$A",], - pair=[ "", "$A", "", "$B",], + single=[ + "", + "$A", + ], + pair=[ + "", + "$A", + "", + "$B", + ], special_tokens=[ ("", 1), ], diff --git a/src/transformers/models/helium/configuration_helium.py b/src/transformers/models/helium/configuration_helium.py index 6c79f9a1d553c9..fd19a1b4262b47 100644 --- a/src/transformers/models/helium/configuration_helium.py +++ b/src/transformers/models/helium/configuration_helium.py @@ -138,4 +138,3 @@ def __init__( __all__ = ["HeliumConfig"] - diff --git a/src/transformers/models/helium/modular_helium.py b/src/transformers/models/helium/modular_helium.py index e84ff9ebe431a5..0c9be5ec805afd 100644 --- a/src/transformers/models/helium/modular_helium.py +++ b/src/transformers/models/helium/modular_helium.py @@ -28,15 +28,12 @@ ) from ..granite.modeling_granite import ( GraniteAttention, - GraniteFlashAttention2, - GraniteSdpaAttention, ) from ..llama.modeling_llama import ( LlamaDecoderLayer, + LlamaMLP, LlamaModel, LlamaPreTrainedModel, - LlamaMLP, - LlamaRMSNorm, LlamaRotaryEmbedding, ) from .configuration_helium import HeliumConfig @@ -172,4 +169,3 @@ def __init__(self, config: HeliumConfig): "HeliumForSequenceClassification", "HeliumForTokenClassification", ] -