Preprocessing_modélisation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jAgSzf8EH8ac"
      },
      "source": [
        "# Import des bilbiothèques nécessaires"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2meB8SBlrXTS",
        "outputId": "9997b04e-b445-4894-bd14-06f98464ae02"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score, recall_score\n",
        "from gensim.models import Word2Vec, FastText\n",
        "from sklearn.model_selection import train_test_split\n",
        "import re\n",
        "import mlflow\n",
        "\n",
        "MLFLOW_TRACKING_URI = \"file:///G:/Mon Drive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/mlruns_local\"\n",
        "\n",
        "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n",
        "directory_path = \"G:/Mon Drive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "xIVORNXwWzxD"
      },
      "outputs": [],
      "source": [
        "# Load preprocessed data\n",
        "from sklearn.model_selection import train_test_split\n",
        "import numpy as np\n",
        "subset_y = np.load(\n",
        "    'G:/Mon Drive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/subset_y.npy')\n",
        "data_lemastem = np.load(\n",
        "    'G:/Mon Drive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/preprocessed_text.npy')\n",
        "\n",
        "\n",
        "X_temp, X_dont_use, y_temp, y_dont_use = train_test_split(\n",
        "    data_lemastem, subset_y, test_size=0.3, random_state=42)\n",
        "X_train, X_temp, y_train, y_temp = train_test_split(\n",
        "    X_temp, y_temp, test_size=0.3, random_state=42)\n",
        "X_val, X_test, y_val, y_test = train_test_split(\n",
        "    X_temp, y_temp, test_size=0.5, random_state=42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qFKuFKU2s_9f"
      },
      "source": [
        "## Utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "IAJdiee2H8ah"
      },
      "outputs": [],
      "source": [
        "# Fonctions\n",
        "\n",
        "def preprocess_text(text):\n",
        "    \"\"\"\n",
        "    Fonction pour le nettoyage de base du texte des tweets.\n",
        "    \"\"\"\n",
        "    # Suppression des URL\n",
        "    text = re.sub(r\"http/S+|www/S+|https/S+\", '', text, flags=re.MULTILINE)\n",
        "    # Suppression des mentions et hashtags\n",
        "    text = re.sub(r'/@/w+|/#', '', text)\n",
        "    # Suppression des caractères spéciaux et numériques\n",
        "    text = re.sub(r'/d+', '', text)\n",
        "    text = re.sub(r'/W+', ' ', text, flags=re.MULTILINE)\n",
        "    # Minuscules\n",
        "    text = text.lower()\n",
        "    return text\n",
        "\n",
        "# Fonction pour générer des vecteurs moyens à partir des plongements pour chaque tweet\n",
        "\n",
        "\n",
        "def vectorize_texts(model, sentences):\n",
        "    \"\"\"\n",
        "    Convertit les textes en vecteurs moyens en utilisant le modèle de plongements donné.\n",
        "    \"\"\"\n",
        "    vectorized = []\n",
        "    for words in tqdm(sentences):\n",
        "        vector = np.mean([model.wv[word] for word in words if word in model.wv] or [\n",
        "                         np.zeros(model.vector_size)], axis=0)\n",
        "        vectorized.append(vector)\n",
        "    return np.array(vectorized)\n",
        "\n",
        "# Fonction pour entraîner et évaluer le modèle, et enregistrer les résultats avec MLflow\n",
        "\n",
        "\n",
        "def train_evaluate(X_train, X_test, y_train, y_test, model_name):\n",
        "    \"\"\"\n",
        "    Entraîne une régression logistique sur les vecteurs fournis et évalue la performance.\n",
        "    Log les résultats avec MLflow.\n",
        "    \"\"\"\n",
        "    # Initialisation de MLflow\n",
        "    mlflow.set_experiment(\"Tweet Sentiment Analysis\")\n",
        "\n",
        "    with mlflow.start_run():\n",
        "        # Entraînement du modèle\n",
        "        model = LogisticRegression(max_iter=1000)\n",
        "        mlflow.tensorflow.autolog(registered_model_name='model')\n",
        "        model.fit(X_train, y_train)\n",
        "\n",
        "        # Prédictions et évaluation\n",
        "        predictions = model.predict(X_test)\n",
        "        accuracy = accuracy_score(y_test, predictions)\n",
        "        recall = recall_score(y_test, predictions)\n",
        "\n",
        "        # Log des paramètres, métriques et modèle\n",
        "        mlflow.log_params(\n",
        "            {\"model_type\": \"Logistic Regression\", \"embedding_type\": model_name})\n",
        "        mlflow.log_metrics({\"accuracy\": accuracy, \"recall\": recall})\n",
        "        mlflow.sklearn.log_model(model, f\"model_{model_name}\")\n",
        "\n",
        "        print(\n",
        "            f\"Results for {model_name}: Accuracy = {accuracy:.4f}, Recall = {recall:.4f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2KHOJC89H8ah"
      },
      "source": [
        "# Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HpVDqAAKH8ai",
        "outputId": "eef8f40c-a524-4f1f-e64c-d56f1652b60d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1600000 entries, 0 to 1599999\n",
            "Data columns (total 3 columns):\n",
            " #   Column   Non-Null Count    Dtype \n",
            "---  ------   --------------    ----- \n",
            " 0   target   1600000 non-null  int64 \n",
            " 1   content  1600000 non-null  object\n",
            " 2   gt       1600000 non-null  int64 \n",
            "dtypes: int64(2), object(1)\n",
            "memory usage: 36.6+ MB\n"
          ]
        }
      ],
      "source": [
        "# Import des données\n",
        "import pandas as pd\n",
        "data = pd.read_csv(directory_path+'sentiment140 (1)/training.1600000.processed.noemoticon.csv',\n",
        "                   names=['target', 'id', 'date',\n",
        "                          'statut', 'usertag', 'content'],\n",
        "                   sep=',',\n",
        "                   encoding_errors='ignore')\n",
        "\n",
        "data = data[['target', 'content']]\n",
        "data['gt'] = data['target'].map({0: 0, 4: 1})\n",
        "# np.save(directory_path+'labels.npy', data['gt'])\n",
        "# Preprocessing\n",
        "data['text_processed'] = data['content'].apply(lambda x: preprocess_text(x))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RfiSO7GII6Mw"
      },
      "source": [
        "## Lemmatisation et Stemmatisation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v0QDMg6E_Ohi"
      },
      "outputs": [],
      "source": [
        "# Subset and split datas\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X = data['content']\n",
        "y = data['gt']\n",
        "\n",
        "# Subsetting the data to avoid OOM issue\n",
        "subset_x, do_not_use_x, subset_y, do_not_use_y = train_test_split(\n",
        "    X, y, train_size=0.2, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Wls8HXY7H8aj"
      },
      "outputs": [],
      "source": [
        "import nltk\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')\n",
        "livrables_path = \"/content/drive/MyDrive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/\"\n",
        "\n",
        "stemmer = PorterStemmer()\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "# Fonction pour le prétraitement du texte avec lemmatisation et stemming\n",
        "\n",
        "\n",
        "def preprocess_text_advanced(text, stop_words):\n",
        "    # Tokenisation\n",
        "    words = nltk.word_tokenize(text)\n",
        "\n",
        "    # Suppression des stopwords\n",
        "    words = [word for word in words if word not in stop_words]\n",
        "\n",
        "    # Lemmatisation\n",
        "    words = [lemmatizer.lemmatize(word) for word in words]\n",
        "\n",
        "    # Stemming\n",
        "    words = [stemmer.stem(word) for word in words]\n",
        "\n",
        "    # Retourne le texte prétraité\n",
        "    return \" \".join(words)\n",
        "\n",
        "\n",
        "data_lemastem = subset_x.apply(\n",
        "    lambda x: preprocess_text_advanced(x, stop_words))\n",
        "np.save(livrables_path + 'subset_y', list(subset_y))\n",
        "# np.save(livrables_path + 'preprocessed_text', list(data_lemastem) )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y-W3OxTxFEWM"
      },
      "source": [
        "## Embedding with Word2Vec and FastText"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W-Gef4oOQ3iE"
      },
      "outputs": [],
      "source": [
        "sentences = [row.split() for row in data['text_processed']]\n",
        "# Word2Vec\n",
        "model_w2v = Word2Vec(sentences, vector_size=100,\n",
        "                     window=5, min_count=1, workers=4)\n",
        "model_w2v.save(directory_path+'w2v_model.model')\n",
        "\n",
        "# FastText\n",
        "model_ft = FastText(sentences, vector_size=100,\n",
        "                    window=5, min_count=1, workers=4)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ahFr3FfRir-",
        "outputId": "14cd623a-0b84-4374-9ed2-f5e0013e7322"
      },
      "outputs": [],
      "source": [
        "# Convert text to vector\n",
        "w2v_vectors = vectorize_texts(model_w2v, sentences)\n",
        "ft_vectors = vectorize_texts(model_ft, sentences)\n",
        "\n",
        "# Save the built vectors\n",
        "\n",
        "# Sauvegarde des vecteurs Word2Vec\n",
        "np.save('/content/drive/MyDrive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/w2v_vectors.npy', w2v_vectors)\n",
        "\n",
        "# Sauvegarde des vecteurs FastText\n",
        "np.save('/content/drive/MyDrive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/cft_vectors.npy', ft_vectors)\n",
        "\n",
        "# Save target array\n",
        "# np.save('/kaggle/input/gt.np', data['gt'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k56aUhARLdpn"
      },
      "source": [
        "## Création de la matrice d'embedding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZL5fD1302z_O"
      },
      "outputs": [],
      "source": [
        "# Create embedding matrix\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from gensim.models import Word2Vec\n",
        "\n",
        "# Et que `data_texts` est votre liste de textes.\n",
        "\n",
        "text_processed = np.load(\n",
        "    '/content/drive/MyDrive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/text_processed.npy', data['text_processed'])\n",
        "# Création et adaptation d'un tokenizer Keras\n",
        "tokenizer = Tokenizer(num_words=10000)\n",
        "tokenizer.fit_on_texts(data['text_processed'])\n",
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "\n",
        "# Taille des vecteurs Word2Vec\n",
        "embedding_dim = model_w2v.vector_size\n",
        "\n",
        "# Initialisation de la matrice d'embedding avec des zéros\n",
        "embedding_matrix = np.zeros((vocab_size, embedding_dim))\n",
        "\n",
        "# Remplissage de la matrice d'embedding\n",
        "for word, i in tokenizer.word_index.items():\n",
        "    if word in model_w2v.wv:\n",
        "        embedding_vector = model_w2v.wv[word]\n",
        "        if embedding_vector is not None:\n",
        "            embedding_matrix[i] = embedding_vector\n",
        "np.save('/content/drive/MyDrive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/embedding_matrix.npy', embedding_matrix)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QwVxx4YTWP3n"
      },
      "source": [
        "## Embedding via USE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2UTWb5WyA_Jg"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "text, text_trash, labels, labels_trash = train_test_split(\n",
        "    data['text_processed'], data['gt'], train_size=0.3, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "duyu8JAVWSYn",
        "outputId": "e9229c54-fa03-47f7-9c22-655ad9781853"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import numpy as np\n",
        "\n",
        "# Chargez le modèle Universal Sentence Encoder de TensorFlow Hub\n",
        "use = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n",
        "\n",
        "\n",
        "def generate_embeddings(sentences):\n",
        "    \"\"\"\n",
        "    Génère des embeddings pour chaque phrase en utilisant USE et les convertit en arrays numpy.\n",
        "\n",
        "    :param sentences: Liste de phrases (strings).\n",
        "    :return: Numpy array des embeddings.\n",
        "    \"\"\"\n",
        "    embeddings = use(sentences)\n",
        "    embeddings_np = np.array(embeddings)\n",
        "    return embeddings_np\n",
        "\n",
        "\n",
        "# Exemple d'utilisation\n",
        "if __name__ == \"__main__\":\n",
        "    # data_lemastem = np.load('/content/drive/MyDrive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/preprocessed_text.npy')\n",
        "    # sentences = [\" \".join(tokens) for tokens in data_lemastem]\n",
        "    sentences = text\n",
        "    embeddings_np = generate_embeddings(sentences)\n",
        "    print(\"Shape of embeddings:\", embeddings_np.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W0reamnUKweM"
      },
      "source": [
        "# Entrainement des modèles"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tRKuu7hoN4s0"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Dropout, Embedding, GlobalMaxPooling1D, LSTM, Conv1D\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "\n",
        "directory_path = \"G:/Mon Drive/Documents/Apprentissage/OpenClassroom/Projet_7_Analyse_de_sentiments/\"\n",
        "vocab_size = 10000\n",
        "labels = np.load(directory_path+'labels.npy')\n",
        "embedding_matrix = np.load(directory_path+'embedding_matrix.npy')\n",
        "w2v_vectors = np.load(directory_path+'w2v_vectors.npy')\n",
        "\n",
        "# Séparation en ensembles d'entraînement et de test\n",
        "X_train, X_do_not_use, y_train, y_do_not_use = train_test_split(\n",
        "    w2v_vectors, labels, train_size=0.2, random_state=42)\n",
        "X_train, X_temp, y_train, y_temp = train_test_split(\n",
        "    X_train, y_train, test_size=0.3, random_state=42)\n",
        "X_val, X_test, y_val, y_test = train_test_split(\n",
        "    X_temp, y_temp, test_size=0.5, random_state=42)\n",
        "\n",
        "X_train = np.maximum(X_train, 0)\n",
        "X_test = np.maximum(X_test, 0)\n",
        "X_val = np.maximum(X_val, 0)\n",
        "\n",
        "embedding_vector_length = X_train.shape[1]\n",
        "maxlen = 100\n",
        "vocab_size = embedding_matrix.shape[0]\n",
        "embedding_dim = embedding_matrix.shape[1]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Construction du modèle baseline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "32uEQwrkeTG-"
      },
      "outputs": [],
      "source": [
        "# Reprise du modèle avec les meilleures performances\n",
        "keras_model = Sequential()\n",
        "keras_model.add(Dense(128, activation='relu',\n",
        "                input_dim=embedding_vector_length))\n",
        "keras_model.add(Dropout(0.2))\n",
        "keras_model.add(Dense(64, activation='relu'))\n",
        "keras_model.add(Dropout(0.2))\n",
        "# Pour la classification binaire, utilisez 'softmax' pour la classification multiclasse\n",
        "keras_model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "early_stopping = EarlyStopping(patience=3,\n",
        "                               monitor='val_loss',\n",
        "                               restore_best_weights=True)\n",
        "checkpoint = ModelCheckpoint(monitor='val_loss',\n",
        "                             filepath=(directory_path+'Models/baseline.keras'),\n",
        "                             save_best_only=True,\n",
        "                             save_weights_only=False)\n",
        "\n",
        "# Compilation du modèle\n",
        "keras_model.compile(optimizer='adam', loss='binary_crossentropy',\n",
        "                    metrics=['accuracy', tf.keras.metrics.AUC()])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Entraînement du modèle\n",
        "mlflow.set_experiment('models_training')\n",
        "mlflow.autolog()\n",
        "keras_model.fit(X_train, y_train, validation_data=(X_test, y_test),\n",
        "                epochs=10,\n",
        "                callbacks=[early_stopping, checkpoint],\n",
        "                batch_size=32)\n",
        "\n",
        "# Sauvegarde du modèle\n",
        "\n",
        "# tf.keras.models.save_model(keras_model, directory_path+'keras_model.keras')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "keras_model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Evaluation\n",
        "keras_model.evaluate(X_val, y_val, return_dict=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Keras embedding seul"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6_sPHkscLkMA"
      },
      "outputs": [],
      "source": [
        "# Construction du modèle\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=vocab_size, output_dim=embedding_dim,\n",
        "                    weights=[embedding_matrix],\n",
        "                    input_length=maxlen,\n",
        "                    trainable=False))\n",
        "model.add(Conv1D(128, 5, activation='relu'))\n",
        "model.add(GlobalMaxPooling1D())\n",
        "model.add(Dense(128, activation='relu',\n",
        "                input_dim=embedding_vector_length))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "early_stopping = EarlyStopping(patience=3,\n",
        "                               monitor='val_loss',\n",
        "                               restore_best_weights=True)\n",
        "checkpoint = ModelCheckpoint(monitor='val_loss',\n",
        "                             filepath=(directory_path +\n",
        "                                       'Models/embedding_model.keras'),\n",
        "                             save_best_only=True,\n",
        "                             save_weights_only=False)\n",
        "\n",
        "# Compilation du modèle\n",
        "model.compile(optimizer='adam', loss='binary_crossentropy',\n",
        "              metrics=['accuracy', tf.keras.metrics.AUC()])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "# Entraînement du modèle\n",
        "mlflow.autolog()\n",
        "model.fit(X_train, y_train, validation_data=(X_test, y_test),\n",
        "          epochs=10,\n",
        "          callbacks=[early_stopping, checkpoint],\n",
        "          batch_size=32)\n",
        "\n",
        "# Sauvegarde du modèle\n",
        "\n",
        "# tf.keras.models.save_model(model, directory_path+'model_embedding_new.keras')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Evaluation\n",
        "\n",
        "model.evaluate(X_val, y_val, return_dict=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lAGjcsL9L48W"
      },
      "source": [
        "## Keras Embedding + LSTM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g9a5GHquL836"
      },
      "outputs": [],
      "source": [
        "# Chargement des embeddings\n",
        "from tensorflow.keras.layers import Dense, Dropout, Bidirectional, LSTM\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "import tensorflow as tf\n",
        "\n",
        "\n",
        "# modèle avec LSTM\n",
        "model_LSTM = Sequential()\n",
        "model_LSTM.add(Embedding(input_dim=vocab_size, output_dim=embedding_dim, weights=[\n",
        "               embedding_matrix], input_length=maxlen, trainable=False))\n",
        "model_LSTM.add(Conv1D(128, 5, activation='relu'))\n",
        "model_LSTM.add(LSTM(32))\n",
        "model_LSTM.add(Dropout(0.5))\n",
        "model_LSTM.add(Dense(64, activation='relu'))\n",
        "model_LSTM.add(Dropout(0.5))\n",
        "model_LSTM.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "# Compilation du modèle\n",
        "model_LSTM.compile(optimizer='adam',\n",
        "                   loss='binary_crossentropy',\n",
        "                   metrics=['accuracy', tf.keras.metrics.AUC()])\n",
        "\n",
        "\n",
        "checkpoint = ModelCheckpoint(monitor='val_loss',\n",
        "                             filepath=(directory_path +\n",
        "                                       'baseline_new_LSTM.keras'),\n",
        "                             save_best_only=True,\n",
        "                             save_weights_only=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "mlflow.autolog()\n",
        "model_LSTM.fit(X_train, y_train, validation_data=(X_test, y_test),\n",
        "               epochs=10,\n",
        "               callbacks=[early_stopping, checkpoint],\n",
        "               batch_size=32)\n",
        "\n",
        "# Sauvegarde du modèle\n",
        "\n",
        "tf.keras.models.save_model(\n",
        "    model_LSTM, directory_path+'model_embedding_LSTM_new.keras')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model_LSTM.evaluate(X_val, y_val, return_dict=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Fine-tuning BERT\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From C:\\Users\\clema\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
            "\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "from transformers import BertTokenizer, TFBertForSequenceClassification\n",
        "from transformers import TFAutoModelForSequenceClassification, AutoTokenizer\n",
        "from transformers import InputExample, InputFeatures\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForSequenceClassification: ['fit_denses.0.weight', 'fit_denses.3.bias', 'fit_denses.3.weight', 'fit_denses.0.bias', 'fit_denses.1.bias', 'fit_denses.2.bias', 'fit_denses.1.weight', 'fit_denses.2.weight', 'fit_denses.4.bias', 'fit_denses.4.weight']\n",
            "- This IS expected if you are initializing TFBertForSequenceClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertForSequenceClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFBertForSequenceClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "995eff37be8e43b7bb848f8f6d1fb14c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "!pip -qq install torch\n",
        "df = data[['text_processed', 'gt']]\n",
        "\n",
        "# Séparation en ensembles d'entraînement et de test\n",
        "train_examples, trash_examples = train_test_split(\n",
        "    df, train_size=0.2, random_state=42)\n",
        "train_examples, temp_examples = train_test_split(\n",
        "    train_examples, test_size=0.3, random_state=42)\n",
        "test_examples, validation_examples = train_test_split(\n",
        "    temp_examples, test_size=0.5, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Fonction de prétraitement pour BERT\n",
        "def convert_example_to_feature(review):\n",
        "    return tokenizer.encode_plus(review,\n",
        "                                 add_special_tokens=True,\n",
        "                                 max_length=160,\n",
        "                                 pad_to_max_length=True,\n",
        "                                 return_attention_mask=True,\n",
        "                                 )\n",
        "\n",
        "\n",
        "def map_example_to_dict(input_ids, attention_masks, token_type_ids, label):\n",
        "    return {\n",
        "        \"input_ids\": input_ids,\n",
        "        \"token_type_ids\": token_type_ids,\n",
        "        \"attention_mask\": attention_masks,\n",
        "    }, label\n",
        "\n",
        "\n",
        "def encode_examples(ds, limit=-1):\n",
        "    input_ids_list = []\n",
        "    token_type_ids_list = []\n",
        "    attention_mask_list = []\n",
        "    label_list = []\n",
        "\n",
        "    for index, row in ds.iterrows():\n",
        "        bert_input = convert_example_to_feature(row['text_processed'])\n",
        "        input_ids_list.append(bert_input['input_ids'])\n",
        "        token_type_ids_list.append(bert_input['token_type_ids'])\n",
        "        attention_mask_list.append(bert_input['attention_mask'])\n",
        "        label_list.append([row['gt']])\n",
        "    return tf.data.Dataset.from_tensor_slices((input_ids_list, attention_mask_list, token_type_ids_list, label_list)).map(map_example_to_dict)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "C:\\Users\\clema\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\transformers\\tokenization_utils_base.py:2688: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Prétraitement des données pour BERT et import du modèle\n",
        "train_data = encode_examples(train_examples)\n",
        "validation_data = encode_examples(validation_examples)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From C:\\Users\\clema\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\keras\\src\\optimizers\\__init__.py:309: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "Model compiled\n"
          ]
        }
      ],
      "source": [
        "\n",
        "# Chargement du modèle et du tokenizer correspondant\n",
        "model_name = 'huawei-noah/TinyBERT_General_4L_312D'\n",
        "model = TFAutoModelForSequenceClassification.from_pretrained(\n",
        "    model_name, num_labels=1, from_pt=True)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name, from_pt=True)\n",
        "\n",
        "\n",
        "model.compile(optimizer='adam', loss='binary_crossentropy',\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "              metrics=['accuracy'])  # tf.keras.metrics.AUC()\n",
        "\n",
        "\n",
        "print('Model compiled')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Entrainement du modèle\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "\n",
        "checkpoint = ModelCheckpoint(monitor='val_loss',\n",
        "                             filepath=(directory_path +\n",
        "                                       'Models/Tiny_bert.keras'),\n",
        "                             save_best_only=True,\n",
        "                             save_weights_only=False,\n",
        "                             save_format='tf')\n",
        "early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2024/04/09 10:46:51 WARNING mlflow.utils.autologging_utils: You are using an unsupported version of transformers. If you encounter errors during autologging, try upgrading / downgrading transformers to a supported version, or try upgrading MLflow.\n",
            "2024/04/09 10:46:51 WARNING mlflow.tracking.fluent: Exception raised while enabling autologging for transformers: '<class 'transformers.utils.import_utils.DummyObject'>' object has no attribute 'train'\n",
            "2024/04/09 10:46:51 INFO mlflow.tracking.fluent: Autologging successfully enabled for keras.\n",
            "2024/04/09 10:46:51 INFO mlflow.tracking.fluent: Autologging successfully enabled for tensorflow.\n",
            "2024/04/09 10:46:54 INFO mlflow.tracking.fluent: Autologging successfully enabled for sklearn.\n",
            "2024/04/09 10:46:55 INFO mlflow.utils.autologging_utils: Created MLflow autologging run with ID '96931c825cf34b56b0c4678e4077746b', which will track hyperparameters, performance metrics, model artifacts, and lineage information for the current tensorflow workflow\n",
            "2024/04/09 10:46:55 WARNING mlflow.tensorflow: Encountered unexpected error while inferring batch size from training dataset: The layer \"tf_bert_for_sequence_classification_2\" has never been called and thus has no defined input shape. Note that the `input_shape` property is only available for Functional and Sequential models.\n",
            "2024/04/09 10:47:01 WARNING mlflow.data.tensorflow_dataset: Failed to infer schema for TensorFlow dataset. Exception: Failed to infer schema for tf.data.Dataset. Schemas can only be inferred if the dataset consists of tensors. Ragged tensors, tensor arrays, and other types are not supported. Additionally, datasets with nested tensors are not supported.\n",
            "2024/04/09 10:47:03 WARNING mlflow.data.tensorflow_dataset: Failed to infer schema for TensorFlow dataset. Exception: Failed to infer schema for tf.data.Dataset. Schemas can only be inferred if the dataset consists of tensors. Ragged tensors, tensor arrays, and other types are not supported. Additionally, datasets with nested tensors are not supported.\n",
            "2024/04/09 10:47:04 WARNING mlflow.utils.autologging_utils: Encountered unexpected error during autologging: Metric 'desktop.ini' is malformed; persisted metric data contained 1 fields. Expected 2 or 3 fields.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "WARNING:tensorflow:From C:\\Users\\clema\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n",
            "\n",
            "WARNING:tensorflow:From C:\\Users\\clema\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\keras\\src\\engine\\base_layer_utils.py:384: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n",
            "\n",
            " 725/7000 [==>...........................] - ETA: 4:01:59 - loss: 0.7083 - accuracy: 0.4984"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "Cell \u001b[1;32mIn[21], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Fine-tuning\u001b[39;00m\n\u001b[0;32m      2\u001b[0m mlflow\u001b[38;5;241m.\u001b[39mautolog()\n\u001b[1;32m----> 3\u001b[0m model\u001b[38;5;241m.\u001b[39mfit(train_data\u001b[38;5;241m.\u001b[39mshuffle(\u001b[38;5;241m100\u001b[39m)\u001b[38;5;241m.\u001b[39mbatch(\u001b[38;5;241m32\u001b[39m),\n\u001b[0;32m      4\u001b[0m           epochs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m,\n\u001b[0;32m      5\u001b[0m           batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m32\u001b[39m,\n\u001b[0;32m      6\u001b[0m           validation_data\u001b[38;5;241m=\u001b[39mvalidation_data\u001b[38;5;241m.\u001b[39mbatch(\u001b[38;5;241m32\u001b[39m),\n\u001b[0;32m      7\u001b[0m           callbacks\u001b[38;5;241m=\u001b[39m[early_stopping, checkpoint])\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\transformers\\modeling_tf_utils.py:1170\u001b[0m, in \u001b[0;36mTFPreTrainedModel.fit\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1167\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(keras\u001b[38;5;241m.\u001b[39mModel\u001b[38;5;241m.\u001b[39mfit)\n\u001b[0;32m   1168\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m   1169\u001b[0m     args, kwargs \u001b[38;5;241m=\u001b[39m convert_batch_encoding(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m-> 1170\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mfit(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:576\u001b[0m, in \u001b[0;36msafe_patch.<locals>.safe_patch_function\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    566\u001b[0m try_log_autologging_event(\n\u001b[0;32m    567\u001b[0m     AutologgingEventLogger\u001b[38;5;241m.\u001b[39mget_logger()\u001b[38;5;241m.\u001b[39mlog_patch_function_start,\n\u001b[0;32m    568\u001b[0m     session,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    572\u001b[0m     kwargs,\n\u001b[0;32m    573\u001b[0m )\n\u001b[0;32m    575\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m patch_is_class:\n\u001b[1;32m--> 576\u001b[0m     patch_function\u001b[38;5;241m.\u001b[39mcall(call_original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    577\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    578\u001b[0m     patch_function(call_original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:167\u001b[0m, in \u001b[0;36mPatchFunction.call\u001b[1;34m(cls, original, *args, **kwargs)\u001b[0m\n\u001b[0;32m    165\u001b[0m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[0;32m    166\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall\u001b[39m(\u001b[38;5;28mcls\u001b[39m, original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 167\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mcls\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m(original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:178\u001b[0m, in \u001b[0;36mPatchFunction.__call__\u001b[1;34m(self, original, *args, **kwargs)\u001b[0m\n\u001b[0;32m    174\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_on_exception(e)\n\u001b[0;32m    175\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m    176\u001b[0m     \u001b[38;5;66;03m# Regardless of what happens during the `_on_exception` callback, reraise\u001b[39;00m\n\u001b[0;32m    177\u001b[0m     \u001b[38;5;66;03m# the original implementation exception once the callback completes\u001b[39;00m\n\u001b[1;32m--> 178\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:171\u001b[0m, in \u001b[0;36mPatchFunction.__call__\u001b[1;34m(self, original, *args, **kwargs)\u001b[0m\n\u001b[0;32m    169\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m    170\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 171\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_patch_implementation(original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    172\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mException\u001b[39;00m, \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    173\u001b[0m         \u001b[38;5;28;01mtry\u001b[39;00m:\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:229\u001b[0m, in \u001b[0;36mwith_managed_run.<locals>.PatchWithManagedRun._patch_implementation\u001b[1;34m(self, original, *args, **kwargs)\u001b[0m\n\u001b[0;32m    226\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m mlflow\u001b[38;5;241m.\u001b[39mactive_run():\n\u001b[0;32m    227\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmanaged_run \u001b[38;5;241m=\u001b[39m create_managed_run()\n\u001b[1;32m--> 229\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m_patch_implementation(original, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    231\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmanaged_run:\n\u001b[0;32m    232\u001b[0m     mlflow\u001b[38;5;241m.\u001b[39mend_run(RunStatus\u001b[38;5;241m.\u001b[39mto_string(RunStatus\u001b[38;5;241m.\u001b[39mFINISHED))\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\tensorflow\\__init__.py:1345\u001b[0m, in \u001b[0;36mautolog.<locals>.FitPatch._patch_implementation\u001b[1;34m(self, original, inst, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1338\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m   1339\u001b[0m         _logger\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[0;32m   1340\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to log training dataset information to \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1341\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMLflow Tracking. Reason: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   1342\u001b[0m             e,\n\u001b[0;32m   1343\u001b[0m         )\n\u001b[1;32m-> 1345\u001b[0m history \u001b[38;5;241m=\u001b[39m original(inst, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m log_models:\n\u001b[0;32m   1348\u001b[0m     _log_keras_model(history, args)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:559\u001b[0m, in \u001b[0;36msafe_patch.<locals>.safe_patch_function.<locals>.call_original\u001b[1;34m(*og_args, **og_kwargs)\u001b[0m\n\u001b[0;32m    556\u001b[0m         original_result \u001b[38;5;241m=\u001b[39m original(\u001b[38;5;241m*\u001b[39m_og_args, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m_og_kwargs)\n\u001b[0;32m    557\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m original_result\n\u001b[1;32m--> 559\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m call_original_fn_with_event_logging(_original_fn, og_args, og_kwargs)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:494\u001b[0m, in \u001b[0;36msafe_patch.<locals>.safe_patch_function.<locals>.call_original_fn_with_event_logging\u001b[1;34m(original_fn, og_args, og_kwargs)\u001b[0m\n\u001b[0;32m    485\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m    486\u001b[0m     try_log_autologging_event(\n\u001b[0;32m    487\u001b[0m         AutologgingEventLogger\u001b[38;5;241m.\u001b[39mget_logger()\u001b[38;5;241m.\u001b[39mlog_original_function_start,\n\u001b[0;32m    488\u001b[0m         session,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    492\u001b[0m         og_kwargs,\n\u001b[0;32m    493\u001b[0m     )\n\u001b[1;32m--> 494\u001b[0m     original_fn_result \u001b[38;5;241m=\u001b[39m original_fn(\u001b[38;5;241m*\u001b[39mog_args, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mog_kwargs)\n\u001b[0;32m    496\u001b[0m     try_log_autologging_event(\n\u001b[0;32m    497\u001b[0m         AutologgingEventLogger\u001b[38;5;241m.\u001b[39mget_logger()\u001b[38;5;241m.\u001b[39mlog_original_function_success,\n\u001b[0;32m    498\u001b[0m         session,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    502\u001b[0m         og_kwargs,\n\u001b[0;32m    503\u001b[0m     )\n\u001b[0;32m    504\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m original_fn_result\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\mlflow\\utils\\autologging_utils\\safety.py:556\u001b[0m, in \u001b[0;36msafe_patch.<locals>.safe_patch_function.<locals>.call_original.<locals>._original_fn\u001b[1;34m(*_og_args, **_og_kwargs)\u001b[0m\n\u001b[0;32m    548\u001b[0m \u001b[38;5;66;03m# Show all non-MLflow warnings as normal (i.e. not as event logs)\u001b[39;00m\n\u001b[0;32m    549\u001b[0m \u001b[38;5;66;03m# during original function execution, even if silent mode is enabled\u001b[39;00m\n\u001b[0;32m    550\u001b[0m \u001b[38;5;66;03m# (`silent=True`), since these warnings originate from the ML framework\u001b[39;00m\n\u001b[0;32m    551\u001b[0m \u001b[38;5;66;03m# or one of its dependencies and are likely relevant to the caller\u001b[39;00m\n\u001b[0;32m    552\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m set_non_mlflow_warnings_behavior_for_current_thread(\n\u001b[0;32m    553\u001b[0m     disable_warnings\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    554\u001b[0m     reroute_warnings\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    555\u001b[0m ):\n\u001b[1;32m--> 556\u001b[0m     original_result \u001b[38;5;241m=\u001b[39m original(\u001b[38;5;241m*\u001b[39m_og_args, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m_og_kwargs)\n\u001b[0;32m    557\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m original_result\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\keras\\src\\utils\\traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\keras\\src\\engine\\training.py:1807\u001b[0m, in \u001b[0;36mModel.fit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1799\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[0;32m   1800\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   1801\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39mepoch,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1804\u001b[0m     _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[0;32m   1805\u001b[0m ):\n\u001b[0;32m   1806\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mon_train_batch_begin(step)\n\u001b[1;32m-> 1807\u001b[0m     tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrain_function(iterator)\n\u001b[0;32m   1808\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[0;32m   1809\u001b[0m         context\u001b[38;5;241m.\u001b[39masync_wait()\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\util\\traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:832\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    829\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    831\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[1;32m--> 832\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)\n\u001b[0;32m    834\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[0;32m    835\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:868\u001b[0m, in \u001b[0;36mFunction._call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    865\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[0;32m    866\u001b[0m   \u001b[38;5;66;03m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[0;32m    867\u001b[0m   \u001b[38;5;66;03m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[1;32m--> 868\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m tracing_compilation\u001b[38;5;241m.\u001b[39mcall_function(\n\u001b[0;32m    869\u001b[0m       args, kwds, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_no_variable_creation_config\n\u001b[0;32m    870\u001b[0m   )\n\u001b[0;32m    871\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_variable_creation_config \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    872\u001b[0m   \u001b[38;5;66;03m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[0;32m    873\u001b[0m   \u001b[38;5;66;03m# in parallel.\u001b[39;00m\n\u001b[0;32m    874\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\tracing_compilation.py:139\u001b[0m, in \u001b[0;36mcall_function\u001b[1;34m(args, kwargs, tracing_options)\u001b[0m\n\u001b[0;32m    137\u001b[0m bound_args \u001b[38;5;241m=\u001b[39m function\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mbind(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    138\u001b[0m flat_inputs \u001b[38;5;241m=\u001b[39m function\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39munpack_inputs(bound_args)\n\u001b[1;32m--> 139\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m function\u001b[38;5;241m.\u001b[39m_call_flat(  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[0;32m    140\u001b[0m     flat_inputs, captured_inputs\u001b[38;5;241m=\u001b[39mfunction\u001b[38;5;241m.\u001b[39mcaptured_inputs\n\u001b[0;32m    141\u001b[0m )\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\concrete_function.py:1323\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[1;34m(self, tensor_inputs, captured_inputs)\u001b[0m\n\u001b[0;32m   1319\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[0;32m   1320\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[0;32m   1321\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[0;32m   1322\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[1;32m-> 1323\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_inference_function\u001b[38;5;241m.\u001b[39mcall_preflattened(args)\n\u001b[0;32m   1324\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[0;32m   1325\u001b[0m     args,\n\u001b[0;32m   1326\u001b[0m     possible_gradient_type,\n\u001b[0;32m   1327\u001b[0m     executing_eagerly)\n\u001b[0;32m   1328\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\atomic_function.py:216\u001b[0m, in \u001b[0;36mAtomicFunction.call_preflattened\u001b[1;34m(self, args)\u001b[0m\n\u001b[0;32m    214\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall_preflattened\u001b[39m(\u001b[38;5;28mself\u001b[39m, args: Sequence[core\u001b[38;5;241m.\u001b[39mTensor]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[0;32m    215\u001b[0m \u001b[38;5;250m  \u001b[39m\u001b[38;5;124;03m\"\"\"Calls with flattened tensor inputs and returns the structured output.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 216\u001b[0m   flat_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcall_flat(\u001b[38;5;241m*\u001b[39margs)\n\u001b[0;32m    217\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mpack_output(flat_outputs)\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\atomic_function.py:251\u001b[0m, in \u001b[0;36mAtomicFunction.call_flat\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    249\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m record\u001b[38;5;241m.\u001b[39mstop_recording():\n\u001b[0;32m    250\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mexecuting_eagerly():\n\u001b[1;32m--> 251\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mcall_function(\n\u001b[0;32m    252\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname,\n\u001b[0;32m    253\u001b[0m         \u001b[38;5;28mlist\u001b[39m(args),\n\u001b[0;32m    254\u001b[0m         \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunction_type\u001b[38;5;241m.\u001b[39mflat_outputs),\n\u001b[0;32m    255\u001b[0m     )\n\u001b[0;32m    256\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    257\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m make_call_op_in_graph(\n\u001b[0;32m    258\u001b[0m         \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    259\u001b[0m         \u001b[38;5;28mlist\u001b[39m(args),\n\u001b[0;32m    260\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mfunction_call_options\u001b[38;5;241m.\u001b[39mas_attrs(),\n\u001b[0;32m    261\u001b[0m     )\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\context.py:1486\u001b[0m, in \u001b[0;36mContext.call_function\u001b[1;34m(self, name, tensor_inputs, num_outputs)\u001b[0m\n\u001b[0;32m   1484\u001b[0m cancellation_context \u001b[38;5;241m=\u001b[39m cancellation\u001b[38;5;241m.\u001b[39mcontext()\n\u001b[0;32m   1485\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cancellation_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m-> 1486\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute(\n\u001b[0;32m   1487\u001b[0m       name\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   1488\u001b[0m       num_outputs\u001b[38;5;241m=\u001b[39mnum_outputs,\n\u001b[0;32m   1489\u001b[0m       inputs\u001b[38;5;241m=\u001b[39mtensor_inputs,\n\u001b[0;32m   1490\u001b[0m       attrs\u001b[38;5;241m=\u001b[39mattrs,\n\u001b[0;32m   1491\u001b[0m       ctx\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m   1492\u001b[0m   )\n\u001b[0;32m   1493\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1494\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[0;32m   1495\u001b[0m       name\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   1496\u001b[0m       num_outputs\u001b[38;5;241m=\u001b[39mnum_outputs,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1500\u001b[0m       cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_context,\n\u001b[0;32m   1501\u001b[0m   )\n",
            "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tensorflow\\python\\eager\\execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     52\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[1;32m---> 53\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m pywrap_tfe\u001b[38;5;241m.\u001b[39mTFE_Py_Execute(ctx\u001b[38;5;241m.\u001b[39m_handle, device_name, op_name,\n\u001b[0;32m     54\u001b[0m                                       inputs, attrs, num_outputs)\n\u001b[0;32m     55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     56\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
            "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "# Fine-tuning\n",
        "mlflow.autolog()\n",
        "model.fit(train_data.shuffle(100).batch(32),\n",
        "          epochs=10,\n",
        "          batch_size=32,\n",
        "          validation_data=validation_data.batch(32),\n",
        "          callbacks=[early_stopping, checkpoint])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1500/1500 [==============================] - 996s 664ms/step - loss: 0.6987 - accuracy: 0.5036\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "{'loss': 0.6986682415008545, 'accuracy': 0.5036458373069763}"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Evaluation\n",
        "model.evaluate(\n",
        "    validation_data.batch(32), return_dict=True)\n",
        "\n",
        "\n",
        "# model.save_pretrained(\"./Models/bert_finetuned.keras\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "RfiSO7GII6Mw",
        "QwVxx4YTWP3n"
      ],
      "provenance": []
    },
    "kaggle": {
      "accelerator": "none",
      "dataSources": [
        {
          "datasetId": 4514566,
          "sourceId": 7727100,
          "sourceType": "datasetVersion"
        }
      ],
      "dockerImageVersionId": 30664,
      "isGpuEnabled": false,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}