training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c508eae1-e931-495f-8b86-e58e9d851527",
   "metadata": {},
   "source": [
    "## GPU Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "116eb584-3c28-4106-b50c-110307ec0776",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:17:58.537463Z",
     "iopub.status.busy": "2025-02-21T08:17:58.536639Z",
     "iopub.status.idle": "2025-02-21T08:17:58.541620Z",
     "shell.execute_reply": "2025-02-21T08:17:58.540867Z",
     "shell.execute_reply.started": "2025-02-21T08:17:58.537435Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# Enable memory growth for the specified GPU devices\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        # Currently, memory growth needs to be the same across GPUs\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        # Memory growth must be set before GPUs have been initialized\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b731632-0885-4e4e-a2d8-3a9e7e286caa",
   "metadata": {},
   "source": [
    "## Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cd7d4d3f-d582-42d4-938f-aba3b339ddf8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:01.408066Z",
     "iopub.status.busy": "2025-02-21T08:18:01.407623Z",
     "iopub.status.idle": "2025-02-21T08:18:05.517932Z",
     "shell.execute_reply": "2025-02-21T08:18:05.517475Z",
     "shell.execute_reply.started": "2025-02-21T08:18:01.408047Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow import keras\n",
    "from keras import Sequential\n",
    "from keras.layers import Embedding, SpatialDropout1D, Dense, Dropout, LSTM,Bidirectional, Flatten,Conv1D, GlobalMaxPooling1D\n",
    "from keras.callbacks import EarlyStopping\n",
    "import transformers\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2146ab77-9392-441f-9c03-12743ca45811",
   "metadata": {},
   "source": [
    "## Data Read and pre - process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "74838c57-6bc5-4238-864f-0f60fa212e43",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:08.034006Z",
     "iopub.status.busy": "2025-02-21T08:18:08.033540Z",
     "iopub.status.idle": "2025-02-21T08:18:09.125050Z",
     "shell.execute_reply": "2025-02-21T08:18:09.124545Z",
     "shell.execute_reply.started": "2025-02-21T08:18:08.033984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>posts</th>\n",
       "      <th>cleaned_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>INFJ</td>\n",
       "      <td>'http://www.youtube.com/watch?v=qsXHcwe3krw|||...</td>\n",
       "      <td>enfp and intj moments sportscenter not top ten...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ENTP</td>\n",
       "      <td>'I'm finding the lack of me in these posts ver...</td>\n",
       "      <td>im finding the lack of me in these posts very ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>INTP</td>\n",
       "      <td>'Good one  _____   https://www.youtube.com/wat...</td>\n",
       "      <td>good one  of course to which i say i know that...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>INTJ</td>\n",
       "      <td>'Dear INTP,   I enjoyed our conversation the o...</td>\n",
       "      <td>dear intp i enjoyed our conversation the other...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ENTJ</td>\n",
       "      <td>'You're fired.|||That's another silly misconce...</td>\n",
       "      <td>youre fired thats another silly misconception ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   type                                              posts  \\\n",
       "0  INFJ  'http://www.youtube.com/watch?v=qsXHcwe3krw|||...   \n",
       "1  ENTP  'I'm finding the lack of me in these posts ver...   \n",
       "2  INTP  'Good one  _____   https://www.youtube.com/wat...   \n",
       "3  INTJ  'Dear INTP,   I enjoyed our conversation the o...   \n",
       "4  ENTJ  'You're fired.|||That's another silly misconce...   \n",
       "\n",
       "                                        cleaned_text  \n",
       "0  enfp and intj moments sportscenter not top ten...  \n",
       "1  im finding the lack of me in these posts very ...  \n",
       "2  good one  of course to which i say i know that...  \n",
       "3  dear intp i enjoyed our conversation the other...  \n",
       "4  youre fired thats another silly misconception ...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"/notebooks/cleaned_mbti_data.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "436b0c73-9112-433b-af3d-e47d941a2db8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:11.920069Z",
     "iopub.status.busy": "2025-02-21T08:18:11.919398Z",
     "iopub.status.idle": "2025-02-21T08:18:11.941266Z",
     "shell.execute_reply": "2025-02-21T08:18:11.940690Z",
     "shell.execute_reply.started": "2025-02-21T08:18:11.920046Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>posts</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>type_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>INFJ</td>\n",
       "      <td>'http://www.youtube.com/watch?v=qsXHcwe3krw|||...</td>\n",
       "      <td>enfp and intj moments sportscenter not top ten...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ENTP</td>\n",
       "      <td>'I'm finding the lack of me in these posts ver...</td>\n",
       "      <td>im finding the lack of me in these posts very ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>INTP</td>\n",
       "      <td>'Good one  _____   https://www.youtube.com/wat...</td>\n",
       "      <td>good one  of course to which i say i know that...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>INTJ</td>\n",
       "      <td>'Dear INTP,   I enjoyed our conversation the o...</td>\n",
       "      <td>dear intp i enjoyed our conversation the other...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ENTJ</td>\n",
       "      <td>'You're fired.|||That's another silly misconce...</td>\n",
       "      <td>youre fired thats another silly misconception ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   type                                              posts  \\\n",
       "0  INFJ  'http://www.youtube.com/watch?v=qsXHcwe3krw|||...   \n",
       "1  ENTP  'I'm finding the lack of me in these posts ver...   \n",
       "2  INTP  'Good one  _____   https://www.youtube.com/wat...   \n",
       "3  INTJ  'Dear INTP,   I enjoyed our conversation the o...   \n",
       "4  ENTJ  'You're fired.|||That's another silly misconce...   \n",
       "\n",
       "                                        cleaned_text  type_index  \n",
       "0  enfp and intj moments sportscenter not top ten...           8  \n",
       "1  im finding the lack of me in these posts very ...           3  \n",
       "2  good one  of course to which i say i know that...          11  \n",
       "3  dear intp i enjoyed our conversation the other...          10  \n",
       "4  youre fired thats another silly misconception ...           2  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "types = np.unique(data.type.values)\n",
    "\n",
    "def get_type_index(string):\n",
    "    return list(types).index(string)\n",
    "\n",
    "data['type_index'] = data['type'].apply(get_type_index)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce8bf745-8848-4bee-8b26-4d30c1b28db4",
   "metadata": {},
   "source": [
    "## Split data into train test and val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "be47e409-773d-4df0-bde0-37553ca88375",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:14.968926Z",
     "iopub.status.busy": "2025-02-21T08:18:14.968409Z",
     "iopub.status.idle": "2025-02-21T08:18:14.976955Z",
     "shell.execute_reply": "2025-02-21T08:18:14.976562Z",
     "shell.execute_reply.started": "2025-02-21T08:18:14.968904Z"
    }
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(data)\n",
    "train, val = train_test_split(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3969d6d5-39a7-4022-9069-0a55ecdf24f0",
   "metadata": {},
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "39b7e88b-f1c4-4bc2-84f9-8fad24656d6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:18.442668Z",
     "iopub.status.busy": "2025-02-21T08:18:18.442128Z",
     "iopub.status.idle": "2025-02-21T08:18:22.684381Z",
     "shell.execute_reply": "2025-02-21T08:18:22.683765Z",
     "shell.execute_reply.started": "2025-02-21T08:18:18.442668Z"
    }
   },
   "outputs": [],
   "source": [
    "vocab_size = 10000\n",
    "trunc_type = \"post\"\n",
    "pad_type = \"post\"\n",
    "oov_tok = \"<OOV>\"\n",
    "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n",
    "tokenizer.fit_on_texts(data.cleaned_text.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6ae2e0ae-d46e-46b0-b759-9e423b382b26",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:26.095938Z",
     "iopub.status.busy": "2025-02-21T08:18:26.095490Z",
     "iopub.status.idle": "2025-02-21T08:18:28.252380Z",
     "shell.execute_reply": "2025-02-21T08:18:28.251939Z",
     "shell.execute_reply.started": "2025-02-21T08:18:26.095919Z"
    }
   },
   "outputs": [],
   "source": [
    "maxlen = 1500\n",
    "train_sequences = tokenizer.texts_to_sequences(train.cleaned_text.values)\n",
    "train_padded = pad_sequences(train_sequences, maxlen = maxlen, truncating = trunc_type, padding = pad_type)\n",
    "\n",
    "val_sequences = tokenizer.texts_to_sequences(val.cleaned_text.values)\n",
    "val_padded = pad_sequences(val_sequences, maxlen = maxlen, truncating = trunc_type, padding = pad_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1ee100c5-6f64-4e10-abe9-4bae08607474",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:30.309666Z",
     "iopub.status.busy": "2025-02-21T08:18:30.309315Z",
     "iopub.status.idle": "2025-02-21T08:18:30.313700Z",
     "shell.execute_reply": "2025-02-21T08:18:30.313298Z",
     "shell.execute_reply.started": "2025-02-21T08:18:30.309647Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  19, 3207,    6, ...,    0,    0,    0],\n",
       "       [   1,   10,   40, ...,   50,   16, 7146],\n",
       "       [ 202,    2,   75, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [  19, 1426,    9, ...,    0,    0,    0],\n",
       "       [ 117,  228,   52, ...,   62,   76,  302],\n",
       "       [   1,   23,   26, ...,    1,    2,  633]], dtype=int32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_padded"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29262900-f105-481b-be7c-05a91ff1fcd5",
   "metadata": {},
   "source": [
    "## Convert labels to categorical values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a83d1cdb-15b1-4c66-b042-623b509d87da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:32.642633Z",
     "iopub.status.busy": "2025-02-21T08:18:32.642023Z",
     "iopub.status.idle": "2025-02-21T08:18:32.645832Z",
     "shell.execute_reply": "2025-02-21T08:18:32.645433Z",
     "shell.execute_reply.started": "2025-02-21T08:18:32.642611Z"
    }
   },
   "outputs": [],
   "source": [
    "one_hot_labels = tf.keras.utils.to_categorical(train.type_index.values, num_classes=16)\n",
    "val_labels= tf.keras.utils.to_categorical(val.type_index.values, num_classes=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6e18c71-2bc2-41a3-97e0-fb02a7335b9c",
   "metadata": {},
   "source": [
    "## Define the models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98bc5eef-a928-4534-bde0-acecad1bfec9",
   "metadata": {},
   "source": [
    "LSTM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d67e09cf-e9ef-458c-b684-6d611dbc7a09",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:34.858241Z",
     "iopub.status.busy": "2025-02-21T08:18:34.857678Z",
     "iopub.status.idle": "2025-02-21T08:18:34.861041Z",
     "shell.execute_reply": "2025-02-21T08:18:34.860684Z",
     "shell.execute_reply.started": "2025-02-21T08:18:34.858218Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_lstm_model():\n",
    "    model = Sequential([\n",
    "        Embedding(vocab_size, 256, input_length=maxlen),\n",
    "        SpatialDropout1D(0.2),\n",
    "        LSTM(100, dropout=0.2, recurrent_dropout=0.2),\n",
    "        Dense(16, activation=\"softmax\")\n",
    "    ])\n",
    "    model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2174a6dc-4ac5-49be-9c35-0665af64fb7c",
   "metadata": {},
   "source": [
    "Bi - Directional LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4dc28285-5549-4b7b-ae59-5b464526c87f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:37.369251Z",
     "iopub.status.busy": "2025-02-21T08:18:37.368629Z",
     "iopub.status.idle": "2025-02-21T08:18:37.372030Z",
     "shell.execute_reply": "2025-02-21T08:18:37.371685Z",
     "shell.execute_reply.started": "2025-02-21T08:18:37.369229Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_bilstm_model():\n",
    "    model = Sequential([\n",
    "        Embedding(vocab_size, 256, input_length=maxlen),\n",
    "        Bidirectional(LSTM(100, return_sequences=True)),\n",
    "        Dropout(0.3),\n",
    "        Bidirectional(LSTM(50)),\n",
    "        Dense(16, activation=\"softmax\")\n",
    "    ])\n",
    "    model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51b1d7b4-dac6-4bb4-b8d6-3cd17a606f2f",
   "metadata": {},
   "source": [
    "BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "628fd5c9-c7de-471c-922f-ff318d434bf9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:18:40.228835Z",
     "iopub.status.busy": "2025-02-21T08:18:40.228302Z",
     "iopub.status.idle": "2025-02-21T08:18:40.232698Z",
     "shell.execute_reply": "2025-02-21T08:18:40.232342Z",
     "shell.execute_reply.started": "2025-02-21T08:18:40.228814Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_bert_model():\n",
    "    tokenizer = transformers.AutoTokenizer.from_pretrained('bert-base-uncased')\n",
    "    \n",
    "    train_input_ids = [tokenizer.encode(str(i), max_length = maxlen , pad_to_max_length = True) for i in train.cleaned_text.values]\n",
    "    val_input_ids = [tokenizer.encode(str(i), max_length = maxlen , pad_to_max_length = True) for i in val.cleaned_text.values]\n",
    "\n",
    "    input_word_ids = tf.keras.layers.Input(shape=(maxlen,), dtype=tf.int32,\n",
    "                                           name=\"input_word_ids\")\n",
    "    bert_layer = transformers.TFBertModel.from_pretrained('bert-base-uncased')\n",
    "    bert_outputs = bert_layer(input_word_ids)[0]\n",
    "    pred = tf.keras.layers.Dense(16, activation='softmax')(bert_outputs[:,0,:])\n",
    "    \n",
    "    model = tf.keras.models.Model(inputs=input_word_ids, outputs=pred)\n",
    "    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(\n",
    "    learning_rate=0.00001), metrics=['accuracy'])\n",
    "    \n",
    "    return model, train_input_ids, val_input_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d2b90cf-18ed-499b-9cb5-2d065ca1ea1b",
   "metadata": {},
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dda9644-8433-442e-a10b-ab8b910b6666",
   "metadata": {},
   "source": [
    "LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "82666c4a-d571-4758-99b4-68361a66cc15",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:19:01.109484Z",
     "iopub.status.busy": "2025-02-21T08:19:01.108745Z",
     "iopub.status.idle": "2025-02-21T08:37:46.686360Z",
     "shell.execute_reply": "2025-02-21T08:37:46.685855Z",
     "shell.execute_reply.started": "2025-02-21T08:19:01.109464Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "Epoch 1/10\n",
      "69/69 [==============================] - 225s 3s/step - loss: 2.3441 - accuracy: 0.2006 - val_loss: 2.2344 - val_accuracy: 0.2131\n",
      "Epoch 2/10\n",
      "69/69 [==============================] - 225s 3s/step - loss: 2.2801 - accuracy: 0.2129 - val_loss: 2.2268 - val_accuracy: 0.2295\n",
      "Epoch 3/10\n",
      "69/69 [==============================] - 226s 3s/step - loss: 2.2712 - accuracy: 0.2079 - val_loss: 2.2413 - val_accuracy: 0.1639\n",
      "Epoch 4/10\n",
      "69/69 [==============================] - 223s 3s/step - loss: 2.2470 - accuracy: 0.2325 - val_loss: 2.2462 - val_accuracy: 0.2275\n",
      "Epoch 5/10\n",
      "69/69 [==============================] - 227s 3s/step - loss: 2.1939 - accuracy: 0.2560 - val_loss: 2.2407 - val_accuracy: 0.2254\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7ff224eae190>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_model = create_lstm_model()\n",
    "lstm_model.fit(train_padded, one_hot_labels, epochs=10, batch_size=64,\n",
    "                validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeee45f4-4001-48d4-a8aa-b70a4f917351",
   "metadata": {},
   "source": [
    "Bi - Directional LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "21a4d3e1-efcb-4310-96da-fc142d9fe5a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:40:37.851458Z",
     "iopub.status.busy": "2025-02-21T08:40:37.850916Z",
     "iopub.status.idle": "2025-02-21T08:42:38.984101Z",
     "shell.execute_reply": "2025-02-21T08:42:38.983596Z",
     "shell.execute_reply.started": "2025-02-21T08:40:37.851438Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "153/153 [==============================] - 36s 208ms/step - loss: 2.3097 - accuracy: 0.2052 - val_loss: 2.2908 - val_accuracy: 0.2188\n",
      "Epoch 2/20\n",
      "153/153 [==============================] - 28s 184ms/step - loss: 2.1742 - accuracy: 0.2605 - val_loss: 2.3197 - val_accuracy: 0.2250\n",
      "Epoch 3/20\n",
      "153/153 [==============================] - 28s 183ms/step - loss: 1.7281 - accuracy: 0.4511 - val_loss: 2.5654 - val_accuracy: 0.2127\n",
      "Epoch 4/20\n",
      "153/153 [==============================] - 28s 184ms/step - loss: 1.1565 - accuracy: 0.6471 - val_loss: 3.0174 - val_accuracy: 0.2004\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7ff1f8245750>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bilstm_model = create_bilstm_model()\n",
    "bilstm_model.fit(train_padded, one_hot_labels, epochs =20, verbose = 1, \n",
    "          validation_data = (val_padded, val_labels),  callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07c04899-9925-480b-9f4c-11d204648db3",
   "metadata": {},
   "source": [
    "BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6e452571-9ce9-4dc8-9277-5b36023d2551",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T08:50:18.691410Z",
     "iopub.status.busy": "2025-02-21T08:50:18.691175Z",
     "iopub.status.idle": "2025-02-21T10:59:10.767357Z",
     "shell.execute_reply": "2025-02-21T10:59:10.766683Z",
     "shell.execute_reply.started": "2025-02-21T08:50:18.691391Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
      "610/610 [==============================] - 803s 1s/step - loss: 2.0424 - accuracy: 0.3329 - val_loss: 1.4103 - val_accuracy: 0.5778\n",
      "Epoch 2/20\n",
      "610/610 [==============================] - 767s 1s/step - loss: 1.2794 - accuracy: 0.6175 - val_loss: 1.2560 - val_accuracy: 0.6300\n",
      "Epoch 3/20\n",
      "610/610 [==============================] - 767s 1s/step - loss: 1.0885 - accuracy: 0.6702 - val_loss: 1.2387 - val_accuracy: 0.6325\n",
      "Epoch 4/20\n",
      "610/610 [==============================] - 767s 1s/step - loss: 0.9342 - accuracy: 0.7159 - val_loss: 1.1834 - val_accuracy: 0.6785\n",
      "Epoch 5/20\n",
      "610/610 [==============================] - 767s 1s/step - loss: 0.7711 - accuracy: 0.7674 - val_loss: 1.0631 - val_accuracy: 0.7191\n",
      "Epoch 6/20\n",
      "610/610 [==============================] - 768s 1s/step - loss: 0.5852 - accuracy: 0.8284 - val_loss: 1.1146 - val_accuracy: 0.7025\n",
      "Epoch 7/20\n",
      "610/610 [==============================] - 766s 1s/step - loss: 0.4213 - accuracy: 0.8739 - val_loss: 1.2318 - val_accuracy: 0.6859\n",
      "Epoch 8/20\n",
      "610/610 [==============================] - 767s 1s/step - loss: 0.3004 - accuracy: 0.9123 - val_loss: 1.2625 - val_accuracy: 0.6982\n",
      "Epoch 9/20\n",
      "610/610 [==============================] - 767s 1s/step - loss: 0.2052 - accuracy: 0.9408 - val_loss: 1.4626 - val_accuracy: 0.6853\n",
      "Epoch 10/20\n",
      "610/610 [==============================] - 771s 1s/step - loss: 0.1272 - accuracy: 0.9633 - val_loss: 1.5342 - val_accuracy: 0.6884\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7fef858cb510>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_model, train_input_ids, val_input_ids =  create_bert_model()\n",
    "bert_model.fit(np.array(train_input_ids), one_hot_labels,validation_data = (np.array(val_input_ids), val_labels),\n",
    "          verbose = 1, epochs = 20, batch_size = 8,  callbacks = [tf.keras.callbacks.EarlyStopping(patience = 5)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42632f95-f20e-4c93-ab6a-9de2e7354aad",
   "metadata": {},
   "source": [
    "Save the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7c42e830-7964-4602-8450-035aff094544",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-21T11:14:52.465506Z",
     "iopub.status.busy": "2025-02-21T11:14:52.465273Z",
     "iopub.status.idle": "2025-02-21T11:14:54.217833Z",
     "shell.execute_reply": "2025-02-21T11:14:54.217284Z",
     "shell.execute_reply.started": "2025-02-21T11:14:52.465488Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
      "  saving_api.save_model(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All models trained & saved successfully!\n"
     ]
    }
   ],
   "source": [
    "lstm_model.save(\"models/lstm_model.h5\")\n",
    "bilstm_model.save(\"models/bilstm_model.h5\")\n",
    "bert_model.save(\"models/bert_model.h5\")\n",
    "\n",
    "print(\"All models trained & saved successfully!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}