fer2013.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import cv2\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization\n",
    "from keras.losses import categorical_crossentropy\n",
    "from keras.optimizers import Adam\n",
    "from keras.regularizers import l2\n",
    "from keras.callbacks import ReduceLROnPlateau, TensorBoard, EarlyStopping, ModelCheckpoint\n",
    "from keras.models import load_model\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "from sklearn.metrics import classification_report\n",
    "import seaborn as sb\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading dataset and removing emotions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"../input/fer2013/fer2013.csv\")\n",
    "data = data.drop('Usage', axis=1)\n",
    "data = data[data['emotion'] != 1]\n",
    "\n",
    "data.loc[data.emotion == 2, 'emotion'] = 1\n",
    "data.loc[data.emotion == 3, 'emotion'] = 2\n",
    "data.loc[data.emotion == 4, 'emotion'] = 3\n",
    "data.loc[data.emotion == 5, 'emotion'] = 4\n",
    "data.loc[data.emotion == 6, 'emotion'] = 5\n",
    "data.loc[data.emotion == 7, 'emotion'] = 6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining values reused throughout the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_labels = 6\n",
    "batch_size = 64\n",
    "epochs = 100\n",
    "width, height = 48, 48"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting output path and name of model file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASEPATH = '../output/kaggle/working'\n",
    "MODELPATH = 'model_fer2013.h5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing images for analysis (includes normalization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "faces = []\n",
    "emotions = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pixels = data['pixels'].tolist()\n",
    "faces = []\n",
    "for pixel_sequence in pixels:\n",
    "    face = [int(pixel) for pixel in pixel_sequence.split(' ')]\n",
    "    face = np.asarray(face).reshape(width, height)\n",
    "    faces.append(face.astype('float32')/ 255.0)\n",
    "\n",
    "faces = np.asarray(faces)\n",
    "faces = np.expand_dims(faces, -1)\n",
    "\n",
    "emotions = pd.get_dummies(data['emotion']).values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining labels which represent each emotion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = {0: 'Angry', 1: 'Fear', 2: 'Happy', 3: 'Sad', 4: 'Surprise', 5: 'Neutral'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Printing sample images from the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(3, 5, figsize=(25, 12))\n",
    "axs = axs.ravel()\n",
    "for i in range(15):\n",
    "    axs[i].imshow(faces[i][:,:,0], cmap='gray')\n",
    "    axs[i].set_title(str(labels[np.argmax(emotions[i])]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting in to train, test, and validation datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(faces, emotions, test_size=0.1, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=41)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential()\n",
    "model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(48,48,1)))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3), activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(tf.keras.layers.Dropout(0.5))\n",
    "model.add(tf.keras.layers.Conv2D(64, kernel_size=(3, 3), activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.Conv2D(64, kernel_size=(3, 3), activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.Dropout(0.5))\n",
    "model.add(tf.keras.layers.Conv2D(128, kernel_size=(3, 3), activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.Dropout(0.5))\n",
    "model.add(tf.keras.layers.Conv2D(128, kernel_size=(3, 3), activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(tf.keras.layers.Dropout(0.5))\n",
    "model.add(tf.keras.layers.Flatten())\n",
    "model.add(tf.keras.layers.Dense(128, activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.Dropout(0.5))\n",
    "model.add(tf.keras.layers.Dense(128, activation='relu'))\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "model.add(tf.keras.layers.Dropout(0.5))\n",
    "model.add(tf.keras.layers.Dense(num_labels, activation='softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Printing model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compiling the model and defining options for the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=categorical_crossentropy,\n",
    "              optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_reducer = ReduceLROnPlateau(monitor='val_loss', factor=0.9, patience=3, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stopper = EarlyStopping(monitor='val_loss', min_delta=0, patience=8, verbose=1, mode='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(MODELPATH, monitor='val_loss', verbose=1, save_best_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = TensorBoard(log_dir='./logs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(np.array(X_train), np.array(y_train),\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(np.array(X_test), np.array(y_test)),\n",
    "          shuffle=True, callbacks=[lr_reducer, tensorboard, early_stopper, checkpointer])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining function for plotting the accuracy and loss of the model over the epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Taken from https://www.kaggle.com/danbrice/keras-plot-history-full-report-and-grid-search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n",
    "    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n",
    "    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n",
    "    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n",
    "    \n",
    "    print(len(loss_list) , len(val_loss_list) , len(acc_list) , len(val_acc_list))\n",
    "    if len(loss_list) == 0:\n",
    "        print('Loss is missing in history')\n",
    "        return \n",
    "    \n",
    "    ## As loss always exists\n",
    "    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n",
    "    \n",
    "    ## Loss\n",
    "    plt.figure(1)\n",
    "    for l in loss_list:\n",
    "        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n",
    "    for l in val_loss_list:\n",
    "        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n",
    "    \n",
    "    plt.title('Loss')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    \n",
    "    ## Accuracy\n",
    "    plt.figure(2)\n",
    "    for l in acc_list:\n",
    "        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n",
    "    for l in val_acc_list:    \n",
    "        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n",
    "\n",
    "    plt.title('Accuracy')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calling the above defined plot_history function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the performance of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.evaluate(np.array(X_test), np.array(y_test), batch_size=batch_size)\n",
    "print(\"Loss: \" + str(scores[0]))\n",
    "print(\"Accuracy: \" + str(scores[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=[np.argmax(im) for im in model.predict(np.array(X_test))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotion_labels = [np.argmax(i) for i in np.array(y_test)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting the confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "em_names = ['Angry','Fear','Happy','Sad','Surprise','Neutral']\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "sb.heatmap(ax=ax, \n",
    "           data=pd.DataFrame(tf.math.confusion_matrix(emotion_labels, predictions, num_classes=6).numpy().astype(int), \n",
    "                     index=em_names, \n",
    "                     columns=em_names\n",
    "           ), \n",
    "           annot=True, \n",
    "           annot_kws={\"size\": 10},\n",
    "           fmt='g'\n",
    "          )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Printing the classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_test = np.argmax(y_test, axis=1) # Convert one-hot to index\n",
    "y_pred = np.argmax(model.predict(X_test), axis=-1)\n",
    "print(classification_report(Y_test, y_pred, target_names= em_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting sample images misclassified by the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = 0\n",
    "cols = 0\n",
    "disagree = []\n",
    "for i in range(len(emotion_labels)):\n",
    "    if emotion_labels[i] != predictions[i]:\n",
    "        disagree.append(i)\n",
    "        if len(disagree) == 10:\n",
    "            break\n",
    "\n",
    "fig, axs = plt.subplots(2, 5, figsize=(20,9))\n",
    "\n",
    "for i in range(10):\n",
    "    image=(np.array(list(X_test)[disagree[i]])/256)[:,:,0]\n",
    "    axs[cols, rows].imshow(image, cmap='gray')\n",
    "    axs[cols, rows].set_title('Predicted: ' + str(labels[predictions[disagree[i]]]) + '\\nGround Truth: ' + str(labels[emotion_labels[disagree[i]]]))\n",
    "    rows = rows+1\n",
    "    if rows == 5:\n",
    "        cols = cols + 1\n",
    "        rows = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating the performance of VGG16 on the same dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalizing and stacking the grayscale images into 3 bands as VGG requires 3 band images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.applications.vgg16 import VGG16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "faces = []\n",
    "for pixel_sequence in pixels:\n",
    "    face = [int(pixel) for pixel in pixel_sequence.split(' ')]\n",
    "    face = np.asarray(face).reshape(width, height)\n",
    "    face = cv2.merge((face,face,face))\n",
    "    faces.append(face.astype('float32')/255.0)\n",
    "\n",
    "faces = np.asarray(faces)\n",
    "faces = np.expand_dims(faces, -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting in to train, test, and validation datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(faces, emotions, test_size=0.1, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=41)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing the VGG16 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_base = VGG16(include_top=False,\n",
    "                      weights='imagenet',\n",
    "                      input_shape=(width, height, 3))\n",
    "for layer in model_base.layers[:-2]:\n",
    "    layer.trainable = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the model using the VGG16 as the base and adding the Softmax layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(model_base)\n",
    "model.add(Flatten())\n",
    "model.add(Dense(num_labels, activation='softmax'))\n",
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compiling, training, and evaluating the VGG16 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=categorical_crossentropy,\n",
    "              optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(np.array(X_train), np.array(y_train),\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(np.array(X_test), np.array(y_test)),\n",
    "          shuffle=True, callbacks=[lr_reducer, tensorboard, early_stopper, checkpointer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.evaluate(np.array(X_test), np.array(y_test), batch_size=batch_size)\n",
    "print(\"Loss: \" + str(scores[0]))\n",
    "print(\"Accuracy: \" + str(scores[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}