CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/pandas/compat/__init__.py:117: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating generators...\n",
      "Creating model...\n",
      "WARNING:tensorflow:From /home/mlsg/mlsg-venv/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "Starting training...\n",
      "WARNING:tensorflow:From /home/mlsg/mlsg-venv/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/mlsg/mlsg-venv/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Epoch 1/10\n",
      "214/214 [==============================] - 1067s 5s/step - loss: 115.1066 - accuracy: 0.6384 - val_loss: 1.8747 - val_accuracy: 0.5505\n",
      "Epoch 2/10\n",
      "214/214 [==============================] - 995s 5s/step - loss: 0.9897 - accuracy: 0.8707 - val_loss: 1.4426 - val_accuracy: 0.5502\n",
      "Epoch 3/10\n",
      "214/214 [==============================] - 1002s 5s/step - loss: 0.5856 - accuracy: 0.9020 - val_loss: 1.3991 - val_accuracy: 0.5950\n",
      "Epoch 4/10\n",
      "214/214 [==============================] - 996s 5s/step - loss: 0.4413 - accuracy: 0.9135 - val_loss: 1.6516 - val_accuracy: 0.5671\n",
      "Epoch 5/10\n",
      "214/214 [==============================] - 994s 5s/step - loss: 0.3701 - accuracy: 0.9201 - val_loss: 1.8849 - val_accuracy: 0.6000\n",
      "Epoch 6/10\n",
      "214/214 [==============================] - 1001s 5s/step - loss: 0.3255 - accuracy: 0.9279 - val_loss: 1.7380 - val_accuracy: 0.6029\n",
      "Epoch 7/10\n",
      "214/214 [==============================] - 999s 5s/step - loss: 0.2918 - accuracy: 0.9335 - val_loss: 1.9303 - val_accuracy: 0.6102\n",
      "Epoch 8/10\n",
      "214/214 [==============================] - 1004s 5s/step - loss: 0.2705 - accuracy: 0.9345 - val_loss: 1.9497 - val_accuracy: 0.6024\n",
      "Epoch 9/10\n",
      "214/214 [==============================] - 994s 5s/step - loss: 0.2541 - accuracy: 0.9387 - val_loss: 2.1082 - val_accuracy: 0.6062\n",
      "Epoch 10/10\n",
      " 59/214 [=======>......................] - ETA: 8:57 - loss: 0.2384 - accuracy: 0.9424"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# Location of frames\n",
    "training_image_src = '/mnt/disks/a/frames'\n",
    "validation_image_src = '/mnt/disks/b/frames'\n",
    "\n",
    "\"\"\"\n",
    "The directory is divided into folders by the candidate number\n",
    "Within each candidate's folder, the frames are further divided by the label\n",
    "Naming convention of the frames is as follows: [candidate number]_[frame_number]_[label]\n",
    "Single digit candidate numbers are padded with a 0\n",
    "Frame numbers are consecutive and not padded\n",
    "Label can be 0, 5 or 10\n",
    "\"\"\"\n",
    "\n",
    "# This function helps to extract data and labels and return it as a Numpy array from a given image file\n",
    "def extract_data_and_label(image_path):\n",
    "    # We use opencv to read the images as grayscale, this will give us the 2d vector of pixels\n",
    "    # Note that it returns a numpy array and not a Python list, but Keras uses Numpy arrays anyway\n",
    "    image = cv2.imread(image_path, cv2.cv2.IMREAD_GRAYSCALE)\n",
    "    # Because some of the images are corrupt, we got to do this\n",
    "    if image is None or image.data is None or image.size == 0:\n",
    "        return None, None\n",
    "\n",
    "    # Scale the images to a fixed size, second argument is the target dimension, chose an arbitrary\n",
    "    # value for now, (100, 100). Additional arguments can be provided to fine-tune the scaling.\n",
    "    image = cv2.resize(image, (100, 100))\n",
    "    image = image / 255\n",
    "\n",
    "    \"\"\"\n",
    "    !!! Should we extract only the faces? By right CNN is supposed to be able to pick out key features\n",
    "    on its own, but this could possibly make it more effective. This can be done using opencv\n",
    "    \"\"\"\n",
    "\n",
    "    # Next is to extract the labels for each image, in our case, it is just the last portion of the filename\n",
    "    file_name = os.path.basename(image_path)\n",
    "    label = int(os.path.splitext(file_name)[0].split('_')[2])\n",
    "    # Convert to 0, 1 - we are only using images with labels 0 and 10 now\n",
    "    label = 0 if label == 0 else 1\n",
    "    \n",
    "    return image, label\n",
    "\n",
    "# Time to actually train the model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, Flatten, MaxPooling2D\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import model_from_json\n",
    "from keras.regularizers import l1\n",
    "from keras.losses import BinaryCrossentropy\n",
    "\n",
    "\"\"\"\n",
    "Gonna have to do the procesing in batches because the images are too big to fit all on the ram at the same time.\n",
    "To do so, we define a generator function that will help pull data in batches from the disks.\n",
    "\n",
    "https://mc.ai/train-keras-model-with-large-dataset-batch-training/\n",
    "\"\"\"\n",
    "def batch_generator(files, batch_size):\n",
    "    counter = 0\n",
    "    while True:\n",
    "        pixels = []\n",
    "        labels = []\n",
    "\n",
    "        # print('Generating batch...')\n",
    "        while len(pixels) < batch_size:\n",
    "            filename = files[counter]\n",
    "            data, label = extract_data_and_label(filename)\n",
    "            \n",
    "            if data is None and label is None:\n",
    "                counter = (counter + 1) % len(files)\n",
    "                continue\n",
    "\n",
    "            counter = (counter + 1) % len(files)\n",
    "            pixels.append(data)\n",
    "            labels.append(label)\n",
    "\n",
    "        pixels = np.array(pixels)\n",
    "        labels = np.array(labels)\n",
    "\n",
    "        \"\"\"\n",
    "        Gotta reformat the data (once again) to a format that the Conv2D layer accepts. Conv2D layer\n",
    "        is just the convulutional layer provided by keras.\n",
    "\n",
    "        The target format is (w, x, y, z) where w is the number of total images, x and y is the shape of each image\n",
    "        and z is 1 which signifies that the images are grayscale\n",
    "        \"\"\"\n",
    "        pixels = pixels.reshape(batch_size, 100, 100, 1)\n",
    "\n",
    "        \"\"\"\n",
    "        We one-hot-encode our labels to create 3 cateogories, 0 being mapped awake, 5 being mapped to normal and 10 being\n",
    "        mapped to sleepy\n",
    "\n",
    "        !!! Perhaps there can be a better way of encoding the output data? Will this method result in a loss of ordinality?\n",
    "        \"\"\"\n",
    "        labels = to_categorical(labels, num_classes=2)\n",
    "        # yield is a Python thing for generators\n",
    "        yield pixels, labels\n",
    "    \n",
    "\n",
    "# Let's instantiate our generators for the training and validation set\n",
    "print('Creating generators...')\n",
    "training_files = []\n",
    "for root, dirs, files in os.walk(training_image_src):\n",
    "    for file in files[:300]:\n",
    "        file_path = os.path.join(root, file)\n",
    "        training_files.append(file_path)      \n",
    "\n",
    "training_files = list(filter(lambda x: '_5.jpg' not in x, training_files))\n",
    "# for f in training_files:\n",
    "#     print(f)\n",
    "\n",
    "\n",
    "validation_files = []\n",
    "for root, dirs, files in os.walk(validation_image_src):\n",
    "    for file in files[:100]:\n",
    "        file_path = os.path.join(root, file)\n",
    "        validation_files.append(file_path)\n",
    "        \n",
    "\n",
    "validation_files = list(filter(lambda x: '_5.jpg' not in x, validation_files))\n",
    "\n",
    "\n",
    "\n",
    "# for f in validation_files:\n",
    "#     print(f)\n",
    "\n",
    "# # Let's sort the files for the heck of it\n",
    "# from functools import cmp_to_key\n",
    "# def compare(a, b):\n",
    "#     candidate_a, frame_a, label_a = os.path.splitext(os.path.basename(a))[0].split('_')\n",
    "#     candidate_b, frame_b, label_b = os.path.splitext(os.path.basename(b))[0].split('_')\n",
    "#     if candidate_a != candidate_b:\n",
    "#         return int(candidate_a) - int(candidate_b)\n",
    "#     elif label_a != label_b:\n",
    "#         return int(label_a) - int(label_b)\n",
    "#     else:\n",
    "#         return int(frame_a) - int(frame_b)\n",
    "\n",
    "# training_files = sorted(training_files, key=cmp_to_key(compare))\n",
    "# validation_files = sorted(validation_files, key=cmp_to_key(compare))\n",
    "\n",
    "\"\"\"\n",
    "On second thoughts, seems more meaningful for the model to receive data unordered, if not at periods where it\n",
    "keeps receiving the same labelled data, it might not be learning much. We can save the ordering if we wish to do lstm\n",
    "\"\"\"\n",
    "\n",
    "import random\n",
    "\n",
    "training_sample_size = len(training_files)\n",
    "validation_sample_size = len(validation_files)\n",
    "\n",
    "# training_sample_size = 25000\n",
    "# validation_sample_size = 12000\n",
    "\n",
    "training_files = random.sample(training_files, training_sample_size)\n",
    "validation_files = random.sample(validation_files, validation_sample_size)\n",
    "\n",
    "batch_size = 100\n",
    "training_generator = batch_generator(training_files, batch_size)\n",
    "validation_generator = batch_generator(validation_files, batch_size)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "Now we create our model. Keras allows you to build models in a sequential manner or a functional manner. Sequential\n",
    "is easier to understand for me. Its only a syntax difference.\n",
    "\"\"\"\n",
    "print('Creating model...')\n",
    "model = Sequential()\n",
    "\n",
    "\"\"\"\n",
    "The model is essentially what we learnt in the course, a series of layers of neurons and in this case, convulutions.\n",
    "\n",
    "We can tweak the attributes of each layer, such as the size, activation function, etc. This is what they mean by\n",
    "playing with the parameters.\n",
    "\n",
    "I believe what is passed between layers are just Numpy arrays, so what happens is that a layer will take in a Numpy\n",
    "array, transform it using its neurons/convulutions and return the resulting Numpy array.\n",
    "\n",
    "Note that the input shape and output shape of each layer must match.\n",
    "\"\"\"\n",
    "\n",
    "# model.add(Conv2D(128, kernel_size=3, activation='relu', input_shape=(100,100,1),  activity_regularizer=l1(0.001)))\n",
    "# model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(100,100,1),  activity_regularizer=l1(0.001)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(32, kernel_size=3, activation='relu', activity_regularizer=l1(0.001)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# Flattens the 2D data into a 1D Numpy array\n",
    "model.add(Flatten())\n",
    "# Dense is your standard MLP layer\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "\n",
    "\"\"\"\n",
    "Conv2D arguments explained:\n",
    "\n",
    "first argument: number of filters that the layer will learn, each filter will have its own kernel to be convolved\n",
    "                with the input, each filter will result in a different 2D activation map. All of these maps \n",
    "                are passed to the subsequent layer in a NumpyArray\n",
    "kernel size   : size of the kernel to convolve the input with. Generally smaller kernel means more processing time\n",
    "                but possibly identify more features (must be odd)\n",
    "strides       : default=(1, 1) - determines how the kernel is moved along the input matrix in the x and y axis\n",
    "padding       : default=valid - no padding is added to the activation map\n",
    "activation    : the activation function to be applied after ecah kernel convulution to introduce non-linearity\n",
    "input_shape   : required if the layer is used as the first layer of the model, if not it is inferred\n",
    "\n",
    "https://www.pyimagesearch.com/2018/12/31/keras-conv2d-and-convolutional-layers/\n",
    "\"\"\"\n",
    "\n",
    "\"\"\"\n",
    "Compiling the model - optimizer is the method used to minimize the loss function e.g. stochastic gradient descent\n",
    "'adam' seems to be an extension of SGD that is less computationally expensive and more appropriate for images\n",
    "\n",
    "metrics is just a function used to judge the performance of the model, this is not used in training, and only serves\n",
    "tool to analyze the effective of the model\n",
    "\n",
    "https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/\n",
    "\"\"\"\n",
    "model.compile(optimizer='adam', loss=BinaryCrossentropy(), metrics=['accuracy'])\n",
    "\n",
    "print('Starting training...')\n",
    "# Training the model only takes a simple function call\n",
    "# Epochs is the number of passes over the dataset we want for the training\n",
    "history = model.fit_generator(training_generator, validation_data=validation_generator, \n",
    "                              epochs=10, steps_per_epoch=np.ceil(len(training_files)/batch_size), \n",
    "                              validation_steps=np.ceil(len(validation_files)/batch_size), \n",
    "                              verbose=1, shuffle=True)\n",
    "\n",
    "\n",
    "# # Save model to json for future use\n",
    "# model_json = model.to_json()\n",
    "# with open(\"cnn.json\", \"w\") as json_file:\n",
    "#     json_file.write(model_json)\n",
    "# # Save weights for future use\n",
    "# model.save_weights(\"model.h5\")\n",
    "\n",
    "# You can save model and weights together\n",
    "model.save(\"cnn.h5\");\n",
    "\n",
    "# What follows is just a few library calls to plot the results throughout the course of the training\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Plot training & validation accuracy values\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('Model accuracy')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.savefig('cnn_accuracy.png')\n",
    "plt.show()\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.savefig('cnn_loss.png')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlsg-venv",
   "language": "python",
   "name": "mlsg-venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}