eye-blink-detection-3-ml-model-part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# EYE BLINK DETECTION:\n",
    "\n",
    "# 3) Machine Learning Model - part1\n",
    "\n",
    "For this part we will use [Eyeblink8](https://www.blinkingmatters.com/research) dataset for EAR values and annotations from the same dataset to construct the dataframe that will be used for classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import utility functions\n",
    "from utils_frame_based import *\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# define three constants.\n",
    "# You can later experiment with these constants by changing them to adaptive variables.\n",
    "EAR_THRESHOLD = 0.21 # eye aspect ratio to indicate blink\n",
    "EAR_CONSEC_FRAMES = 3 # number of consecutive frames the eye must be below the threshold\n",
    "SKIP_FIRST_FRAMES = 150 # how many frames we should skip at the beggining"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "See input folder structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['26122013_224532_cam.avi', '26122013_224532_cam.txt', '26122013_224532_cam.tag']\n"
     ]
    }
   ],
   "source": [
    "print(os.listdir('../input/blinkdata/eyeblink8/2'))"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Now we process 8 videos to get EAR values of frames."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/blinkdata/eyeblink8/10/27122013_153916_cam.avi\n",
      "Processing 27122013_153916_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/2/26122013_224532_cam.avi\n",
      "Processing 26122013_224532_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/1/26122013_223310_cam.avi\n",
      "Processing 26122013_223310_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/11/27122013_154548_cam.avi\n",
      "Processing 27122013_154548_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/3/26122013_230103_cam.avi\n",
      "Processing 26122013_230103_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/4/26122013_230654_cam.avi\n",
      "Processing 26122013_230654_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/8/27122013_151644_cam.avi\n",
      "Processing 27122013_151644_cam.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/eyeblink8/9/27122013_152435_cam.avi\n",
      "Processing 27122013_152435_cam.avi has done.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# create a folder named 'train'\n",
    "os.mkdir('./train')\n",
    "\n",
    "# read all videos\n",
    "directory = \"../input/blinkdata/eyeblink8\"\n",
    "subjects=os.listdir(directory)\n",
    "for subject in subjects:\n",
    "    video_names=os.listdir(directory+'/'+subject)\n",
    "    for video_name in video_names:\n",
    "        clean_name = os.path.splitext(video_name)[0]\n",
    "        extension = os.path.splitext(video_name)[1]\n",
    "        if extension=='.avi': \n",
    "            file_path = directory+'/'+subject+'/'+video_name\n",
    "            print(file_path)\n",
    "            frame_info_df, video_info_dict = process_video_v2(file_path, subject=subject, external_factors=None,facial_actions=clean_name, \\\n",
    "                                                            ear_th=EAR_THRESHOLD, consec_th=EAR_CONSEC_FRAMES, skip_n=SKIP_FIRST_FRAMES)\n",
    "            frame_info_df.to_pickle('./train/{}_{}_frame_info_df.pkl'.format(subject,clean_name))\n",
    "            video_info_dict.to_pickle('./train/{}_{}_video_info_df.pkl'.format(subject,clean_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Then we read annotations of 8 videos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define read_annotations_v2\n",
    "def read_annotations_v2(input_file, len_video):\n",
    "    # Read .tag file using readlines() \n",
    "    file1 = open(input_file) \n",
    "    Lines = file1.readlines() \n",
    "\n",
    "    # find \"#start\" line \n",
    "    start_line = 1\n",
    "    for line in Lines: \n",
    "        clean_line=line.strip()\n",
    "        if clean_line==\"#start\":\n",
    "            break\n",
    "        start_line += 1\n",
    "\n",
    "    # length of annotations\n",
    "    len_annot = len(Lines[start_line : -1]) # -1 since last line will be\"#end\"\n",
    "\n",
    "    blink_list = [0] * len_video\n",
    "    closeness_list = [0] * len_video\n",
    "\n",
    "    # convert tag file to readable format and build \"closeness_list\" and \"blink_list\"\n",
    "    for i in range(len_annot): \n",
    "        annotation=Lines[start_line+i].split(':')\n",
    "\n",
    "        if int(annotation[1]) > 0:\n",
    "            # it means a new blink\n",
    "            blink_frame = int(annotation[0])\n",
    "            blink_list[blink_frame] = 1\n",
    "\n",
    "        # if current annotation consist fully closed eyes, append it also to \"closeness_list\" \n",
    "        if annotation[3] == \"C\" and annotation[5] == \"C\":\n",
    "            closed_frame = int(annotation[0])\n",
    "            closeness_list[closed_frame] = 1\n",
    "\n",
    "        file1.close()\n",
    "\n",
    "    result_df = pd.DataFrame(list(zip(closeness_list, blink_list)), columns=['closeness_annot', 'blink_annot'])\n",
    "    return result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/blinkdata/eyeblink8/10/27122013_153916_cam.tag\n",
      "../input/blinkdata/eyeblink8/2/26122013_224532_cam.tag\n",
      "../input/blinkdata/eyeblink8/1/26122013_223310_cam.tag\n",
      "../input/blinkdata/eyeblink8/11/27122013_154548_cam.tag\n",
      "../input/blinkdata/eyeblink8/3/26122013_230103_cam.tag\n",
      "../input/blinkdata/eyeblink8/4/26122013_230654_cam.tag\n",
      "../input/blinkdata/eyeblink8/8/27122013_151644_cam.tag\n",
      "../input/blinkdata/eyeblink8/9/27122013_152435_cam.tag\n"
     ]
    }
   ],
   "source": [
    "# full path of a tag file by using read_annotations() utility function\n",
    "directory = \"../input/blinkdata/eyeblink8\"\n",
    "subjects=os.listdir(directory)\n",
    "for subject in subjects:\n",
    "    video_names=os.listdir(directory+'/'+subject)\n",
    "    for video_name in video_names:\n",
    "        clean_name = os.path.splitext(video_name)[0]\n",
    "        extension = os.path.splitext(video_name)[1]\n",
    "        if extension=='.tag': \n",
    "            file_path = directory+'/'+subject+'/'+video_name\n",
    "            print(file_path)\n",
    "            #length of video\n",
    "            frame_info_df = pd.read_pickle(\"./train/\" + subject + '_' + clean_name + \"_frame_info_df.pkl\")\n",
    "            len_video = len(frame_info_df)\n",
    "            # read tag file\n",
    "            annot_df = read_annotations_v2(file_path, len_video)\n",
    "            annot_df.to_pickle('./train/{}_{}_annotations.pkl'.format(subject,clean_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Merge relevant frame_info_df with annot file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_pickles(directory):\n",
    "    annots=[]\n",
    "    frame_infos=[]\n",
    "    video_infos=[]\n",
    "\n",
    "    files = os.listdir(directory)\n",
    "    for file in files:\n",
    "        clean_name = os.path.splitext(file)[0]\n",
    "        if clean_name.endswith('annotations'):\n",
    "            annots.append(file)\n",
    "        if clean_name.endswith('video_info_df'):\n",
    "            video_infos.append(file)\n",
    "        if clean_name.endswith('frame_info_df'):\n",
    "            frame_infos.append(file)\n",
    "\n",
    "    for file in annots:\n",
    "        clean_name = os.path.splitext(file)[0]\n",
    "        first_part = clean_name[:-12]\n",
    "\n",
    "        for file2 in frame_infos:\n",
    "            clean_name2 = os.path.splitext(file2)[0]\n",
    "            first_part2 = clean_name2[:-14]\n",
    "            if first_part == first_part2:\n",
    "                frame_info_df = pd.read_pickle(directory+'/'+file2)\n",
    "                annotation = pd.read_pickle(directory+'/'+file)\n",
    "                if len(frame_info_df) !=len(annotation):\n",
    "                    os.mkdir(directory+'/fix/')\n",
    "                    os.rename(directory+file, directory+'/fix/'+file)\n",
    "                    os.rename(directory+file2, directory+'/fix/'+file2)\n",
    "                    print(file2, len(frame_info_df))\n",
    "                    print(file, len(annotation))\n",
    "                else: \n",
    "                    result=pd.concat([frame_info_df,annotation], axis=1)\n",
    "                    result.to_pickle(directory+'/'+first_part+'_merged_df.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge \"*_frame_info_df.pkl\" with \"*_annotations.pkl\" by using the function above\n",
    "merge_pickles(\"./train\")"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Concatenate pickle files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append all of pickles ending particular string (i.e. \"merged_df\") in a directory\n",
    "def concat_pickles(directory, ending, output_name):\n",
    "    pickles = os.listdir(directory)\n",
    "    pickle_list=[]\n",
    "\n",
    "    for pickle_file in pickles:\n",
    "        clean_name = os.path.splitext(pickle_file)[0]\n",
    "        if clean_name.endswith(ending):\n",
    "            pickle = pd.read_pickle(directory+'/'+pickle_file)\n",
    "            pickle_list.append(pickle)\n",
    "\n",
    "    result = pd.concat(pickle_list)\n",
    "    result.reset_index(inplace=True, drop=True)\n",
    "    result.to_pickle(directory+'/'+ output_name + '.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append all of pickles ending with \"merged_df\" in ./train\n",
    "concat_pickles(\"./train\",\"merged_df\",\"training_set\")"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "process test set with the same steps above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/blinkdata/talkingFace/talking.avi\n",
      "Processing talking.avi has done.\n",
      "\n",
      "\n",
      "../input/blinkdata/talkingFace/talking.tag\n"
     ]
    }
   ],
   "source": [
    "# create a folder named 'test'\n",
    "os.mkdir('./test')\n",
    "\n",
    "# read all videos\n",
    "directory = \"../input/blinkdata/talkingFace\"\n",
    "files = os.listdir(directory)\n",
    "for file in files:\n",
    "    clean_name = os.path.splitext(file)[0]\n",
    "    extension = os.path.splitext(file)[1]\n",
    "    if extension=='.avi': \n",
    "        file_path = directory+'/'+ file\n",
    "        print(file_path)\n",
    "        frame_info_df, video_info_dict = process_video_v2(file_path, subject='talkingFace', external_factors=None,facial_actions=clean_name, \\\n",
    "                                                        ear_th=EAR_THRESHOLD, consec_th=EAR_CONSEC_FRAMES, skip_n=SKIP_FIRST_FRAMES)\n",
    "        frame_info_df.to_pickle('./test/{}_frame_info_df.pkl'.format(clean_name))\n",
    "        video_info_dict.to_pickle('./train/{}_video_info_df.pkl'.format(clean_name))\n",
    "\n",
    "# read tag file\n",
    "for file in files:\n",
    "    clean_name = os.path.splitext(file)[0]\n",
    "    extension = os.path.splitext(file)[1]\n",
    "    if extension=='.tag': \n",
    "        file_path = directory+'/'+ file\n",
    "        print(file_path)\n",
    "        #length of video\n",
    "        frame_info_df = pd.read_pickle('./test/{}_frame_info_df.pkl'.format(clean_name))\n",
    "        len_video = len(frame_info_df)\n",
    "        # read tag file\n",
    "        annot_df = read_annotations_v2(file_path, len_video)\n",
    "        annot_df.to_pickle('./test/{}_annotations.pkl'.format(clean_name))\n",
    "\n",
    "# merge annotations and frame_info_df\n",
    "merge_pickles(\"./test\")\n",
    "\n",
    "# append all of pickles ending with \"merged_df\" in ./test\n",
    "concat_pickles(\"./test\",\"merged_df\",\"test_set\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}