Action_Detection_KSL_Final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import and Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: opencv-python in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (4.8.0.76)\n",
      "Requirement already satisfied: mediapipe in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (0.10.0)\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (1.2.1)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (3.7.0)\n",
      "Requirement already satisfied: numpy>=1.19.3 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from opencv-python) (1.23.5)\n",
      "Requirement already satisfied: protobuf<4,>=3.11 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from mediapipe) (3.20.3)\n",
      "Requirement already satisfied: absl-py in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from mediapipe) (1.4.0)\n",
      "Requirement already satisfied: opencv-contrib-python in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from mediapipe) (4.7.0.72)\n",
      "Requirement already satisfied: attrs>=19.1.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from mediapipe) (22.1.0)\n",
      "Requirement already satisfied: flatbuffers>=2.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from mediapipe) (23.5.26)\n",
      "Requirement already satisfied: sounddevice>=0.4.4 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from mediapipe) (0.4.6)\n",
      "Requirement already satisfied: joblib>=1.1.1 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from scikit-learn) (1.1.1)\n",
      "Requirement already satisfied: scipy>=1.3.2 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from scikit-learn) (1.10.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from scikit-learn) (2.2.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (4.25.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (22.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (3.0.9)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (9.4.0)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (1.0.5)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from matplotlib) (1.4.4)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "Requirement already satisfied: CFFI>=1.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from sounddevice>=0.4.4->mediapipe) (1.15.1)\n",
      "Requirement already satisfied: pycparser in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from CFFI>=1.0->sounddevice>=0.4.4->mediapipe) (2.21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "!pip install opencv-python mediapipe scikit-learn matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "import time\n",
    "import mediapipe as mp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Keypoints using MP Holistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_holistic = mp.solutions.holistic # Holistic model\n",
    "mp_drawing = mp.solutions.drawing_utils # Drawing utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mediapipe_detection(image, model): \n",
    "    #image = feed frame\n",
    "    #model = Holistic model\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR -> RGB\n",
    "    image.flags.writeable = False                  # Image is no longer writeable\n",
    "    results = model.process(image)                 # Make prediction\n",
    "    image.flags.writeable = True                   # Image is now writeable \n",
    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB -> BGR\n",
    "    return image, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#connection_drawing_spec 커스텀 (optional)\n",
    "def draw_styled_landmarks(image, results):\n",
    "    #Draw pose connections\n",
    "    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS,\n",
    "                             mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4), \n",
    "                             mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2)\n",
    "                             ) \n",
    "    # Draw left hand connections\n",
    "    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
    "                             mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4), \n",
    "                             mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)\n",
    "                             ) \n",
    "    # Draw right hand connections  \n",
    "    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
    "                             mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4), \n",
    "                             mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)\n",
    "                             ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#------------------------------------custom function 실행------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_custom_pose_landmarks(image, results):\n",
    "    \n",
    "    pose_landmark_subset = landmark_pb2.NormalizedLandmarkList(\n",
    "      landmark = [\n",
    "          #eye keypoints\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.LEFT_EYE],#0 \n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.RIGHT_EYE],#1\n",
    "          #right arm keypoints\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.RIGHT_WRIST],#2\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.RIGHT_ELBOW],#3\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.RIGHT_SHOULDER],#4 \n",
    "          #left arm keypoints\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.LEFT_WRIST],#5 \n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.LEFT_ELBOW],#6\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.LEFT_SHOULDER],#7\n",
    "          #body keypoints\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.LEFT_HIP],#8\n",
    "          results.pose_landmarks.landmark[mp_pose.PoseLandmark.RIGHT_HIP]])#9\n",
    "    \n",
    "    #connection = [pose_landmark_subset index1, pose_landmark_subset index2]\n",
    "    pose_landmark_subset_connections = [\n",
    "        [2, 3], [3, 4], #right arm\n",
    "        [5, 6], [6, 7], #left arm\n",
    "        [4, 7], [4, 9], [8, 9], [7, 8]] #body\n",
    "\n",
    "    #Draw pose connections\n",
    "    mp_drawing.draw_landmarks(image, \n",
    "                             landmark_list = pose_landmark_subset, \n",
    "                             connections = pose_landmark_subset_connections,\n",
    "                             landmark_drawing_spec = mp_drawing.DrawingSpec(color=(98,129,205), thickness=2, circle_radius=2), \n",
    "                             connection_drawing_spec = mp_drawing.DrawingSpec(color=(122,160,255), thickness=2, circle_radius=2)\n",
    "                             ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_custom_hand_landmarks(image, results):\n",
    "    # Draw left hand connections\n",
    "    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
    "                             mp_drawing.DrawingSpec(color=(149,140,205), thickness=2, circle_radius=2), \n",
    "                             mp_drawing.DrawingSpec(color=(185,174,255), thickness=2, circle_radius=2)\n",
    "                             ) \n",
    "    # Draw right hand connections  \n",
    "    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, \n",
    "                             mp_drawing.DrawingSpec(color=(205,182,141), thickness=2, circle_radius=2), \n",
    "                             mp_drawing.DrawingSpec(color=(255,226,176), thickness=2, circle_radius=2)\n",
    "                             ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_custom_face_landmarks(image, results):\n",
    "    #face landmark subset\n",
    "    face_landmark_subset = landmark_pb2.NormalizedLandmarkList(\n",
    "          landmark = [\n",
    "              #right eyebrow keypoints\n",
    "              results.face_landmarks.landmark[46],#0 \n",
    "              results.face_landmarks.landmark[53],#1\n",
    "              results.face_landmarks.landmark[52],#2\n",
    "              results.face_landmarks.landmark[65],#3\n",
    "              results.face_landmarks.landmark[55],#4\n",
    "\n",
    "              #left eyebrow keypoints\n",
    "              results.face_landmarks.landmark[285],#5 \n",
    "              results.face_landmarks.landmark[295],#6\n",
    "              results.face_landmarks.landmark[282],#7\n",
    "              results.face_landmarks.landmark[283],#8\n",
    "              results.face_landmarks.landmark[276],#9\n",
    "\n",
    "              #lip keypoints\n",
    "              results.face_landmarks.landmark[61],#10 \n",
    "              results.face_landmarks.landmark[81],#11\n",
    "              results.face_landmarks.landmark[13],#12 \n",
    "              results.face_landmarks.landmark[311],#13\n",
    "              results.face_landmarks.landmark[291],#14\n",
    "              results.face_landmarks.landmark[402],#15 \n",
    "              results.face_landmarks.landmark[14],#16\n",
    "              results.face_landmarks.landmark[178]])#17\n",
    "    \n",
    "    #connection = {face_landmark_subset index1, face_landmark_subset index2}\n",
    "    face_landmark_subset_connections = [\n",
    "        [0, 1], [1, 2], [2, 3], [3, 4], #right eybrow\n",
    "        [5, 6], [6, 7], [7, 8], [8, 9], #left eyebrow\n",
    "        [10, 11],[11, 12], [12, 13], [13, 14], [14, 15], [15, 16], [16, 17], [17, 10]] #lip\n",
    "    \n",
    "    mp_drawing.draw_landmarks(image, \n",
    "                             face_landmark_subset, \n",
    "                             face_landmark_subset_connections,\n",
    "                             mp_drawing.DrawingSpec(color=(112,190,205), thickness=2, circle_radius=4), \n",
    "                             mp_drawing.DrawingSpec(color=(139,236,255), thickness=2, circle_radius=2)\n",
    "                             ) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Extract Keypoint Values\n",
    "results의 landmarks값 -> numpy array로 변환하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "poseIndex = np.array([22, 16, 14, 12, 24, 21, 15, 13, 11, 23])\n",
    "faceIndex = np.array([61, 291, 81, 13, 311, 178, 14, 402, 285, 295, 282, 283, 276, 46, 53, 52, 65, 55])\n",
    "\n",
    "def extract_keypoints_220(results):\n",
    "    # result의 landmarks의 모든 key point values -> 하나의 numpy array 로 flatten\n",
    "    #if landmarks has no value, fill numpy array with zero\n",
    "    \n",
    "    if results.pose_landmarks: #pose landmarks\n",
    "        #pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten()\n",
    "        i = 0\n",
    "        pose = np.array([])\n",
    "        for res in results.pose_landmarks.landmark:\n",
    "            if i in poseIndex:\n",
    "                pose = np.append(pose, np.array([res.x, res.y, res.z, res.visibility]))\n",
    "            i = i + 1\n",
    "    else:\n",
    "        pose = np.zeros(132) #33*4\n",
    "\n",
    "    if results.left_hand_landmarks: #left hand landmarks\n",
    "        lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        lh = np.zeros(63) #21*3\n",
    "\n",
    "    if results.right_hand_landmarks: #right hand landmarks\n",
    "        rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        rh = np.zeros(63) #21*3\n",
    "        \n",
    "    #face landmarks\n",
    "    if results.face_landmarks:\n",
    "        #face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten()\n",
    "        i = 0\n",
    "        face = np.array([])\n",
    "        for res in results.face_landmarks.landmark:\n",
    "            if i in faceIndex:\n",
    "                face = np.append(face, np.array([res.x, res.y, res.z]))\n",
    "            i = i + 1\n",
    "    else:\n",
    "        face = np.zeros(54) #18*3\n",
    "    \n",
    "    \n",
    "    #return np.concatenate([pose, lh, rh])\n",
    "    return np.concatenate([pose, lh, rh, face])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keypoints_258(results):\n",
    "    # result의 landmarks의 모든 key point values -> 하나의 numpy array 로 flatten\n",
    "    #if landmarks has no value, fill numpy array with zero\n",
    "    \n",
    "    if results.pose_landmarks: #pose landmarks\n",
    "        pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        pose = np.zeros(132) #33*4\n",
    "\n",
    "    if results.left_hand_landmarks: #left hand landmarks\n",
    "        lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        lh = np.zeros(63) #21*3\n",
    "\n",
    "    if results.right_hand_landmarks: #right hand landmarks\n",
    "        rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        rh = np.zeros(63) #21*3\n",
    "    \n",
    "    \n",
    "    #return np.concatenate([pose, lh, rh])\n",
    "    return np.concatenate([pose, lh, rh])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keypoints_1662(results):\n",
    "    # result의 landmarks의 모든 key point values -> 하나의 numpy array 로 flatten\n",
    "    #if landmarks has no value, fill numpy array with zero\n",
    "    \n",
    "    if results.pose_landmarks: #pose landmarks\n",
    "        pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        pose = np.zeros(132) #33*4\n",
    "\n",
    "    if results.left_hand_landmarks: #left hand landmarks\n",
    "        lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        lh = np.zeros(63) #21*3\n",
    "\n",
    "    if results.right_hand_landmarks: #right hand landmarks\n",
    "        rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        rh = np.zeros(63) #21*3\n",
    "    \n",
    "    #face landmarks\n",
    "    if results.face_landmarks:\n",
    "        face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() \n",
    "    else:\n",
    "        face = np.zeros(1404) #468*3\n",
    "    \n",
    "    #return np.concatenate([pose, lh, rh])\n",
    "    return np.concatenate([pose, lh, rh, face])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Setup Folders for Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "^C\n",
      "Requirement already satisfied: xlrd in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (2.0.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openpyxl in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (3.0.10)\n",
      "Requirement already satisfied: et_xmlfile in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from openpyxl) (1.1.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (1.5.3)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from pandas) (2022.7)\n",
      "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from pandas) (1.23.5)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\wjdgu\\anaconda3\\lib\\site-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -pencv-python (c:\\users\\wjdgu\\anaconda3\\lib\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "#pandas\n",
    "!pip install xlrd\n",
    "!pip install openpyxl\n",
    "!pip install pandas\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['가슴', '개', '귀', '내일', '누나', '다리', '동생', '뒤', '딸', '머리', '목', '물', '발', '배', '불', '뼈', '선생님', '손', '아기', '아내', '아들', '아빠', '앞', '어제', '엄마', '옆쪽', '오늘', '오른쪽', '왼쪽', '위', '친구', '코', '팔', '학교', 1, 2, 3, 4, 5, 6, 7, 8, 9, 0]\n"
     ]
    }
   ],
   "source": [
    "# 파일명\n",
    "file_name = '45.xlsx'\n",
    "\n",
    "# Daraframe형식으로 엑셀 파일 읽기\n",
    "df = pd.read_excel(file_name, sheet_name='Sheet1')\n",
    "\n",
    "list_res = df.loc[:,'단어'].to_list()\n",
    "print(list_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path for exported data, numpy arrays\n",
    "DATA_PATH = os.path.join('datas') \n",
    "\n",
    "#num of videos\n",
    "no_sequences = 100\n",
    "\n",
    "#actions = np.array(['가스', '가슴', '갇히다'])\n",
    "actions = np.array(list_res)\n",
    "\n",
    "# 1 Video = 50 frames\n",
    "sequence_length = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(actions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Collect Keypoint Values for Training and Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "##extract keypoints from video\n",
    "def frame_capture(action, sequence, video):\n",
    "    cap = cv2.VideoCapture(video)\n",
    "    \n",
    "    # Set mediapipe model \n",
    "    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "        frame_num = 0\n",
    "        while cap.isOpened():\n",
    "            # Read feed\n",
    "            ret, frame = cap.read()\n",
    "            if ret == False:\n",
    "                break\n",
    "                \n",
    "            # Make detections\n",
    "            image, results = mediapipe_detection(frame, holistic)  \n",
    "            # Draw landmarks\n",
    "            #draw_styled_landmarks(image, results)\n",
    "            draw_custom_pose_landmarks(image, results)\n",
    "            draw_custom_hand_landmarks(image, results)\n",
    "            draw_custom_face_landmarks(image, results)\n",
    "\n",
    "            # Show to screen\n",
    "            cv2.imshow('OpenCV Video', image)\n",
    "\n",
    "            # export keypoints\n",
    "            keypoints = extract_keypoints(results) #results of frame -> numpy array\n",
    "            npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "            np.save(npy_path, keypoints) #save numpy array in directory\n",
    "            \n",
    "            #increase frame_num\n",
    "            frame_num = frame_num + 1\n",
    "                \n",
    "            # Break gracefully\n",
    "            if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "                break\n",
    "                \n",
    "        cap.release()\n",
    "        cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Preprocess Data and Create Labels and Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = {label:num for num, label in enumerate(actions)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetKeypoint(src):\n",
    "    poseIndex = np.array([22, 16, 14, 12, 24, 21, 15, 13, 11, 23])\n",
    "    faceIndex = np.array([61, 291, 81, 13, 311, 178, 14, 402, 285, 295, 282, 283, 276, 46, 53, 52, 65, 55])\n",
    "    result = np.array([])\n",
    "    #for p in poseIndex:\n",
    "    #    for i in range(0,4):\n",
    "    #        result = np.append(result, src[4*p + i])\n",
    "    #        \n",
    "    #result = np.append(result, src[133:133+126])\n",
    "    result = np.append(result, src[0:258])\n",
    "    \n",
    "    for f in faceIndex:\n",
    "        for i in range(0,3):\n",
    "            result = np.append(result, src[258 + 3*f + i])\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ------------------------양손, 양손 + 포즈, 양손 + 포즈 + 얼굴 ------------------------\n",
    "\n",
    "import fnmatch #폴더 안에 있는 npy 개수 계산하기 위함\n",
    "\n",
    "#create two blank arrays\n",
    "sequences, labels = [], [] \n",
    "#sequences(=videos) = feature data = x data \n",
    "#labels = label data = y data\n",
    "\n",
    "for action in actions: \n",
    "    for sequence in range(no_sequences): #each action has 60 sequences(=videos)\n",
    "        window = []\n",
    "        \n",
    "        #폴더 안에 있는 npy 개수(= frame 수) 계산\n",
    "        file_count = len(fnmatch.filter(os.listdir(os.path.join(DATA_PATH, action, str(sequence))), '*.npy'))\n",
    "        \n",
    "        #저장된 npy 수가 원하는 sequence_length보다 같거나 많을 경우\n",
    "        if(file_count >= sequence_length):  \n",
    "            #앞에 있는 npy 버리고, start부터 끝까지 sequence_length개의 npy 가져오기\n",
    "            start = (file_count - sequence_length)//2\n",
    "            for frame_num in range(sequence_length):\n",
    "                res = np.load(os.path.join(DATA_PATH, action, str(sequence), \"{}.npy\".format(start + frame_num)))\n",
    "                #res_tmp = res[132:258]     # 양손만\n",
    "                #res_tmp = GetKeypoint(res)  # 포즈 + 손 + 일부 얼굴\n",
    "                res_tmp = res[0:258]    # 포즈 + 양손\n",
    "\n",
    "                window.append(res_tmp)\n",
    "        #저장된 npy 수가 원하는 sequence_length보다 적을 경우\n",
    "        else:\n",
    "            #앞에 0으로 채운 npy 추가하기\n",
    "            for frame_num in range(sequence_length - file_count):\n",
    "                res = np.zeros(258) #132 + 63 + 63\n",
    "                window.append(res)\n",
    "            for frame_num in range(file_count):\n",
    "                res = np.load(os.path.join(DATA_PATH, action, str(sequence), \"{}.npy\".format(frame_num)))\n",
    "                window.append(res)\n",
    "                              \n",
    "        #map label\n",
    "        sequences.append(window)\n",
    "        labels.append(label_map[action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4400, 30, 258)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = to_categorical(labels).astype(int) \n",
    "#converted label from int to binary array\n",
    "#1 -> [1,0,0], 2 -> [0,1,0], 3 -> [0,0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)\n",
    "#split x,y data into train data and test data\n",
    "#train data 95%, test data size 5% "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Build and Train LSTM Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense\n",
    "from tensorflow.keras.callbacks import TensorBoard "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = os.path.join('Logs')\n",
    "tb_callback = TensorBoard(log_dir=log_dir)\n",
    "#tensor board is used to monitor neural network training and it's accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_3\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_4 (InputLayer)           [(None, 30, 258)]    0           []                               \n",
      "                                                                                                  \n",
      " conv1d_30 (Conv1D)             (None, 28, 64)       49600       ['input_4[0][0]']                \n",
      "                                                                                                  \n",
      " batch_normalization_30 (BatchN  (None, 28, 64)      256         ['conv1d_30[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_30 (ReLU)                (None, 28, 64)       0           ['batch_normalization_30[0][0]'] \n",
      "                                                                                                  \n",
      " max_pooling1d_12 (MaxPooling1D  (None, 14, 64)      0           ['re_lu_30[0][0]']               \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " conv1d_31 (Conv1D)             (None, 14, 64)       12352       ['max_pooling1d_12[0][0]']       \n",
      "                                                                                                  \n",
      " batch_normalization_31 (BatchN  (None, 14, 64)      256         ['conv1d_31[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_31 (ReLU)                (None, 14, 64)       0           ['batch_normalization_31[0][0]'] \n",
      "                                                                                                  \n",
      " conv1d_32 (Conv1D)             (None, 14, 64)       12352       ['re_lu_31[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_32 (BatchN  (None, 14, 64)      256         ['conv1d_32[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_32 (ReLU)                (None, 14, 64)       0           ['batch_normalization_32[0][0]'] \n",
      "                                                                                                  \n",
      " conv1d_33 (Conv1D)             (None, 14, 64)       12352       ['re_lu_32[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_33 (BatchN  (None, 14, 64)      256         ['conv1d_33[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " add_9 (Add)                    (None, 14, 64)       0           ['batch_normalization_33[0][0]', \n",
      "                                                                  'max_pooling1d_12[0][0]']       \n",
      "                                                                                                  \n",
      " re_lu_33 (ReLU)                (None, 14, 64)       0           ['add_9[0][0]']                  \n",
      "                                                                                                  \n",
      " max_pooling1d_13 (MaxPooling1D  (None, 7, 64)       0           ['re_lu_33[0][0]']               \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " conv1d_34 (Conv1D)             (None, 7, 64)        12352       ['max_pooling1d_13[0][0]']       \n",
      "                                                                                                  \n",
      " batch_normalization_34 (BatchN  (None, 7, 64)       256         ['conv1d_34[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_34 (ReLU)                (None, 7, 64)        0           ['batch_normalization_34[0][0]'] \n",
      "                                                                                                  \n",
      " conv1d_35 (Conv1D)             (None, 7, 64)        12352       ['re_lu_34[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_35 (BatchN  (None, 7, 64)       256         ['conv1d_35[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_35 (ReLU)                (None, 7, 64)        0           ['batch_normalization_35[0][0]'] \n",
      "                                                                                                  \n",
      " conv1d_36 (Conv1D)             (None, 7, 64)        12352       ['re_lu_35[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_36 (BatchN  (None, 7, 64)       256         ['conv1d_36[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " add_10 (Add)                   (None, 7, 64)        0           ['batch_normalization_36[0][0]', \n",
      "                                                                  'max_pooling1d_13[0][0]']       \n",
      "                                                                                                  \n",
      " re_lu_36 (ReLU)                (None, 7, 64)        0           ['add_10[0][0]']                 \n",
      "                                                                                                  \n",
      " max_pooling1d_14 (MaxPooling1D  (None, 4, 64)       0           ['re_lu_36[0][0]']               \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " conv1d_37 (Conv1D)             (None, 4, 64)        12352       ['max_pooling1d_14[0][0]']       \n",
      "                                                                                                  \n",
      " batch_normalization_37 (BatchN  (None, 4, 64)       256         ['conv1d_37[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_37 (ReLU)                (None, 4, 64)        0           ['batch_normalization_37[0][0]'] \n",
      "                                                                                                  \n",
      " conv1d_38 (Conv1D)             (None, 4, 64)        12352       ['re_lu_37[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_38 (BatchN  (None, 4, 64)       256         ['conv1d_38[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " re_lu_38 (ReLU)                (None, 4, 64)        0           ['batch_normalization_38[0][0]'] \n",
      "                                                                                                  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " conv1d_39 (Conv1D)             (None, 4, 64)        12352       ['re_lu_38[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_39 (BatchN  (None, 4, 64)       256         ['conv1d_39[0][0]']              \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " add_11 (Add)                   (None, 4, 64)        0           ['batch_normalization_39[0][0]', \n",
      "                                                                  'max_pooling1d_14[0][0]']       \n",
      "                                                                                                  \n",
      " re_lu_39 (ReLU)                (None, 4, 64)        0           ['add_11[0][0]']                 \n",
      "                                                                                                  \n",
      " max_pooling1d_15 (MaxPooling1D  (None, 2, 64)       0           ['re_lu_39[0][0]']               \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " bidirectional_12 (Bidirectiona  (None, 2, 128)      66048       ['max_pooling1d_15[0][0]']       \n",
      " l)                                                                                               \n",
      "                                                                                                  \n",
      " dropout_12 (Dropout)           (None, 2, 128)       0           ['bidirectional_12[0][0]']       \n",
      "                                                                                                  \n",
      " bidirectional_13 (Bidirectiona  (None, 2, 256)      263168      ['dropout_12[0][0]']             \n",
      " l)                                                                                               \n",
      "                                                                                                  \n",
      " dropout_13 (Dropout)           (None, 2, 256)       0           ['bidirectional_13[0][0]']       \n",
      "                                                                                                  \n",
      " bidirectional_14 (Bidirectiona  (None, 2, 128)      164352      ['dropout_13[0][0]']             \n",
      " l)                                                                                               \n",
      "                                                                                                  \n",
      " dropout_14 (Dropout)           (None, 2, 128)       0           ['bidirectional_14[0][0]']       \n",
      "                                                                                                  \n",
      " bidirectional_15 (Bidirectiona  (None, 2, 128)      98816       ['dropout_14[0][0]']             \n",
      " l)                                                                                               \n",
      "                                                                                                  \n",
      " dropout_15 (Dropout)           (None, 2, 128)       0           ['bidirectional_15[0][0]']       \n",
      "                                                                                                  \n",
      " lstm_19 (LSTM)                 (None, 64)           49408       ['dropout_15[0][0]']             \n",
      "                                                                                                  \n",
      " dense_3 (Dense)                (None, 44)           2860        ['lstm_19[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 807,980\n",
      "Trainable params: 806,700\n",
      "Non-trainable params: 1,280\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Add, Input, Embedding, GlobalAveragePooling1D, Conv1D, ReLU, MaxPooling1D, LSTM, Dense, Bidirectional, Dropout, BatchNormalization\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "\n",
    "#Residual Block 구현\n",
    "def residual_block(x, filters, kernel_size=3, stride=1):\n",
    "    # Shortcut Connection\n",
    "    shortcut = x\n",
    "    \n",
    "    # Main Path\n",
    "    x = Conv1D(filters, kernel_size=kernel_size, strides=stride, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU()(x)\n",
    "    \n",
    "    x = Conv1D(filters, kernel_size=kernel_size, strides=stride, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU()(x)\n",
    "    \n",
    "    x = Conv1D(filters, kernel_size=kernel_size, strides=stride, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    # Shortcut Connection 추가\n",
    "    x = Add()([x, shortcut])\n",
    "    x = ReLU()(x)\n",
    "    \n",
    "    return x\n",
    "\n",
    "\n",
    "input_layer = Input(shape=(30,258))\n",
    "\n",
    "x = Conv1D(64, kernel_size=3, activation='relu')(input_layer)\n",
    "x = BatchNormalization()(x)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling1D(pool_size=2)(x)\n",
    "\n",
    "# --------------------------------------------------------\n",
    "x = residual_block(x, filters=64)\n",
    "x = MaxPooling1D(pool_size=2, padding='same')(x)\n",
    "\n",
    "x = residual_block(x, filters=64)\n",
    "x = MaxPooling1D(pool_size=2, padding='same')(x)\n",
    "\n",
    "x = residual_block(x, filters=64)\n",
    "x = MaxPooling1D(pool_size=2)(x)\n",
    "\n",
    "# --------------------------------------------------------\n",
    "\n",
    "x = Bidirectional(LSTM(64, return_sequences=True, activation='tanh'))(x)\n",
    "x = Dropout(0.3)(x)\n",
    "x = Bidirectional(LSTM(128, return_sequences=True, activation='tanh'))(x)\n",
    "x = Dropout(0.3)(x)\n",
    "x = Bidirectional(LSTM(64, return_sequences=True, activation='tanh'))(x)\n",
    "x = Dropout(0.3)(x)\n",
    "x = Bidirectional(LSTM(64, return_sequences=True, activation='tanh'))(x)\n",
    "x = Dropout(0.3)(x)\n",
    "x = LSTM(64, return_sequences=False, activation='relu')(x)\n",
    "\n",
    "\n",
    "output_layer = Dense(actions.shape[0], activation='softmax')(x)\n",
    "model = Model(inputs=input_layer, outputs=output_layer)\n",
    "\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "124/124 [==============================] - 15s 38ms/step - loss: 3.5292 - categorical_accuracy: 0.0366\n",
      "Epoch 2/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 3.2223 - categorical_accuracy: 0.0412\n",
      "Epoch 3/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 3.1687 - categorical_accuracy: 0.0631\n",
      "Epoch 4/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 3.1051 - categorical_accuracy: 0.0715\n",
      "Epoch 5/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 3.0523 - categorical_accuracy: 0.0843\n",
      "Epoch 6/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.9685 - categorical_accuracy: 0.1040\n",
      "Epoch 7/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.9008 - categorical_accuracy: 0.1136\n",
      "Epoch 8/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.6612 - categorical_accuracy: 0.1321\n",
      "Epoch 9/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.5049 - categorical_accuracy: 0.1657\n",
      "Epoch 10/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.3216 - categorical_accuracy: 0.1977\n",
      "Epoch 11/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.1804 - categorical_accuracy: 0.2354\n",
      "Epoch 12/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 2.0490 - categorical_accuracy: 0.2558\n",
      "Epoch 13/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 1.9088 - categorical_accuracy: 0.2874\n",
      "Epoch 14/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 1.9025 - categorical_accuracy: 0.3010\n",
      "Epoch 15/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.8118 - categorical_accuracy: 0.3172\n",
      "Epoch 16/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.6917 - categorical_accuracy: 0.3528\n",
      "Epoch 17/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.6431 - categorical_accuracy: 0.3811\n",
      "Epoch 18/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.5600 - categorical_accuracy: 0.4179\n",
      "Epoch 19/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.5947 - categorical_accuracy: 0.4126\n",
      "Epoch 20/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.4563 - categorical_accuracy: 0.4518\n",
      "Epoch 21/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.3966 - categorical_accuracy: 0.4692\n",
      "Epoch 22/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.3692 - categorical_accuracy: 0.4957\n",
      "Epoch 23/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.3507 - categorical_accuracy: 0.5023\n",
      "Epoch 24/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.2268 - categorical_accuracy: 0.5301\n",
      "Epoch 25/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.1953 - categorical_accuracy: 0.5404\n",
      "Epoch 26/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.1950 - categorical_accuracy: 0.5434\n",
      "Epoch 27/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.1240 - categorical_accuracy: 0.5725\n",
      "Epoch 28/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.0883 - categorical_accuracy: 0.5843\n",
      "Epoch 29/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.0915 - categorical_accuracy: 0.5806\n",
      "Epoch 30/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.9866 - categorical_accuracy: 0.6109\n",
      "Epoch 31/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.0479 - categorical_accuracy: 0.6038\n",
      "Epoch 32/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 1.0134 - categorical_accuracy: 0.6116\n",
      "Epoch 33/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.9976 - categorical_accuracy: 0.6154\n",
      "Epoch 34/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.8950 - categorical_accuracy: 0.6515\n",
      "Epoch 35/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.9140 - categorical_accuracy: 0.6341\n",
      "Epoch 36/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.8859 - categorical_accuracy: 0.6609\n",
      "Epoch 37/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.8348 - categorical_accuracy: 0.6631\n",
      "Epoch 38/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.8219 - categorical_accuracy: 0.6912\n",
      "Epoch 39/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.7622 - categorical_accuracy: 0.6970\n",
      "Epoch 40/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 0.7690 - categorical_accuracy: 0.6985\n",
      "Epoch 41/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.7845 - categorical_accuracy: 0.6914\n",
      "Epoch 42/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 0.7894 - categorical_accuracy: 0.6957\n",
      "Epoch 43/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 0.6868 - categorical_accuracy: 0.7285\n",
      "Epoch 44/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.7081 - categorical_accuracy: 0.7341\n",
      "Epoch 45/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.7320 - categorical_accuracy: 0.7232\n",
      "Epoch 46/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.6640 - categorical_accuracy: 0.7452\n",
      "Epoch 47/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.6427 - categorical_accuracy: 0.7545\n",
      "Epoch 48/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5934 - categorical_accuracy: 0.7657\n",
      "Epoch 49/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.6003 - categorical_accuracy: 0.7687\n",
      "Epoch 50/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.6235 - categorical_accuracy: 0.7624\n",
      "Epoch 51/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5594 - categorical_accuracy: 0.7816\n",
      "Epoch 52/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5423 - categorical_accuracy: 0.7896\n",
      "Epoch 53/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5213 - categorical_accuracy: 0.8045\n",
      "Epoch 54/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5595 - categorical_accuracy: 0.7952\n",
      "Epoch 55/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5390 - categorical_accuracy: 0.7947\n",
      "Epoch 56/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5555 - categorical_accuracy: 0.7881\n",
      "Epoch 57/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5348 - categorical_accuracy: 0.8000\n",
      "Epoch 58/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.5160 - categorical_accuracy: 0.8061\n",
      "Epoch 59/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4595 - categorical_accuracy: 0.8316\n",
      "Epoch 60/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4626 - categorical_accuracy: 0.8225\n",
      "Epoch 61/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4435 - categorical_accuracy: 0.8258\n",
      "Epoch 62/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4438 - categorical_accuracy: 0.8364\n",
      "Epoch 63/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4602 - categorical_accuracy: 0.8301\n",
      "Epoch 64/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4500 - categorical_accuracy: 0.8308\n",
      "Epoch 65/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4915 - categorical_accuracy: 0.8283\n",
      "Epoch 66/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4145 - categorical_accuracy: 0.8399\n",
      "Epoch 67/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4133 - categorical_accuracy: 0.8465\n",
      "Epoch 68/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4019 - categorical_accuracy: 0.8525\n",
      "Epoch 69/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3522 - categorical_accuracy: 0.8626\n",
      "Epoch 70/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3977 - categorical_accuracy: 0.8518\n",
      "Epoch 71/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3822 - categorical_accuracy: 0.8631\n",
      "Epoch 72/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3838 - categorical_accuracy: 0.8614\n",
      "Epoch 73/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.4185 - categorical_accuracy: 0.8510\n",
      "Epoch 74/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3233 - categorical_accuracy: 0.8768\n",
      "Epoch 75/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3221 - categorical_accuracy: 0.8813\n",
      "Epoch 76/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3300 - categorical_accuracy: 0.8788\n",
      "Epoch 77/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3259 - categorical_accuracy: 0.8745\n",
      "Epoch 78/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3666 - categorical_accuracy: 0.8687\n",
      "Epoch 79/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3470 - categorical_accuracy: 0.8737\n",
      "Epoch 80/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3227 - categorical_accuracy: 0.8833\n",
      "Epoch 81/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3350 - categorical_accuracy: 0.8788\n",
      "Epoch 82/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2685 - categorical_accuracy: 0.8980\n",
      "Epoch 83/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3428 - categorical_accuracy: 0.8806\n",
      "Epoch 84/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2829 - categorical_accuracy: 0.8919\n",
      "Epoch 85/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 0.3015 - categorical_accuracy: 0.8879\n",
      "Epoch 86/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2663 - categorical_accuracy: 0.8955\n",
      "Epoch 87/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2853 - categorical_accuracy: 0.8982\n",
      "Epoch 88/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2810 - categorical_accuracy: 0.8992\n",
      "Epoch 89/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2750 - categorical_accuracy: 0.9020\n",
      "Epoch 90/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.3041 - categorical_accuracy: 0.8939\n",
      "Epoch 91/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2692 - categorical_accuracy: 0.9061\n",
      "Epoch 92/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2724 - categorical_accuracy: 0.9056\n",
      "Epoch 93/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2927 - categorical_accuracy: 0.8995\n",
      "Epoch 94/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2235 - categorical_accuracy: 0.9205\n",
      "Epoch 95/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2370 - categorical_accuracy: 0.9164\n",
      "Epoch 96/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2442 - categorical_accuracy: 0.9124\n",
      "Epoch 97/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2473 - categorical_accuracy: 0.9146\n",
      "Epoch 98/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2074 - categorical_accuracy: 0.9285\n",
      "Epoch 99/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2419 - categorical_accuracy: 0.9177\n",
      "Epoch 100/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2707 - categorical_accuracy: 0.9030\n",
      "Epoch 101/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2014 - categorical_accuracy: 0.9260\n",
      "Epoch 102/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2109 - categorical_accuracy: 0.9215\n",
      "Epoch 103/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2089 - categorical_accuracy: 0.9280\n",
      "Epoch 104/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2507 - categorical_accuracy: 0.9139\n",
      "Epoch 105/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1961 - categorical_accuracy: 0.9295\n",
      "Epoch 106/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2434 - categorical_accuracy: 0.9207\n",
      "Epoch 107/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2012 - categorical_accuracy: 0.9263\n",
      "Epoch 108/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1776 - categorical_accuracy: 0.9384\n",
      "Epoch 109/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 0.1972 - categorical_accuracy: 0.9298\n",
      "Epoch 110/200\n",
      "124/124 [==============================] - 5s 36ms/step - loss: 0.1795 - categorical_accuracy: 0.9343\n",
      "Epoch 111/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2156 - categorical_accuracy: 0.9260\n",
      "Epoch 112/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2569 - categorical_accuracy: 0.9174\n",
      "Epoch 113/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2124 - categorical_accuracy: 0.9295\n",
      "Epoch 114/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1940 - categorical_accuracy: 0.9306\n",
      "Epoch 115/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2096 - categorical_accuracy: 0.9306\n",
      "Epoch 116/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1957 - categorical_accuracy: 0.9336\n",
      "Epoch 117/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1971 - categorical_accuracy: 0.9354\n",
      "Epoch 118/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1599 - categorical_accuracy: 0.9455\n",
      "Epoch 119/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1801 - categorical_accuracy: 0.9386\n",
      "Epoch 120/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.2279 - categorical_accuracy: 0.9253\n",
      "Epoch 121/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1710 - categorical_accuracy: 0.9414\n",
      "Epoch 122/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1657 - categorical_accuracy: 0.9429\n",
      "Epoch 123/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1783 - categorical_accuracy: 0.9427\n",
      "Epoch 124/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1400 - categorical_accuracy: 0.9525\n",
      "Epoch 125/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1288 - categorical_accuracy: 0.9543\n",
      "Epoch 126/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1916 - categorical_accuracy: 0.9364\n",
      "Epoch 127/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1515 - categorical_accuracy: 0.9510\n",
      "Epoch 128/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1543 - categorical_accuracy: 0.9500\n",
      "Epoch 129/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1485 - categorical_accuracy: 0.9495\n",
      "Epoch 130/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1857 - categorical_accuracy: 0.9374\n",
      "Epoch 131/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1669 - categorical_accuracy: 0.9437\n",
      "Epoch 132/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1174 - categorical_accuracy: 0.9556\n",
      "Epoch 133/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1616 - categorical_accuracy: 0.9444\n",
      "Epoch 134/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1361 - categorical_accuracy: 0.9573\n",
      "Epoch 135/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1557 - categorical_accuracy: 0.9515\n",
      "Epoch 136/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1427 - categorical_accuracy: 0.9533\n",
      "Epoch 137/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1145 - categorical_accuracy: 0.9616\n",
      "Epoch 138/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1322 - categorical_accuracy: 0.9563\n",
      "Epoch 139/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1379 - categorical_accuracy: 0.9558\n",
      "Epoch 140/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1715 - categorical_accuracy: 0.9475\n",
      "Epoch 141/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1328 - categorical_accuracy: 0.9558\n",
      "Epoch 142/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1383 - categorical_accuracy: 0.9583\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 143/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1538 - categorical_accuracy: 0.9508\n",
      "Epoch 144/200\n",
      "124/124 [==============================] - 4s 36ms/step - loss: 0.1125 - categorical_accuracy: 0.9611\n",
      "Epoch 145/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1371 - categorical_accuracy: 0.9588\n",
      "Epoch 146/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1365 - categorical_accuracy: 0.9576\n",
      "Epoch 147/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1364 - categorical_accuracy: 0.9553\n",
      "Epoch 148/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1916 - categorical_accuracy: 0.9412\n",
      "Epoch 149/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1322 - categorical_accuracy: 0.9573\n",
      "Epoch 150/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1057 - categorical_accuracy: 0.9664\n",
      "Epoch 151/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1117 - categorical_accuracy: 0.9649\n",
      "Epoch 152/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1707 - categorical_accuracy: 0.9470\n",
      "Epoch 153/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0706 - categorical_accuracy: 0.9773\n",
      "Epoch 154/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1195 - categorical_accuracy: 0.9646\n",
      "Epoch 155/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1062 - categorical_accuracy: 0.9624\n",
      "Epoch 156/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1072 - categorical_accuracy: 0.9636\n",
      "Epoch 157/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1165 - categorical_accuracy: 0.9609\n",
      "Epoch 158/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1241 - categorical_accuracy: 0.9604\n",
      "Epoch 159/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1472 - categorical_accuracy: 0.9563\n",
      "Epoch 160/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0767 - categorical_accuracy: 0.9755\n",
      "Epoch 161/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1324 - categorical_accuracy: 0.9588\n",
      "Epoch 162/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1327 - categorical_accuracy: 0.9616\n",
      "Epoch 163/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1061 - categorical_accuracy: 0.9662\n",
      "Epoch 164/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0996 - categorical_accuracy: 0.9720\n",
      "Epoch 165/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0868 - categorical_accuracy: 0.9725\n",
      "Epoch 166/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1388 - categorical_accuracy: 0.9588\n",
      "Epoch 167/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0923 - categorical_accuracy: 0.9758\n",
      "Epoch 168/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0981 - categorical_accuracy: 0.9699\n",
      "Epoch 169/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0957 - categorical_accuracy: 0.9710\n",
      "Epoch 170/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0894 - categorical_accuracy: 0.9720\n",
      "Epoch 171/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0971 - categorical_accuracy: 0.9674\n",
      "Epoch 172/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1027 - categorical_accuracy: 0.9715\n",
      "Epoch 173/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1250 - categorical_accuracy: 0.9649\n",
      "Epoch 174/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0715 - categorical_accuracy: 0.9780\n",
      "Epoch 175/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0937 - categorical_accuracy: 0.9740\n",
      "Epoch 176/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0827 - categorical_accuracy: 0.9758\n",
      "Epoch 177/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0964 - categorical_accuracy: 0.9715\n",
      "Epoch 178/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0771 - categorical_accuracy: 0.9765\n",
      "Epoch 179/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1309 - categorical_accuracy: 0.9616\n",
      "Epoch 180/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0774 - categorical_accuracy: 0.9758\n",
      "Epoch 181/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1049 - categorical_accuracy: 0.9699\n",
      "Epoch 182/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0976 - categorical_accuracy: 0.9707\n",
      "Epoch 183/200\n",
      "124/124 [==============================] - 5s 38ms/step - loss: 0.1093 - categorical_accuracy: 0.9682\n",
      "Epoch 184/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0881 - categorical_accuracy: 0.9717\n",
      "Epoch 185/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0736 - categorical_accuracy: 0.9778\n",
      "Epoch 186/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0961 - categorical_accuracy: 0.9715\n",
      "Epoch 187/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1406 - categorical_accuracy: 0.9619\n",
      "Epoch 188/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0580 - categorical_accuracy: 0.9826\n",
      "Epoch 189/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0709 - categorical_accuracy: 0.9785\n",
      "Epoch 190/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0848 - categorical_accuracy: 0.9763\n",
      "Epoch 191/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1261 - categorical_accuracy: 0.9578\n",
      "Epoch 192/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1210 - categorical_accuracy: 0.9646\n",
      "Epoch 193/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0976 - categorical_accuracy: 0.9715\n",
      "Epoch 194/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0707 - categorical_accuracy: 0.9793\n",
      "Epoch 195/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0775 - categorical_accuracy: 0.9750\n",
      "Epoch 196/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0836 - categorical_accuracy: 0.9765\n",
      "Epoch 197/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.1075 - categorical_accuracy: 0.9684\n",
      "Epoch 198/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0918 - categorical_accuracy: 0.9747\n",
      "Epoch 199/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0585 - categorical_accuracy: 0.9795\n",
      "Epoch 200/200\n",
      "124/124 [==============================] - 5s 37ms/step - loss: 0.0779 - categorical_accuracy: 0.9722\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2043a435780>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs=200, callbacks=[tb_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv1d_2 (Conv1D)           (None, 88, 64)            24256     \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 88, 64)           256       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " re_lu_2 (ReLU)              (None, 88, 64)            0         \n",
      "                                                                 \n",
      " max_pooling1d_2 (MaxPooling  (None, 44, 64)           0         \n",
      " 1D)                                                             \n",
      "                                                                 \n",
      " lstm_29 (LSTM)              (None, 44, 512)           1181696   \n",
      "                                                                 \n",
      " dropout_28 (Dropout)        (None, 44, 512)           0         \n",
      "                                                                 \n",
      " lstm_30 (LSTM)              (None, 44, 256)           787456    \n",
      "                                                                 \n",
      " dropout_29 (Dropout)        (None, 44, 256)           0         \n",
      "                                                                 \n",
      " lstm_31 (LSTM)              (None, 44, 128)           197120    \n",
      "                                                                 \n",
      " dropout_30 (Dropout)        (None, 44, 128)           0         \n",
      "                                                                 \n",
      " lstm_32 (LSTM)              (None, 64)                49408     \n",
      "                                                                 \n",
      " dropout_31 (Dropout)        (None, 64)                0         \n",
      "                                                                 \n",
      " dense_20 (Dense)            (None, 64)                4160      \n",
      "                                                                 \n",
      " dense_21 (Dense)            (None, 32)                2080      \n",
      "                                                                 \n",
      " dense_22 (Dense)            (None, 3)                 99        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 2,246,531\n",
      "Trainable params: 2,246,403\n",
      "Non-trainable params: 128\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7/7 [==============================] - 2s 4ms/step\n"
     ]
    }
   ],
   "source": [
    "predict_res = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9. Save Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#단일 file/folder 에 모델의 아키텍처, 가중치 및 훈련 구성을 저장\n",
    "# pose + hands : model_44_258.h5\n",
    "# hands : model_44_126.h5\n",
    "# pose 일부 + hands + face 일부 : model_44.220.h5\n",
    "# pose + hands + face 일부 : model_44_312.h5\n",
    "model.save('model_44_258(3).h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('model_44_258.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10. Evaluation using Confusion Matrix and Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import multilabel_confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 2s 9ms/step\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y_test의 값들이 가리키는 label들을 하나의 리스트로 변환\n",
    "#ex. [[0, 0, 1], [0, 1, 0]] -> [2, 1]\n",
    "ytrue = np.argmax(y_test, axis=1).tolist()\n",
    "#yhat의 값들이 가리키는 label들을 하나의 리스트로 변환\n",
    "#ex. [[0.1, 0.2, 0.7], [0.1, 0.8, 0.1]] -> [2, 1]\n",
    "yhat = np.argmax(yhat, axis=1).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[427,   0],\n",
       "        [  0,  13]],\n",
       "\n",
       "       [[433,   0],\n",
       "        [  0,   7]],\n",
       "\n",
       "       [[430,   0],\n",
       "        [  0,  10]],\n",
       "\n",
       "       [[429,   0],\n",
       "        [  1,  10]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[428,   1],\n",
       "        [  0,  11]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[426,   0],\n",
       "        [  0,  14]],\n",
       "\n",
       "       [[430,   0],\n",
       "        [  1,   9]],\n",
       "\n",
       "       [[427,   0],\n",
       "        [  0,  13]],\n",
       "\n",
       "       [[432,   0],\n",
       "        [  0,   8]],\n",
       "\n",
       "       [[429,   0],\n",
       "        [  0,  11]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[432,   0],\n",
       "        [  0,   8]],\n",
       "\n",
       "       [[432,   0],\n",
       "        [  0,   8]],\n",
       "\n",
       "       [[432,   0],\n",
       "        [  0,   8]],\n",
       "\n",
       "       [[428,   0],\n",
       "        [  0,  12]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[429,   0],\n",
       "        [  0,  11]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[425,   0],\n",
       "        [  6,   9]],\n",
       "\n",
       "       [[429,   0],\n",
       "        [  1,  10]],\n",
       "\n",
       "       [[427,   0],\n",
       "        [  0,  13]],\n",
       "\n",
       "       [[432,   0],\n",
       "        [  0,   8]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[428,   0],\n",
       "        [  0,  12]],\n",
       "\n",
       "       [[435,   0],\n",
       "        [  0,   5]],\n",
       "\n",
       "       [[426,   1],\n",
       "        [  0,  13]],\n",
       "\n",
       "       [[421,   0],\n",
       "        [  0,  19]],\n",
       "\n",
       "       [[428,   0],\n",
       "        [  0,  12]],\n",
       "\n",
       "       [[430,   0],\n",
       "        [  0,  10]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  0,   9]],\n",
       "\n",
       "       [[431,   0],\n",
       "        [  4,   5]],\n",
       "\n",
       "       [[428,   1],\n",
       "        [  0,  11]],\n",
       "\n",
       "       [[426,   8],\n",
       "        [  0,   6]],\n",
       "\n",
       "       [[433,   0],\n",
       "        [  6,   1]],\n",
       "\n",
       "       [[432,   0],\n",
       "        [  0,   8]],\n",
       "\n",
       "       [[430,   0],\n",
       "        [  9,   1]],\n",
       "\n",
       "       [[416,  12],\n",
       "        [  7,   5]],\n",
       "\n",
       "       [[419,  14],\n",
       "        [  0,   7]],\n",
       "\n",
       "       [[432,   3],\n",
       "        [  3,   2]],\n",
       "\n",
       "       [[429,   0],\n",
       "        [  2,   9]]], dtype=int64)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#returns a confusion matrix sorted by the label order\n",
    "multilabel_confusion_matrix(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9090909090909091"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11-1. Test using video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "#한글 텍스트 출력\n",
    "from PIL import ImageFont, ImageDraw, Image\n",
    "\n",
    "def putKoreanText(src, text, pos, font_size, font_color):\n",
    "    img_pil = Image.fromarray(src)\n",
    "    draw = ImageDraw.Draw(img_pil)\n",
    "    font = ImageFont.truetype('C:/Users/User/ActionDetectionforSignLanguage/fonts/gulim.ttc', font_size)\n",
    "    draw.text(pos, text, font=font, fill= font_color)\n",
    "    return np.array(img_pil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prediction using video\n",
    "def video_prediction(video):\n",
    "    \n",
    "    # 1. detection variables\n",
    "    sequence = [] #collect 60 frames to make a sequence(=video)\n",
    "    sentence = [] #concatenate history of predictions together\n",
    "    threshold = 0.995\n",
    "\n",
    "    cap = cv2.VideoCapture(video)\n",
    "    \n",
    "    #clear subtitles when {clear_cycle} frames passed without new subtitle added\n",
    "    frames_without_new_subtitle = 0\n",
    "    clear_cycle = 300\n",
    "    \n",
    "    word = ''\n",
    "    cnt = 0\n",
    "    \n",
    "    # Set mediapipe model \n",
    "    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "        while cap.isOpened():\n",
    "            # Read video\n",
    "            ret, frame = cap.read()\n",
    "            if ret == False:\n",
    "                break\n",
    "                \n",
    "            # Make detections\n",
    "            image, results = mediapipe_detection(frame, holistic)\n",
    "            print(results)\n",
    "            # Draw landmarks\n",
    "            draw_styled_landmarks(image, results)\n",
    "\n",
    "            # 2. Prediction logic\n",
    "            keypoints = extract_keypoints(results)\n",
    "            sequence.append(keypoints)\n",
    "            sequence = sequence[-60:] #generate sequence with last 30 frames\n",
    "            \n",
    "            #frames_without_new_subtitle += 1\n",
    "\n",
    "            if len(sequence) == 60:\n",
    "                #sequence.shape = (60, 1662)\n",
    "                #the input shape model expects = (number of sequences, 60, 1662)\n",
    "                res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "                print(actions[np.argmax(res)])\n",
    "                \n",
    "                if word == actions[np.argmax(res)]:\n",
    "                    cnt = cnt + 1\n",
    "                else :\n",
    "                    word = actions[np.argmax(res)]\n",
    "                    cnt = 0\n",
    "                    \n",
    "\n",
    "                #3. Rendering logic\n",
    "                #ex. res = [0.1, 0.2, 0.7]\n",
    "                #np.argmax(res) = 2, res[np.argmax(res)] = 0.7\n",
    "                if res[np.argmax(res)] > threshold and cnt >= 30: \n",
    "                    #new subtitle added\n",
    "                    #frames_without_new_subtitle = 0\n",
    "\n",
    "                    cur_action_korean = actions[np.argmax(res)]\n",
    "\n",
    "                    if len(sentence) > 0: \n",
    "                        #sentence에 저장된 prediction이 있는 경우 \n",
    "                        #새로운 prediction인 경우에만 sentence에 추가\n",
    "                        if cur_action_korean != sentence[-1]:\n",
    "                            sentence.append(cur_action_korean)\n",
    "                    else: \n",
    "                        #sentence에 저장된 prediction 없는 경우 바로 sentence에 추가\n",
    "                        sentence.append(cur_action_korean)\n",
    "\n",
    "                #sentence가 너무 길어지지 않도록 마지막 5개의 prediction만 유지\n",
    "                if len(sentence) > 5: \n",
    "                    sentence = sentence[-5:]\n",
    "                \n",
    "                #Clear subtitles if needed\n",
    "#                if frames_without_new_subtitle >= clear_cycle:\n",
    "#                    sentence.clear()\n",
    "                \n",
    "                #Render subtitles\n",
    "                cv2.rectangle(image, (0,0), (640, 40), (0, 0, 0), -1) \n",
    "#                if target == 'ko':\n",
    "#                    #putKoreanText(src, text, pos, font_size, font_color\n",
    "#                    image = putKoreanText(image, ' '.join(sentence), (3,10), 20, (255, 255, 255))\n",
    "#                else:\n",
    "#                    cv2.putText(image, ' '.join(sentence), (3,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.LINE_AA)\n",
    "                image = putKoreanText(image, ' '.join(sentence), (3,10),(20),(255,255,255))\n",
    "                # Show to screen\n",
    "                cv2.imshow('Video Prediction', image)\n",
    "\n",
    "            # Break gracefully\n",
    "            if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "                break\n",
    "                \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prediction using video\n",
    "def custom_video_prediction(video):\n",
    "    \n",
    "    # 1. detection variables\n",
    "    sequence = [] #collect 60 frames to make a sequence(=video)\n",
    "    sentence = [] #concatenate history of predictions together\n",
    "    threshold = 0.995\n",
    "\n",
    "    cap = cv2.VideoCapture(video)\n",
    "    \n",
    "    #clear subtitles when {clear_cycle} frames passed without new subtitle added\n",
    "    frames_without_new_subtitle = 0\n",
    "    clear_cycle = 300\n",
    "    \n",
    "    word = ''\n",
    "    cnt = 0\n",
    "    frame_num = 0\n",
    "    \n",
    "    # Set mediapipe model \n",
    "    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "        while cap.isOpened():\n",
    "            # Read video\n",
    "            ret, frame = cap.read()\n",
    "            if ret == False:\n",
    "                break\n",
    "                \n",
    "            frame_num = frame_num+1\n",
    "            \n",
    "            if frame_num%2 == 0:\n",
    "                continue\n",
    "\n",
    "            \n",
    "            # Make detections\n",
    "            image, results = mediapipe_detection(frame, holistic)\n",
    "            print(results)\n",
    "            # Draw landmarks\n",
    "            draw_styled_landmarks(image, results)\n",
    "            #draw_custom_pose_landmarks(image, results)\n",
    "            #draw_custom_hand_landmarks(image, results)\n",
    "            #draw_custom_face_landmarks(image, results)\n",
    "\n",
    "            # 2. Prediction logic\n",
    "            keypoints = extract_keypoints_258(results)\n",
    "            sequence.append(keypoints)\n",
    "            sequence = sequence[-30:] #generate sequence with last 30 frames\n",
    "            \n",
    "            #frames_without_new_subtitle += 1\n",
    "\n",
    "            if len(sequence) == 30:\n",
    "                #sequence.shape = (60, 1662)\n",
    "                #the input shape model expects = (number of sequences, 60, 1662)\n",
    "                res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "                print(actions[np.argmax(res)])\n",
    "                \n",
    "                if word == actions[np.argmax(res)]:\n",
    "                    cnt = cnt + 1\n",
    "                else :\n",
    "                    word = actions[np.argmax(res)]\n",
    "                    cnt = 1\n",
    "                    \n",
    "\n",
    "                #3. Rendering logic\n",
    "                #ex. res = [0.1, 0.2, 0.7]\n",
    "                #np.argmax(res) = 2, res[np.argmax(res)] = 0.7\n",
    "                if res[np.argmax(res)] > threshold and cnt >= 10: \n",
    "                    #new subtitle added\n",
    "                    #frames_without_new_subtitle = 0\n",
    "\n",
    "                    cur_action_korean = actions[np.argmax(res)]\n",
    "\n",
    "                    if len(sentence) > 0: \n",
    "                        #sentence에 저장된 prediction이 있는 경우 \n",
    "                        #새로운 prediction인 경우에만 sentence에 추가\n",
    "                        if cur_action_korean != sentence[-1]:\n",
    "                            sentence.append(cur_action_korean)\n",
    "                    else: \n",
    "                        #sentence에 저장된 prediction 없는 경우 바로 sentence에 추가\n",
    "                        sentence.append(cur_action_korean)\n",
    "\n",
    "                #sentence가 너무 길어지지 않도록 마지막 5개의 prediction만 유지\n",
    "                if len(sentence) > 5: \n",
    "                    sentence = sentence[-5:]\n",
    "                \n",
    "                #Clear subtitles if needed\n",
    "#                if frames_without_new_subtitle >= clear_cycle:\n",
    "#                    sentence.clear()\n",
    "                \n",
    "                #Render subtitles\n",
    "                cv2.rectangle(image, (0,0), (640, 40), (0, 0, 0), -1) \n",
    "#                if target == 'ko':\n",
    "#                    #putKoreanText(src, text, pos, font_size, font_color\n",
    "#                    image = putKoreanText(image, ' '.join(sentence), (3,10), 20, (255, 255, 255))\n",
    "#                else:\n",
    "#                    cv2.putText(image, ' '.join(sentence), (3,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.LINE_AA)\n",
    "                image = putKoreanText(image, ' '.join(sentence), (3,10),(20),(255,255,255))\n",
    "                # Show to screen\n",
    "                cv2.imshow('Video Prediction', image)\n",
    "\n",
    "            # Break gracefully\n",
    "            if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "                break\n",
    "                \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 31ms/step\n",
      "팔\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 17ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "팔\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 16ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 13ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 15ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "1/1 [==============================] - 0s 14ms/step\n",
      "손\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n",
      "<class 'mediapipe.python.solution_base.SolutionOutputs'>\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[112], line 5\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m(file\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.mp4\u001b[39m\u001b[38;5;124m\"\u001b[39m)):\n\u001b[0;32m      4\u001b[0m     path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(curDir, file)\n\u001b[1;32m----> 5\u001b[0m     \u001b[43mcustom_video_prediction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[110], line 34\u001b[0m, in \u001b[0;36mcustom_video_prediction\u001b[1;34m(video)\u001b[0m\n\u001b[0;32m     30\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[0;32m     33\u001b[0m \u001b[38;5;66;03m# Make detections\u001b[39;00m\n\u001b[1;32m---> 34\u001b[0m image, results \u001b[38;5;241m=\u001b[39m \u001b[43mmediapipe_detection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mframe\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mholistic\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     35\u001b[0m \u001b[38;5;28mprint\u001b[39m(results)\n\u001b[0;32m     36\u001b[0m \u001b[38;5;66;03m# Draw landmarks\u001b[39;00m\n",
      "Cell \u001b[1;32mIn[3], line 6\u001b[0m, in \u001b[0;36mmediapipe_detection\u001b[1;34m(image, model)\u001b[0m\n\u001b[0;32m      4\u001b[0m image \u001b[38;5;241m=\u001b[39m cv2\u001b[38;5;241m.\u001b[39mcvtColor(image, cv2\u001b[38;5;241m.\u001b[39mCOLOR_BGR2RGB) \u001b[38;5;66;03m# COLOR CONVERSION BGR -> RGB\u001b[39;00m\n\u001b[0;32m      5\u001b[0m image\u001b[38;5;241m.\u001b[39mflags\u001b[38;5;241m.\u001b[39mwriteable \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m                  \u001b[38;5;66;03m# Image is no longer writeable\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimage\u001b[49m\u001b[43m)\u001b[49m                 \u001b[38;5;66;03m# Make prediction\u001b[39;00m\n\u001b[0;32m      7\u001b[0m image\u001b[38;5;241m.\u001b[39mflags\u001b[38;5;241m.\u001b[39mwriteable \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m                   \u001b[38;5;66;03m# Image is now writeable \u001b[39;00m\n\u001b[0;32m      8\u001b[0m image \u001b[38;5;241m=\u001b[39m cv2\u001b[38;5;241m.\u001b[39mcvtColor(image, cv2\u001b[38;5;241m.\u001b[39mCOLOR_RGB2BGR) \u001b[38;5;66;03m# COLOR COVERSION RGB -> BGR\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\mediapipe\\python\\solutions\\holistic.py:160\u001b[0m, in \u001b[0;36mHolistic.process\u001b[1;34m(self, image)\u001b[0m\n\u001b[0;32m    136\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprocess\u001b[39m(\u001b[38;5;28mself\u001b[39m, image: np\u001b[38;5;241m.\u001b[39mndarray) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m NamedTuple:\n\u001b[0;32m    137\u001b[0m   \u001b[38;5;124;03m\"\"\"Processes an RGB image and returns the pose landmarks, left and right hand landmarks, and face landmarks on the most prominent person detected.\u001b[39;00m\n\u001b[0;32m    138\u001b[0m \n\u001b[0;32m    139\u001b[0m \u001b[38;5;124;03m  Args:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    157\u001b[0m \u001b[38;5;124;03m         \"enable_segmentation\" is set to true.\u001b[39;00m\n\u001b[0;32m    158\u001b[0m \u001b[38;5;124;03m  \"\"\"\u001b[39;00m\n\u001b[1;32m--> 160\u001b[0m   results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mimage\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mimage\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    161\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m results\u001b[38;5;241m.\u001b[39mpose_landmarks:  \u001b[38;5;66;03m# pytype: disable=attribute-error\u001b[39;00m\n\u001b[0;32m    162\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m landmark \u001b[38;5;129;01min\u001b[39;00m results\u001b[38;5;241m.\u001b[39mpose_landmarks\u001b[38;5;241m.\u001b[39mlandmark:  \u001b[38;5;66;03m# pytype: disable=attribute-error\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\mediapipe\\python\\solution_base.py:365\u001b[0m, in \u001b[0;36mSolutionBase.process\u001b[1;34m(self, input_data)\u001b[0m\n\u001b[0;32m    359\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    360\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_graph\u001b[38;5;241m.\u001b[39madd_packet_to_input_stream(\n\u001b[0;32m    361\u001b[0m         stream\u001b[38;5;241m=\u001b[39mstream_name,\n\u001b[0;32m    362\u001b[0m         packet\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_packet(input_stream_type,\n\u001b[0;32m    363\u001b[0m                                  data)\u001b[38;5;241m.\u001b[39mat(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_simulated_timestamp))\n\u001b[1;32m--> 365\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_graph\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_until_idle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    366\u001b[0m \u001b[38;5;66;03m# Create a NamedTuple object where the field names are mapping to the graph\u001b[39;00m\n\u001b[0;32m    367\u001b[0m \u001b[38;5;66;03m# output stream names.\u001b[39;00m\n\u001b[0;32m    368\u001b[0m solution_outputs \u001b[38;5;241m=\u001b[39m collections\u001b[38;5;241m.\u001b[39mnamedtuple(\n\u001b[0;32m    369\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSolutionOutputs\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_stream_type_info\u001b[38;5;241m.\u001b[39mkeys())\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "curDir = \"C:/Users/wjdgu/JupyterMain/ActionDetectionforSignLanguage/videos/손\"\n",
    "for file in os.listdir(curDir):\n",
    "    if(file.endswith(\".mp4\")):\n",
    "        path = os.path.join(curDir, file)\n",
    "        custom_video_prediction(path)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11-2. Test in Real Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#한글 텍스트 출력\n",
    "from PIL import ImageFont, ImageDraw, Image\n",
    "\n",
    "def putKoreanText(src, text, pos, font_size, font_color):\n",
    "    img_pil = Image.fromarray(src)\n",
    "    draw = ImageDraw.Draw(img_pil)\n",
    "    font = ImageFont.truetype('C:/Users/User/ActionDetectionforSignLanguage/fonts/gulim.ttc', font_size)\n",
    "    draw.text(pos, text, font=font, fill= font_color)\n",
    "    return np.array(img_pil)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. detection variables\n",
    "sequence = [] #collect 60 frames to make a sequence(=video)\n",
    "sentence = [] #concatenate history of predictions together\n",
    "threshold = 0.995\n",
    "\n",
    "word = ''\n",
    "cnt = 0\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "\n",
    "        # Read feed\n",
    "        ret, frame = cap.read()\n",
    "        # Make detections\n",
    "        image, results = mediapipe_detection(frame, holistic)\n",
    "        print(results)\n",
    "        # Draw landmarks\n",
    "        draw_styled_landmarks(image, results)\n",
    "        \n",
    "        # 2. Prediction logic\n",
    "        keypoints = extract_keypoints(results)\n",
    "        sequence.append(keypoints)\n",
    "        sequence = sequence[-30:] #generate sequence with last 30 frames\n",
    "        \n",
    "        if len(sequence) == 30:\n",
    "            #sequence.shape = (60, 1662)\n",
    "            #the input shape model expects = (number of sequences, 60, 1662)\n",
    "            res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "            print(actions[np.argmax(res)])\n",
    "\n",
    "            if word == actions[np.argmax(res)]:\n",
    "                cnt = cnt + 1\n",
    "            else :\n",
    "                word = actions[np.argmax(res)]\n",
    "                cnt = 1\n",
    "\n",
    "\n",
    "            #3. Rendering logic\n",
    "            #ex. res = [0.1, 0.2, 0.7]\n",
    "            #np.argmax(res) = 2, res[np.argmax(res)] = 0.7\n",
    "            if res[np.argmax(res)] > threshold and cnt >= 10: \n",
    "                #new subtitle added\n",
    "                #frames_without_new_subtitle = 0\n",
    "\n",
    "                cur_action_korean = actions[np.argmax(res)]\n",
    "\n",
    "                if len(sentence) > 0: \n",
    "                    #sentence에 저장된 prediction이 있는 경우 \n",
    "                    #새로운 prediction인 경우에만 sentence에 추가\n",
    "                    if cur_action_korean != sentence[-1]:\n",
    "                        sentence.append(cur_action_korean)\n",
    "                else: \n",
    "                    #sentence에 저장된 prediction 없는 경우 바로 sentence에 추가\n",
    "                    sentence.append(cur_action_korean)\n",
    "\n",
    "            #sentence가 너무 길어지지 않도록 마지막 5개의 prediction만 유지\n",
    "            if len(sentence) > 5: \n",
    "                sentence = sentence[-5:]\n",
    "\n",
    "            #Clear subtitles if needed\n",
    "#                if frames_without_new_subtitle >= clear_cycle:\n",
    "#                    sentence.clear()\n",
    "\n",
    "            #Render subtitles\n",
    "            cv2.rectangle(image, (0,0), (640, 40), (0, 0, 0), -1) \n",
    "#                if target == 'ko':\n",
    "#                    #putKoreanText(src, text, pos, font_size, font_color\n",
    "#                    image = putKoreanText(image, ' '.join(sentence), (3,10), 20, (255, 255, 255))\n",
    "#                else:\n",
    "#                    cv2.putText(image, ' '.join(sentence), (3,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.LINE_AA)\n",
    "            image = putKoreanText(image, ' '.join(sentence), (3,10),(20),(255,255,255))\n",
    "            # Show to screen\n",
    "            cv2.imshow('OpenCV Feed', image)\n",
    "\n",
    "        # Break gracefully\n",
    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "            break\n",
    "            \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}