Add files via upload

Author: nishagandhi

Course_Demo.mkv

@@ -0,0 +1,8 @@
+Steps to run the driver drowsiness detection project (CSE 547 Discrete Mathematics):
+
+1) Run the run_extract_eyes.sh program to track the eyes for different videos(training data) and to store the patches of the eyes in a folder for every video. (Alert and Drowsy)
+2) Use this training data to retrain the CNN model(Inception V3 model).
+3) Run extract_features.py to extract the features from the second last layer of the CNN model which is a 2048-d vector and to create a sequence of frames as a single vector to be given as an input to the LSTM which is a part of Recurrent Neural Networks(RNN) 
+4) Run data.py and models.py
+5) Finally run train.py to get the final predictions for the test sequence of data and the alarm will sound if the model predicts the sequence to be in a drowsy state.
+ 

Drowsy-Driver-Detection.pdf

@@ -0,0 +1,86 @@
+import csv
+import csv
+import numpy as np
+import random
+import glob
+import os.path
+import sys
+import operator
+import threading
+from keras.utils import to_categorical
+
+class Dataset():
+
+	def __init__(self, seq_length=26, class_limit=2, image_shape=(56, 24, 3)):
+	        self.seq_length = seq_length
+	        self.class_limit = class_limit
+	        self.sequence_path = os.path.join('data', 'sequences')
+	        # Get the data.
+	        self.data = self.get_data()
+		# Get the classes.
+	        self.classes = self.get_classes()
+
+	def get_data(self):
+		with open(os.path.join('data', 'data_file.csv'), 'r') as fin:
+	        	reader = csv.reader(fin)
+            		data = list(reader)
+
+        	return data
+
+
+	def get_classes(self):
+		classes = []
+		for item in self.data:
+		    if item[1] not in classes:
+		        classes.append(item[1])
+
+		# Sort them.
+		classes = sorted(classes)
+
+		# Return.
+		if self.class_limit is not None:
+			return classes[:self.class_limit]
+		else:
+			return classes
+
+	def get_class_one_hot(self, class_str):
+        	# Encode it first.
+        	label_encoded = self.classes.index(class_str)
+
+        	# Now one-hot it.
+        	label_hot = to_categorical(label_encoded, len(self.classes))
+
+        	assert len(label_hot) == len(self.classes)
+
+		return label_hot
+
+	def get_all_sequences_in_memory(self, train_test):
+
+                #train, test = self.split_train_test()
+		#data = train if train_test == 'train' else test
+
+		print"Loading samples into memory for --> ",train_test
+
+		X, y = [], []
+
+		for videos in self.data:
+					if(videos[0] == train_test):
+						sequence = self.get_extracted_sequence(videos)
+						if sequence is None:
+							print("Can't find sequence. Did you generate them?")
+							raise
+						X.append(sequence)
+						y.append(self.get_class_one_hot(videos[1]))
+
+        	return np.array(X), np.array(y)
+
+
+	def get_extracted_sequence(self,video):
+		"""Get the saved extracted features."""
+	        filename = video[2]
+	        path = os.path.join(self.sequence_path, filename + '-' + str(26) + \
+	            '-' + 'features' + '.npy')
+	        if os.path.isfile(path):
+	        	return np.load(path)
+	        else:
+			return None

Readme

@@ -0,0 +1,19 @@
+training,Drowsy,Drowsy_Video1,26
+training,Drowsy,Drowsy_Video2,26
+training,Drowsy,Drowsy_Video3,26
+training,Drowsy,Drowsy_Video4,26
+training,Drowsy,Drowsy_Video5,26
+training,Drowsy,Drowsy_Video6,26
+training,Drowsy,Drowsy_Video7,26
+training,Drowsy,Drowsy_Video8,26
+training,Alert,Alert_Video1,26
+training,Alert,Alert_Video2,26
+training,Alert,Alert_Video3,26
+training,Alert,Alert_Video4,26
+training,Alert,Alert_Video5,26
+training,Alert,Alert_Video6,26
+training,Alert,Alert_Video7,26
+training,Alert,Alert_Video8,26
+testing,Alert,Test_Alert1,26
+testing,Drowsy,Test_Drowsy1,26
+testing,Alert,Test_Alert2,26

TensorBoard.png

@@ -0,0 +1,51 @@
+import numpy as np
+import os.path
+import csv
+import glob
+import tensorflow as tf
+import h5py as h5py
+from keras.preprocessing import image
+from keras.applications.inception_v3 import InceptionV3, preprocess_input
+from keras.models import Model, load_model
+
+def extractor(image_path):
+
+	with open('/home/tejas/Desktop/Course-Project-CV/output_graph.pb', 'rb') as graph_file:
+		graph_def = tf.GraphDef()
+        	graph_def.ParseFromString(graph_file.read())
+    		tf.import_graph_def(graph_def, name='')
+
+	with tf.Session() as sess:
+	    pooling_tensor = sess.graph.get_tensor_by_name('pool_3:0')
+	    image_data = tf.gfile.FastGFile(image_path, 'rb').read()	
+	    pooling_features = sess.run(pooling_tensor, \
+                 {'DecodeJpeg/contents:0': image_data})
+            pooling_features = pooling_features[0]
+	    
+	return pooling_features
+
+def extract_features():
+	with open('data/data_file.csv','r') as f:
+		reader = csv.reader(f)
+		for videos in reader:
+			path = os.path.join('data', 'sequences', videos[2] + '-' + str(26) + \
+'-features.npy')
+		
+			path_frames = os.path.join('data', videos[0], videos[1])
+        		filename = videos[2]
+			frames = sorted(glob.glob(os.path.join(path_frames, filename + '/*jpg')))
+		
+			sequence = []
+			for image in frames:
+				with tf.Graph().as_default():
+					features = extractor(image)
+					print 'Appending sequence of image:',image,' of the video:',videos				
+				  
+                   
+              		             																																																																																																																																			
+					sequence.append(features)
+
+			np.save(path,sequence)
+			print 'Sequences saved successfully'						
+
+extract_features()																																																																				

data.py

@@ -0,0 +1,137 @@
+import os
+import sys
+import cv2
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
+eye_cascade_left = cv2.CascadeClassifier('haarcascade_lefteye_2splits.xml')
+eye_cascade_right = cv2.CascadeClassifier('haarcascade_righteye_2splits.xml')
+def help_message():
+   print("Usage: [Question_Number] [Input_Video] [Output_Directory]")
+   print("[Question Number]")
+   print("1 Camshift")
+   print("[Input_Video]")
+   print("Path to the input video")
+   print("[Output_Directory]")
+   print("Output directory")
+   print("Example usages:")
+   print(sys.argv[0] + " 1 " + "02-1.avi " + "./")
+
+
+def detect_one_face(im):
+    gray=cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
+
+    faces = face_cascade.detectMultiScale(gray, 1.2, 3)
+    if len(faces) == 0:
+        return (0,0,0,0)
+    return faces[0]
+
+def hsv_histogram_for_window(frame, window):
+    # set up the ROI for tracking
+    c,r,w,h = window
+    roi = frame[r:r+h, c:c+w]
+    hsv_roi =  cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)
+    mask = cv2.inRange(hsv_roi, np.array((0., 60.,32.)), np.array((180.,255.,255.)))
+    roi_hist = cv2.calcHist([hsv_roi],[0],mask,[180],[0,180])
+    cv2.normalize(roi_hist,roi_hist,0,255,cv2.NORM_MINMAX)
+    return roi_hist
+
+
+def resample(weights):
+    n = len(weights)
+    indices = []
+    C = [0.] + [sum(weights[:i+1]) for i in range(n)]
+    u0, j = np.random.random(), 0
+    for u in [(u0+i)/n for i in range(n)]:
+      while u > C[j]:
+          j+=1
+      indices.append(j-1)
+    return indices
+
+# Camshift
+
+def skeleton_tracker1(v, file_name):
+    # Open output file
+    output_name = sys.argv[3] + file_name
+    output = open(output_name,"w")
+
+
+    frameCounter = 0
+    # read first frame
+    ret ,frame = v.read()
+    if ret == False:
+        return
+
+
+    # detect face in first frame
+    c,r,w,h = detect_one_face(frame)
+    pt = (0,c+w/2,r+h/2)
+    # Write track point for first frame
+    output.write("%d,%d,%d\n" % pt) # Write as 0,pt_x,pt_y
+    frameCounter = frameCounter + 1
+
+    # set the initial tracking window
+    track_window = (c,r,w,h)
+
+    # calculate the HSV histogram in the window
+    # NOTE: you do not need this in the Kalman, Particle or OF trackers
+    roi_hist = hsv_histogram_for_window(frame, (c,r,w,h)) # this is provided for you
+    count = 0
+    term_crit = ( cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 1 )
+    while(1):
+        ret ,frame = v.read() # read another frame
+        if ret == False:
+            break
+        hsv = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
+        dst = cv2.calcBackProject([hsv],[0],roi_hist,[0,180],1)
+        ret, track_window = cv2.CamShift(dst, track_window, term_crit)
+        x,y,w,h = track_window
+        # write the result to the output file
+        pt = (frameCounter,x+w/2,y+h/2)
+        output.write("%d,%d,%d\n" % pt) # Write as frame_index,pt_x,pt_y
+        cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        roi_gray = gray[y:y+h, x:x+w]
+        roi_color = frame[y:y+h, x:x+w]
+
+        eyes_left = eye_cascade_left.detectMultiScale(roi_gray)
+        eyes_right = eye_cascade_right.detectMultiScale(roi_gray)
+        for (ex1,ey1,ew1,eh1) in eyes_left:
+            break
+        for (ex,ey,ew,eh) in eyes_right:
+            break
+
+        frame[ey+eh:ey, ex:ex1+ew1]
+        cv2.rectangle(roi_color,(ex-5,ey-5),(ex+(ex1-ex)+ew1+5,ey+eh+5),(0,255,0),2)
+
+        croppedImg = roi_color[ey1:ey1+eh1, ex:ex1+ew1]
+        if(croppedImg.shape[0]<=0 or croppedImg.shape[1]<=0):
+            frameCounter = frameCounter + 1
+            continue
+        print croppedImg.shape
+        cv2.imshow('img',frame)
+        cv2.imshow('img1',croppedImg)
+        output_name = "./"+sys.argv[4]+str(frameCounter)+".jpg"
+        cv2.imwrite(output_name, croppedImg)
+        k = cv2.waitKey(1) & 0xff
+        if k == 27:
+            break
+        frameCounter = frameCounter + 1
+    output.close()
+
+
+if __name__ == '__main__':
+    question_number = -1
+
+    question_number = int(sys.argv[1])
+    if (question_number > 4 or question_number < 1):
+        print("Input parameters out of bound ...")
+        sys.exit()
+
+    # read video file
+    video = cv2.VideoCapture(sys.argv[2]);
+    if (question_number == 1):
+        skeleton_tracker1(video, "output_camshift.txt")
+

data.pyc

@@ -0,0 +1,55 @@
+from keras.layers import Dense, Flatten, Dropout, ZeroPadding3D
+from keras.layers.recurrent import LSTM
+from keras.models import Sequential, load_model
+from keras.optimizers import Adam, RMSprop
+from keras.layers.wrappers import TimeDistributed
+from keras.layers.convolutional import (Conv2D, MaxPooling3D, Conv3D,
+    MaxPooling2D)
+from collections import deque
+import sys
+
+class ResearchModels():
+    def __init__(self, nb_classes, model, seq_length,
+                 saved_model=None, features_length=2048):
+
+        # Set defaults.
+        self.seq_length = seq_length
+        self.load_model = load_model
+        self.saved_model = saved_model
+        self.nb_classes = nb_classes
+        self.feature_queue = deque()
+
+        # Set the metrics. Only use top k if there's a need.
+        metrics = ['accuracy']
+        #if self.nb_classes >= 10:
+        #   metrics.append('top_k_categorical_accuracy')
+
+        # Get the appropriate model.
+        print("Loading LSTM model.")
+        self.input_shape = (seq_length, features_length)
+        self.model = self.lstm()
+        # Now compile the network.
+        optimizer = Adam(lr=0.00005)
+        self.model.compile(loss='categorical_crossentropy', optimizer=optimizer,
+                           metrics=metrics)
+	
+        print(self.model.summary())
+
+    def lstm(self):
+        """Build a simple LSTM network. We pass the extracted features from
+        our CNN to this model predomenently."""
+        # Model.
+	print self.input_shape
+        model = Sequential()
+        model.add(LSTM(2048, return_sequences=True,
+                       input_shape=self.input_shape,
+                       dropout=0.5))
+	model.add(Flatten())
+        model.add(Dense(1024, activation='relu'))
+        model.add(Dense(512, activation='relu'))
+        model.add(Dropout(0.5))
+        model.add(Dense(self.nb_classes, activation='softmax'))
+	
+        return model
+
+    

data_file.csv

@@ -0,0 +1,2 @@
+alert eyes
+drowsy eyes