versions/0.01/1-Collect_Data_Preprocessed-rgb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4e51db7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Task 1 - Collection of data for autowalking\n",
    "#Here we will consider the rgb no processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd27d17a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from grabscreen import grab_screen\n",
    "import cv2\n",
    "import time\n",
    "from getkeys import key_check\n",
    "from getgamepad import gamepad_check\n",
    "import os\n",
    "\n",
    "w = [1,0,0,0,0,0,0,0,0]\n",
    "s = [0,1,0,0,0,0,0,0,0]\n",
    "a = [0,0,1,0,0,0,0,0,0]\n",
    "d = [0,0,0,1,0,0,0,0,0]\n",
    "wa = [0,0,0,0,1,0,0,0,0]\n",
    "wd = [0,0,0,0,0,1,0,0,0]\n",
    "sa = [0,0,0,0,0,0,1,0,0]\n",
    "sd = [0,0,0,0,0,0,0,1,0]\n",
    "nk = [0,0,0,0,0,0,0,0,1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d8eeafbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File does not exist, starting fresh! 1\n"
     ]
    }
   ],
   "source": [
    "starting_value = 1\n",
    "\n",
    "while True:\n",
    "    file_name = 'preprocessed_training_data-{}.npy'.format(starting_value)\n",
    "\n",
    "    if os.path.isfile(file_name):\n",
    "        print('File exists, moving along',starting_value)\n",
    "        starting_value += 1\n",
    "    else:\n",
    "        print('File does not exist, starting fresh!',starting_value)\n",
    "        \n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a76fff26",
   "metadata": {},
   "outputs": [],
   "source": [
    "def keys_to_output(keys):\n",
    "    '''\n",
    "    Convert keys to a ...multi-hot... array\n",
    "     0  1  2  3  4   5   6   7    8\n",
    "    [W, S, A, D, WA, WD, SA, SD, NOKEY] boolean values.\n",
    "    '''\n",
    "    output = [0,0,0,0,0,0,0,0,0]\n",
    "\n",
    "    if 'W' in keys and 'A' in keys:\n",
    "        output = wa\n",
    "    elif 'W' in keys and 'D' in keys:\n",
    "        output = wd\n",
    "    elif 'S' in keys and 'A' in keys:\n",
    "        output = sa\n",
    "    elif 'S' in keys and 'D' in keys:\n",
    "        output = sd\n",
    "    elif 'W' in keys:\n",
    "        output = w\n",
    "    elif 'S' in keys:\n",
    "        output = s\n",
    "    elif 'A' in keys:\n",
    "        output = a\n",
    "    elif 'D' in keys:\n",
    "        output = d\n",
    "    else:\n",
    "        output = nk\n",
    "    return output\n",
    "\n",
    "def gamepad_keys_to_output(lista):\n",
    "    '''\n",
    "    Convert list of boolean to values 0 or 1\n",
    "    ['LT','RT','Lx','Ly','Rx','Ry','UP','DOWN','LEFT','RIGHT','START','SELECT','L3','R3','LB','RB','A','B','X','Y']\n",
    "    example\n",
    "    [0.0,0.0,0.0,-0.1,0.1,-0.0,False,False,False,False,False,False,False,False,False,False,False,False,False,False]\n",
    "    '''\n",
    "    output = [int(elem) for elem in lista]\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8602a5b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Done by Frannecklp\n",
    "\n",
    "import cv2\n",
    "import numpy as np\n",
    "import win32gui, win32ui, win32con, win32api\n",
    "\n",
    "def grab_screen(region=None):\n",
    "\n",
    "    hwin = win32gui.GetDesktopWindow()\n",
    "\n",
    "    if region:\n",
    "            left,top,x2,y2 = region\n",
    "            width = x2 - left + 1\n",
    "            height = y2 - top + 1\n",
    "    else:\n",
    "        width = win32api.GetSystemMetrics(win32con.SM_CXVIRTUALSCREEN)\n",
    "        height = win32api.GetSystemMetrics(win32con.SM_CYVIRTUALSCREEN)\n",
    "        left = win32api.GetSystemMetrics(win32con.SM_XVIRTUALSCREEN)\n",
    "        top = win32api.GetSystemMetrics(win32con.SM_YVIRTUALSCREEN)\n",
    "\n",
    "    hwindc = win32gui.GetWindowDC(hwin)\n",
    "    srcdc = win32ui.CreateDCFromHandle(hwindc)\n",
    "    memdc = srcdc.CreateCompatibleDC()\n",
    "    bmp = win32ui.CreateBitmap()\n",
    "    bmp.CreateCompatibleBitmap(srcdc, width, height)\n",
    "    memdc.SelectObject(bmp)\n",
    "    memdc.BitBlt((0, 0), (width, height), srcdc, (left, top), win32con.SRCCOPY)\n",
    "    \n",
    "    signedIntsArray = bmp.GetBitmapBits(True)\n",
    "    img = np.fromstring(signedIntsArray, dtype='uint8')\n",
    "    img.shape = (height,width,4)\n",
    "\n",
    "    srcdc.DeleteDC()\n",
    "    memdc.DeleteDC()\n",
    "    win32gui.ReleaseDC(hwin, hwindc)\n",
    "    win32gui.DeleteObject(bmp.GetHandle())\n",
    "\n",
    "    return cv2.cvtColor(img, cv2.COLOR_BGRA2RGB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4924e5b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import cv2 as cv\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from numpy import load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7d303d78",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_horizontal(img):\n",
    "    # Median filter is used to suppress image noise. The edge of the signal can be protected from blurring while noise is filtered out.\n",
    "    img = cv.medianBlur(img,9)\n",
    "    # Simple threshold\n",
    "    ret, frame = cv.threshold(img,100,255,cv.THRESH_BINARY) #night/day\n",
    "    # edge detection\n",
    "    processed_img = cv2.Sobel(frame,cv2.CV_64F,0 ,1, ksize=5) #Horizontal Gradient\n",
    "    return processed_img\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4112f6d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_imagen(img):\n",
    "    # Median filter is used to suppress image noise. The edge of the signal can be protected from blurring while noise is filtered out.\n",
    "    img = cv.medianBlur(img,9)\n",
    "    # Simple threshold\n",
    "    ret, frame = cv.threshold(img,100,255,cv.THRESH_BINARY) #night/day\n",
    "    # edge detection\n",
    "    processed_img = cv2.Canny(frame, threshold1 = 200, threshold2=300)\n",
    "   # y= cv2.Sobel(frame,cv2.CV_64F,0 ,1, ksize=5) #Horizontal Gradient\n",
    "    return processed_img\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d2c65d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_red(img):\n",
    "    # Median filter is used to suppress image noise. The edge of the signal can be protected from blurring while noise is filtered out.\n",
    "    img = cv.medianBlur(img,5)\n",
    "    # Simple threshold\n",
    "    #ret,th1 = cv.threshold(img,121,255,cv.THRESH_BINARY) #day\n",
    "    #ret,th1 = cv.threshold(img,61,255,cv.THRESH_BINARY) #night\n",
    "    ret,th1 = cv.threshold(img,100,255,cv.THRESH_BINARY) #night/day\n",
    "    frame=th1\n",
    "    y= cv2.Sobel(frame,cv2.CV_64F,0 ,1, ksize=5) #Horizontal Gradient\n",
    "    #edge= cv2.Canny(frame, 100, 200)#Edge Detection\n",
    "    src =y\n",
    "    #print(src.shape)\n",
    "    # extract red channel\n",
    "    red_channel = src[:,:,2]\n",
    "    # create empty image with same shape as that of src image\n",
    "    red_img = np.zeros(src.shape)\n",
    "    #assign the red channel of src to empty image\n",
    "    red_img[:,:,2] = red_channel\n",
    "    return red_img\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "87e5c4ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "#only day and night\n",
    "def parmeterv(x):\n",
    "    y= 0.8660735375167304*x + 23.147783639083652\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "973ebef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#only day and night and afternoon\n",
    "def parmeterv2(x):\n",
    "    y=-7.79328704433755e-5*x**3 + 0.0262383299518016*x**2 - 1.50672776260312*x + 77.6895731638604\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "159923a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parameter(img):\n",
    "    #img = cv2.imread('way-night.png')\n",
    "    #img = cv2.imread('way-day.png')\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "    oned=img.ravel()\n",
    "    #y, x, _ = plt.hist(oned)\n",
    "    histog=np.histogram(oned)\n",
    "    y, x = histog[0],histog[1]\n",
    "    # Get the maximum element from a Numpy array\n",
    "    y_max = np.amax(y)\n",
    "    # Get the indices of maximum element in numpy array\n",
    "    result = np.where(y == np.amax(y))\n",
    "    position_max=result[0].item()\n",
    "    x_max=x[position_max]\n",
    "    #print(x_max)\n",
    "    #intensity = parmeterv(x_max)\n",
    "    intensity = parmeterv2(x_max)   \n",
    "    return  intensity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3a6040f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_black_contour(img):\n",
    "    image = img\n",
    "    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)   \n",
    "    #thresh = cv2.threshold(gray,90, 255,cv2.THRESH_BINARY_INV)[1] # only for night\n",
    "    #thresh = cv2.threshold(gray,200, 255,cv2.THRESH_BINARY_INV)[1] # only for day  \n",
    "    intensity=parameter(image)\n",
    "    thresh = cv2.threshold(gray,intensity, 255,cv2.THRESH_BINARY_INV)[1] # any time or weather\n",
    "    cnts, h = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n",
    "\n",
    "    label = '1'\n",
    "    count = 0\n",
    "\n",
    "    # Get inner list of hierarchy\n",
    "    for layer in zip(cnts, h[0]):\n",
    "        contour = layer[0]\n",
    "        hierarchy = layer[1]\n",
    "\n",
    "        # If we find new contour (not inner) reset label\n",
    "        if hierarchy[1] >= 0:\n",
    "            label = '1'\n",
    "        # Ensure that we only have outer contour\n",
    "        if count % 2 == 0:\n",
    "            cv2.drawContours(image, [contour], -1, (36, 255, 12), 2)\n",
    "            x,y,w,h = cv2.boundingRect(contour)\n",
    "            cv2.putText(image, label, (x +50,y+ 70), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (36,255,12), 3)\n",
    "            label = str(int(label) * -1)\n",
    "\n",
    "        count += 1\n",
    "    return thresh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0471ff9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from skimage import io\n",
    "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
    "import numpy as np\n",
    "import time\n",
    "import sys\n",
    "def _extract_features(img):\n",
    "    h, w, ch = img.shape\n",
    "    features = img.reshape((h*w, ch))\n",
    "    return features\n",
    "def _clusterize(colors, n_clusters):\n",
    "    t = time.time()\n",
    "    kmeans = MiniBatchKMeans(n_clusters=n_clusters,\n",
    "                             random_state=0,\n",
    "                             batch_size=2**12).fit(colors)\n",
    "    #print('Time to clusterize: {} s'.format(time.time() - t))\n",
    "\n",
    "    return kmeans.labels_, kmeans.cluster_centers_\n",
    "def _reduce_colors(output_shape, labels, labels_colors):\n",
    "    out = np.zeros((labels.size, 3))\n",
    "  \n",
    "    for label, color in enumerate(labels_colors):\n",
    "        out[labels == label] = color\n",
    "\n",
    "    out = out.reshape(output_shape).astype(np.uint8)\n",
    "    return out\n",
    "def color_quantization(img, n_colors=3):\n",
    "    colors = _extract_features(img)\n",
    "    labels, labels_colors = _clusterize(colors, n_colors)\n",
    "    out = _reduce_colors(img.shape, labels, labels_colors)\n",
    "    return  out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "08129221",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(file_name, starting_value):\n",
    "    file_name = file_name\n",
    "    starting_value = starting_value\n",
    "    training_data = []\n",
    "    for i in list(range(4))[::-1]:\n",
    "        print(i+1)\n",
    "        time.sleep(1)\n",
    "\n",
    "    last_time = time.time()\n",
    "    paused = False\n",
    "    print('STARTING!!!')\n",
    "    while(True):\n",
    "        \n",
    "        if not paused:\n",
    "            screen = grab_screen(region=(0,40,1920,1120))\n",
    "\n",
    "            # 800x600 windowed mode\n",
    "            #screen = grab_screen(region=(0,40,800,640))\n",
    "            last_time = time.time()\n",
    "            # resize to something a bit more acceptable for a CNN\n",
    "            screen = cv2.resize(screen, (480,270))\n",
    "            #screen = cv2.resize(screen, (160,120))\n",
    "            # run a color convert:\n",
    "            screen = cv2.cvtColor(screen, cv2.COLOR_BGR2RGB)\n",
    "            \n",
    "        \n",
    "            #screen=process_red(screen)# Only red color\n",
    "            #screen=process_imagen(screen) # Normal\n",
    "            #screen=process_horizontal(screen) # Finding edges\n",
    "            #screen=color_quantization(screen) # Test reduction of colors\n",
    "            #screen = process_black_contour(screen) #Autowalking\n",
    "            \n",
    "            keys = key_check()\n",
    "            #print(keys)\n",
    "            #print(type(keys))\n",
    "            output = keys_to_output(keys)\n",
    "            #print(output)\n",
    "            #print(type(output))\n",
    "            gamepad_keys = gamepad_check()\n",
    "            #print(gamepad_keys)\n",
    "            #print(type(gamepad_keys))\n",
    "            output_gamepad = gamepad_keys_to_output(gamepad_keys)\n",
    "            #print(output_gamepad)\n",
    "            #print(type(output_gamepad))\n",
    "            #training_data.append([screen,output,output_gamepad])  #(Nx3)\n",
    "            \n",
    "            #We have to concatetnate the keys and the gamepad\n",
    "            output=np.concatenate((output, output_gamepad), axis=None) \n",
    "            #then append\n",
    "            training_data.append([screen,output]) #(Nx2)\n",
    "                               \n",
    "            #print('loop took {} seconds'.format(time.time()-last_time))\n",
    "            last_time = time.time()\n",
    "            cv2.imshow('window',cv2.resize(screen,(640,360)))\n",
    "            if cv2.waitKey(25) & 0xFF == ord('q'):\n",
    "                cv2.destroyAllWindows()\n",
    "                break\n",
    "\n",
    "            if len(training_data) % 100 == 0:\n",
    "                print(len(training_data))\n",
    "                \n",
    "                if len(training_data) == 500:\n",
    "                    np.save(file_name,training_data)\n",
    "                    print('SAVED')\n",
    "                    training_data = []\n",
    "                    starting_value += 1\n",
    "                    file_name = 'preprocessed_training_data-{}.npy'.format(starting_value)\n",
    "               \n",
    "        keys = key_check()\n",
    "        if 'T' in keys:\n",
    "            if paused:\n",
    "                paused = False\n",
    "                print('unpaused!')\n",
    "                time.sleep(1)\n",
    "            else:\n",
    "                print('Pausing!')\n",
    "                paused = True\n",
    "                time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "55a33508",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "3\n",
      "2\n",
      "1\n",
      "STARTING!!!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\envs\\pygta5\\lib\\site-packages\\ipykernel_launcher.py:30: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\envs\\pygta5\\lib\\site-packages\\numpy\\lib\\npyio.py:528: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  arr = np.asanyarray(arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "SAVED\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "main(file_name, starting_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d07ea952",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (pygta5)",
   "language": "python",
   "name": "pygta5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}