notebook/prepare_lgb_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>guid/image</th>\n",
       "      <th>gt_label</th>\n",
       "      <th>gt_ymin</th>\n",
       "      <th>gt_xmin</th>\n",
       "      <th>gt_ymax</th>\n",
       "      <th>gt_xmax</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bc097e0a-99bf-438d-9826-5ba0f228ea34/0074</td>\n",
       "      <td>1</td>\n",
       "      <td>406</td>\n",
       "      <td>1488</td>\n",
       "      <td>476</td>\n",
       "      <td>1750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f075920b-5e7a-4c75-8ce8-3368d5f4254b/0033</td>\n",
       "      <td>1</td>\n",
       "      <td>409</td>\n",
       "      <td>667</td>\n",
       "      <td>442</td>\n",
       "      <td>700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bc097e0a-99bf-438d-9826-5ba0f228ea34/0008</td>\n",
       "      <td>1</td>\n",
       "      <td>437</td>\n",
       "      <td>0</td>\n",
       "      <td>495</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6b08aacf-49fe-445b-8085-41d09f937be5/0081</td>\n",
       "      <td>2</td>\n",
       "      <td>398</td>\n",
       "      <td>717</td>\n",
       "      <td>428</td>\n",
       "      <td>745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65725892-b191-4d0c-831c-5f4f84a3b0a9/0086</td>\n",
       "      <td>1</td>\n",
       "      <td>432</td>\n",
       "      <td>1687</td>\n",
       "      <td>499</td>\n",
       "      <td>1914</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  guid/image  gt_label  gt_ymin  gt_xmin  \\\n",
       "0  bc097e0a-99bf-438d-9826-5ba0f228ea34/0074         1      406     1488   \n",
       "1  f075920b-5e7a-4c75-8ce8-3368d5f4254b/0033         1      409      667   \n",
       "2  bc097e0a-99bf-438d-9826-5ba0f228ea34/0008         1      437        0   \n",
       "3  6b08aacf-49fe-445b-8085-41d09f937be5/0081         2      398      717   \n",
       "4  65725892-b191-4d0c-831c-5f4f84a3b0a9/0086         1      432     1687   \n",
       "\n",
       "   gt_ymax  gt_xmax  \n",
       "0      476     1750  \n",
       "1      442      700  \n",
       "2      495      211  \n",
       "3      428      745  \n",
       "4      499     1914  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "IMG_WIDTH = 1914\n",
    "IMG_HEIGHT = 1052\n",
    "\n",
    "train_labels = pd.read_csv('bbox_features/train_labels.csv',usecols=[0,3,4,5,6,7])\n",
    "test_labels = pd.read_csv('bbox_features/test_labels.csv',usecols=[0,3,4,5,6,7])\n",
    "\n",
    "self_train_test = train_labels.append(test_labels, ignore_index=True)\n",
    "self_train_test.columns = ['guid/image','gt_label','gt_xmin','gt_ymin','gt_xmax','gt_ymax']\n",
    "self_train_test['guid/image'] = self_train_test['guid/image'].apply(lambda x : x[:36]+'/'+x[37:41])\n",
    "gt = self_train_test[['guid/image','gt_label','gt_ymin','gt_xmin','gt_ymax','gt_xmax']]\n",
    "gt['gt_label'] = gt['gt_label'].apply(lambda x : int(x[5]))\n",
    "gt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>guid/image</th>\n",
       "      <th>cocov1-can1-label</th>\n",
       "      <th>cocov1-can1-score</th>\n",
       "      <th>cocov1-can1-ymin</th>\n",
       "      <th>cocov1-can1-xmin</th>\n",
       "      <th>cocov1-can1-ymax</th>\n",
       "      <th>cocov1-can1-xmax</th>\n",
       "      <th>cocov1-can1-box-width</th>\n",
       "      <th>cocov1-can1-box-height</th>\n",
       "      <th>cocov1-can1-box-ratio</th>\n",
       "      <th>cocov1-can1-box-area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eb46684b-f8c9-455e-80ff-1e168d7d2f7c/0050</td>\n",
       "      <td>1</td>\n",
       "      <td>0.975790</td>\n",
       "      <td>362.343965</td>\n",
       "      <td>1699.528092</td>\n",
       "      <td>418.631981</td>\n",
       "      <td>1894.136616</td>\n",
       "      <td>194.608524</td>\n",
       "      <td>56.288016</td>\n",
       "      <td>0.289237</td>\n",
       "      <td>10954.127733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af3ae5e6-27ef-4699-beb3-f2c72831a594/0175</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998079</td>\n",
       "      <td>396.512519</td>\n",
       "      <td>649.628415</td>\n",
       "      <td>603.140260</td>\n",
       "      <td>940.874612</td>\n",
       "      <td>291.246197</td>\n",
       "      <td>206.627741</td>\n",
       "      <td>0.709461</td>\n",
       "      <td>60179.543850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2a279bda-651c-4740-a616-6862aa904c24/0071</td>\n",
       "      <td>1</td>\n",
       "      <td>0.820020</td>\n",
       "      <td>412.332509</td>\n",
       "      <td>802.611483</td>\n",
       "      <td>452.656441</td>\n",
       "      <td>857.709605</td>\n",
       "      <td>55.098122</td>\n",
       "      <td>40.323932</td>\n",
       "      <td>0.731857</td>\n",
       "      <td>2221.772926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>630f5200-f86e-490f-8764-1ecf1d398f5b/0144</td>\n",
       "      <td>2</td>\n",
       "      <td>0.980499</td>\n",
       "      <td>384.796856</td>\n",
       "      <td>907.498804</td>\n",
       "      <td>561.867051</td>\n",
       "      <td>1013.929615</td>\n",
       "      <td>106.430811</td>\n",
       "      <td>177.070195</td>\n",
       "      <td>1.663712</td>\n",
       "      <td>18845.724458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e01aefe6-f367-4c00-b006-f9327a1ffc5f/0012</td>\n",
       "      <td>1</td>\n",
       "      <td>0.274682</td>\n",
       "      <td>406.261092</td>\n",
       "      <td>685.235806</td>\n",
       "      <td>482.196054</td>\n",
       "      <td>786.925202</td>\n",
       "      <td>101.689396</td>\n",
       "      <td>75.934962</td>\n",
       "      <td>0.746734</td>\n",
       "      <td>7721.780390</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  guid/image  cocov1-can1-label  \\\n",
       "0  eb46684b-f8c9-455e-80ff-1e168d7d2f7c/0050                  1   \n",
       "1  af3ae5e6-27ef-4699-beb3-f2c72831a594/0175                  1   \n",
       "2  2a279bda-651c-4740-a616-6862aa904c24/0071                  1   \n",
       "3  630f5200-f86e-490f-8764-1ecf1d398f5b/0144                  2   \n",
       "4  e01aefe6-f367-4c00-b006-f9327a1ffc5f/0012                  1   \n",
       "\n",
       "   cocov1-can1-score  cocov1-can1-ymin  cocov1-can1-xmin  cocov1-can1-ymax  \\\n",
       "0           0.975790        362.343965       1699.528092        418.631981   \n",
       "1           0.998079        396.512519        649.628415        603.140260   \n",
       "2           0.820020        412.332509        802.611483        452.656441   \n",
       "3           0.980499        384.796856        907.498804        561.867051   \n",
       "4           0.274682        406.261092        685.235806        482.196054   \n",
       "\n",
       "   cocov1-can1-xmax  cocov1-can1-box-width  cocov1-can1-box-height  \\\n",
       "0       1894.136616             194.608524               56.288016   \n",
       "1        940.874612             291.246197              206.627741   \n",
       "2        857.709605              55.098122               40.323932   \n",
       "3       1013.929615             106.430811              177.070195   \n",
       "4        786.925202             101.689396               75.934962   \n",
       "\n",
       "   cocov1-can1-box-ratio  cocov1-can1-box-area  \n",
       "0               0.289237          10954.127733  \n",
       "1               0.709461          60179.543850  \n",
       "2               0.731857           2221.772926  \n",
       "3               1.663712          18845.724458  \n",
       "4               0.746734           7721.780390  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ast import literal_eval\n",
    "\n",
    "cocov1_self_train = pd.read_csv('tf_features/cocov1-steps-201k-self-train.tsv',sep='\\t')\n",
    "cocov1_self_test = pd.read_csv('tf_features/cocov1-steps-201k-self-test.tsv',sep='\\t')\n",
    "cocov1 = cocov1_self_train.append(cocov1_self_test, ignore_index=True)\n",
    "\n",
    "cocov1['cocov1-can1-label'] = cocov1['labels'].apply(lambda x : int(x[1]))\n",
    "cocov1['cocov1-can1-score'] = cocov1['scores'].apply(lambda x : literal_eval(x)[0])\n",
    "cocov1['cocov1-can1-ymin'] = cocov1['boxes'].apply(lambda x : IMG_HEIGHT*literal_eval(x)[0][0])\n",
    "cocov1['cocov1-can1-xmin'] = cocov1['boxes'].apply(lambda x : IMG_WIDTH*literal_eval(x)[0][1])\n",
    "cocov1['cocov1-can1-ymax'] = cocov1['boxes'].apply(lambda x : IMG_HEIGHT*literal_eval(x)[0][2])\n",
    "cocov1['cocov1-can1-xmax'] = cocov1['boxes'].apply(lambda x : IMG_WIDTH*literal_eval(x)[0][3])\n",
    "cocov1['cocov1-can1-box-width'] = cocov1['cocov1-can1-xmax'] - cocov1['cocov1-can1-xmin']\n",
    "cocov1['cocov1-can1-box-height'] = cocov1['cocov1-can1-ymax'] - cocov1['cocov1-can1-ymin']\n",
    "cocov1['cocov1-can1-box-ratio'] = cocov1['cocov1-can1-box-height']/cocov1['cocov1-can1-box-width']\n",
    "cocov1['cocov1-can1-box-area'] = cocov1['cocov1-can1-box-height']*cocov1['cocov1-can1-box-width']\n",
    "\n",
    "\n",
    "cocov1 = cocov1.drop(['labels','scores','boxes'],axis=1)\n",
    "cocov1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>guid/image</th>\n",
       "      <th>kitti-can1-label</th>\n",
       "      <th>kitti-can1-score</th>\n",
       "      <th>kitti-can1-ymin</th>\n",
       "      <th>kitti-can1-xmin</th>\n",
       "      <th>kitti-can1-ymax</th>\n",
       "      <th>kitti-can1-xmax</th>\n",
       "      <th>kitti-can1-box-width</th>\n",
       "      <th>kitti-can1-box-height</th>\n",
       "      <th>kitti-can1-box-ratio</th>\n",
       "      <th>kitti-can1-box-area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eb46684b-f8c9-455e-80ff-1e168d7d2f7c/0050</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>362.778912</td>\n",
       "      <td>1692.372902</td>\n",
       "      <td>420.560069</td>\n",
       "      <td>1892.685477</td>\n",
       "      <td>200.312575</td>\n",
       "      <td>57.781157</td>\n",
       "      <td>0.288455</td>\n",
       "      <td>11574.292344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af3ae5e6-27ef-4699-beb3-f2c72831a594/0175</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>396.934329</td>\n",
       "      <td>644.185045</td>\n",
       "      <td>603.435534</td>\n",
       "      <td>937.336775</td>\n",
       "      <td>293.151731</td>\n",
       "      <td>206.501204</td>\n",
       "      <td>0.704417</td>\n",
       "      <td>60536.185374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2a279bda-651c-4740-a616-6862aa904c24/0071</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>415.893474</td>\n",
       "      <td>806.525509</td>\n",
       "      <td>451.183522</td>\n",
       "      <td>854.433418</td>\n",
       "      <td>47.907909</td>\n",
       "      <td>35.290048</td>\n",
       "      <td>0.736623</td>\n",
       "      <td>1690.672391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>630f5200-f86e-490f-8764-1ecf1d398f5b/0144</td>\n",
       "      <td>2</td>\n",
       "      <td>0.999962</td>\n",
       "      <td>380.184312</td>\n",
       "      <td>911.494514</td>\n",
       "      <td>563.327743</td>\n",
       "      <td>1011.196978</td>\n",
       "      <td>99.702464</td>\n",
       "      <td>183.143430</td>\n",
       "      <td>1.836900</td>\n",
       "      <td>18259.851258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e01aefe6-f367-4c00-b006-f9327a1ffc5f/0012</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999994</td>\n",
       "      <td>412.813512</td>\n",
       "      <td>691.069227</td>\n",
       "      <td>481.966588</td>\n",
       "      <td>788.431329</td>\n",
       "      <td>97.362102</td>\n",
       "      <td>69.153076</td>\n",
       "      <td>0.710267</td>\n",
       "      <td>6732.888875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  guid/image  kitti-can1-label  \\\n",
       "0  eb46684b-f8c9-455e-80ff-1e168d7d2f7c/0050                 1   \n",
       "1  af3ae5e6-27ef-4699-beb3-f2c72831a594/0175                 1   \n",
       "2  2a279bda-651c-4740-a616-6862aa904c24/0071                 1   \n",
       "3  630f5200-f86e-490f-8764-1ecf1d398f5b/0144                 2   \n",
       "4  e01aefe6-f367-4c00-b006-f9327a1ffc5f/0012                 1   \n",
       "\n",
       "   kitti-can1-score  kitti-can1-ymin  kitti-can1-xmin  kitti-can1-ymax  \\\n",
       "0          0.999999       362.778912      1692.372902       420.560069   \n",
       "1          1.000000       396.934329       644.185045       603.435534   \n",
       "2          1.000000       415.893474       806.525509       451.183522   \n",
       "3          0.999962       380.184312       911.494514       563.327743   \n",
       "4          0.999994       412.813512       691.069227       481.966588   \n",
       "\n",
       "   kitti-can1-xmax  kitti-can1-box-width  kitti-can1-box-height  \\\n",
       "0      1892.685477            200.312575              57.781157   \n",
       "1       937.336775            293.151731             206.501204   \n",
       "2       854.433418             47.907909              35.290048   \n",
       "3      1011.196978             99.702464             183.143430   \n",
       "4       788.431329             97.362102              69.153076   \n",
       "\n",
       "   kitti-can1-box-ratio  kitti-can1-box-area  \n",
       "0              0.288455         11574.292344  \n",
       "1              0.704417         60536.185374  \n",
       "2              0.736623          1690.672391  \n",
       "3              1.836900         18259.851258  \n",
       "4              0.710267          6732.888875  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kitti_self_train = pd.read_csv('tf_features/kitti-steps-660k-self-train.tsv',sep='\\t')\n",
    "kitti_self_test = pd.read_csv('tf_features/kitti-steps-660k-self-test.tsv',sep='\\t')\n",
    "kitti = kitti_self_train.append(kitti_self_test, ignore_index=True)\n",
    "\n",
    "kitti['kitti-can1-label'] = kitti['labels'].apply(lambda x : int(x[1]))\n",
    "kitti['kitti-can1-score'] = kitti['scores'].apply(lambda x : literal_eval(x)[0])\n",
    "kitti['kitti-can1-ymin'] = kitti['boxes'].apply(lambda x : IMG_HEIGHT*literal_eval(x)[0][0])\n",
    "kitti['kitti-can1-xmin'] = kitti['boxes'].apply(lambda x : IMG_WIDTH*literal_eval(x)[0][1])\n",
    "kitti['kitti-can1-ymax'] = kitti['boxes'].apply(lambda x : IMG_HEIGHT*literal_eval(x)[0][2])\n",
    "kitti['kitti-can1-xmax'] = kitti['boxes'].apply(lambda x : IMG_WIDTH*literal_eval(x)[0][3])\n",
    "\n",
    "kitti['kitti-can1-box-width'] = kitti['kitti-can1-xmax'] - kitti['kitti-can1-xmin']\n",
    "kitti['kitti-can1-box-height'] = kitti['kitti-can1-ymax'] - kitti['kitti-can1-ymin']\n",
    "kitti['kitti-can1-box-ratio'] = kitti['kitti-can1-box-height']/kitti['kitti-can1-box-width']\n",
    "kitti['kitti-can1-box-area'] = kitti['kitti-can1-box-height']*kitti['kitti-can1-box-width']\n",
    "\n",
    "kitti = kitti.drop(['labels','scores','boxes'],axis=1)\n",
    "kitti.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>guid/image</th>\n",
       "      <th>incep-can1-label</th>\n",
       "      <th>incep-can1-score</th>\n",
       "      <th>incep-can1-ymin</th>\n",
       "      <th>incep-can1-xmin</th>\n",
       "      <th>incep-can1-ymax</th>\n",
       "      <th>incep-can1-xmax</th>\n",
       "      <th>incep-can1-box-width</th>\n",
       "      <th>incep-can1-box-height</th>\n",
       "      <th>incep-can1-box-ratio</th>\n",
       "      <th>incep-can1-box-area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eb46684b-f8c9-455e-80ff-1e168d7d2f7c/0050</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999855</td>\n",
       "      <td>363.840868</td>\n",
       "      <td>1695.449843</td>\n",
       "      <td>421.730410</td>\n",
       "      <td>1893.519882</td>\n",
       "      <td>198.070039</td>\n",
       "      <td>57.889541</td>\n",
       "      <td>0.292268</td>\n",
       "      <td>11466.183689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af3ae5e6-27ef-4699-beb3-f2c72831a594/0175</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999989</td>\n",
       "      <td>398.162358</td>\n",
       "      <td>645.912779</td>\n",
       "      <td>604.717519</td>\n",
       "      <td>936.441164</td>\n",
       "      <td>290.528385</td>\n",
       "      <td>206.555161</td>\n",
       "      <td>0.710964</td>\n",
       "      <td>60010.137421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2a279bda-651c-4740-a616-6862aa904c24/0071</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999995</td>\n",
       "      <td>417.145330</td>\n",
       "      <td>806.700171</td>\n",
       "      <td>449.917056</td>\n",
       "      <td>855.081982</td>\n",
       "      <td>48.381811</td>\n",
       "      <td>32.771726</td>\n",
       "      <td>0.677356</td>\n",
       "      <td>1585.555460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>630f5200-f86e-490f-8764-1ecf1d398f5b/0144</td>\n",
       "      <td>2</td>\n",
       "      <td>0.999981</td>\n",
       "      <td>380.780816</td>\n",
       "      <td>910.119183</td>\n",
       "      <td>565.098694</td>\n",
       "      <td>1013.076158</td>\n",
       "      <td>102.956975</td>\n",
       "      <td>184.317878</td>\n",
       "      <td>1.790242</td>\n",
       "      <td>18976.811143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e01aefe6-f367-4c00-b006-f9327a1ffc5f/0012</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999938</td>\n",
       "      <td>413.273039</td>\n",
       "      <td>690.567603</td>\n",
       "      <td>481.418743</td>\n",
       "      <td>789.933692</td>\n",
       "      <td>99.366089</td>\n",
       "      <td>68.145704</td>\n",
       "      <td>0.685804</td>\n",
       "      <td>6771.372080</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  guid/image  incep-can1-label  \\\n",
       "0  eb46684b-f8c9-455e-80ff-1e168d7d2f7c/0050                 1   \n",
       "1  af3ae5e6-27ef-4699-beb3-f2c72831a594/0175                 1   \n",
       "2  2a279bda-651c-4740-a616-6862aa904c24/0071                 1   \n",
       "3  630f5200-f86e-490f-8764-1ecf1d398f5b/0144                 2   \n",
       "4  e01aefe6-f367-4c00-b006-f9327a1ffc5f/0012                 1   \n",
       "\n",
       "   incep-can1-score  incep-can1-ymin  incep-can1-xmin  incep-can1-ymax  \\\n",
       "0          0.999855       363.840868      1695.449843       421.730410   \n",
       "1          0.999989       398.162358       645.912779       604.717519   \n",
       "2          0.999995       417.145330       806.700171       449.917056   \n",
       "3          0.999981       380.780816       910.119183       565.098694   \n",
       "4          0.999938       413.273039       690.567603       481.418743   \n",
       "\n",
       "   incep-can1-xmax  incep-can1-box-width  incep-can1-box-height  \\\n",
       "0      1893.519882            198.070039              57.889541   \n",
       "1       936.441164            290.528385             206.555161   \n",
       "2       855.081982             48.381811              32.771726   \n",
       "3      1013.076158            102.956975             184.317878   \n",
       "4       789.933692             99.366089              68.145704   \n",
       "\n",
       "   incep-can1-box-ratio  incep-can1-box-area  \n",
       "0              0.292268         11466.183689  \n",
       "1              0.710964         60010.137421  \n",
       "2              0.677356          1585.555460  \n",
       "3              1.790242         18976.811143  \n",
       "4              0.685804          6771.372080  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incep_self_train = pd.read_csv('tf_features/incep-steps-200k-self-train.tsv',sep='\\t')\n",
    "incep_self_test = pd.read_csv('tf_features/incep-steps-200k-self-test.tsv',sep='\\t')\n",
    "incep = incep_self_train.append(incep_self_test, ignore_index=True)\n",
    "\n",
    "incep['incep-can1-label'] = incep['labels'].apply(lambda x : int(x[1]))\n",
    "incep['incep-can1-score'] = incep['scores'].apply(lambda x : literal_eval(x)[0])\n",
    "incep['incep-can1-ymin'] = incep['boxes'].apply(lambda x : IMG_HEIGHT*literal_eval(x)[0][0])\n",
    "incep['incep-can1-xmin'] = incep['boxes'].apply(lambda x : IMG_WIDTH*literal_eval(x)[0][1])\n",
    "incep['incep-can1-ymax'] = incep['boxes'].apply(lambda x : IMG_HEIGHT*literal_eval(x)[0][2])\n",
    "incep['incep-can1-xmax'] = incep['boxes'].apply(lambda x : IMG_WIDTH*literal_eval(x)[0][3])\n",
    "\n",
    "incep['incep-can1-box-width'] = incep['incep-can1-xmax'] - incep['incep-can1-xmin']\n",
    "incep['incep-can1-box-height'] = incep['incep-can1-ymax'] - incep['incep-can1-ymin']\n",
    "incep['incep-can1-box-ratio'] = incep['incep-can1-box-height']/incep['incep-can1-box-width']\n",
    "incep['incep-can1-box-area'] = incep['incep-can1-box-height']*incep['incep-can1-box-width']\n",
    "\n",
    "incep = incep.drop(['labels','scores','boxes'],axis=1)\n",
    "incep.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_cocov1 = gt.merge(cocov1,how='left',on='guid/image',sort=True)\n",
    "gt_cocov1_kitti = gt_cocov1.merge(kitti,how='left',on='guid/image',sort=True)\n",
    "gt_cocov1_kitti_incep = gt_cocov1_kitti.merge(incep,how='left',on='guid/image',sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>guid/image</th>\n",
       "      <th>gt_label</th>\n",
       "      <th>gt_ymin</th>\n",
       "      <th>gt_xmin</th>\n",
       "      <th>gt_ymax</th>\n",
       "      <th>gt_xmax</th>\n",
       "      <th>cocov1-can1-label</th>\n",
       "      <th>cocov1-can1-score</th>\n",
       "      <th>cocov1-can1-ymin</th>\n",
       "      <th>cocov1-can1-xmin</th>\n",
       "      <th>...</th>\n",
       "      <th>incep-can1-label</th>\n",
       "      <th>incep-can1-score</th>\n",
       "      <th>incep-can1-ymin</th>\n",
       "      <th>incep-can1-xmin</th>\n",
       "      <th>incep-can1-ymax</th>\n",
       "      <th>incep-can1-xmax</th>\n",
       "      <th>incep-can1-box-width</th>\n",
       "      <th>incep-can1-box-height</th>\n",
       "      <th>incep-can1-box-ratio</th>\n",
       "      <th>incep-can1-box-area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>047b864f-0753-448b-9483-f990ae41abaf/0000</td>\n",
       "      <td>2</td>\n",
       "      <td>336</td>\n",
       "      <td>1530</td>\n",
       "      <td>406</td>\n",
       "      <td>1716</td>\n",
       "      <td>2</td>\n",
       "      <td>0.993506</td>\n",
       "      <td>333.420827</td>\n",
       "      <td>1544.805710</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0.993259</td>\n",
       "      <td>335.282574</td>\n",
       "      <td>1523.964975</td>\n",
       "      <td>401.099354</td>\n",
       "      <td>1723.260268</td>\n",
       "      <td>199.295294</td>\n",
       "      <td>65.816780</td>\n",
       "      <td>0.330248</td>\n",
       "      <td>13116.974478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>047b864f-0753-448b-9483-f990ae41abaf/0001</td>\n",
       "      <td>1</td>\n",
       "      <td>415</td>\n",
       "      <td>856</td>\n",
       "      <td>450</td>\n",
       "      <td>900</td>\n",
       "      <td>2</td>\n",
       "      <td>0.045906</td>\n",
       "      <td>406.718048</td>\n",
       "      <td>824.814145</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>416.218501</td>\n",
       "      <td>855.756898</td>\n",
       "      <td>448.187051</td>\n",
       "      <td>901.657055</td>\n",
       "      <td>45.900157</td>\n",
       "      <td>31.968550</td>\n",
       "      <td>0.696480</td>\n",
       "      <td>1467.361447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>047b864f-0753-448b-9483-f990ae41abaf/0002</td>\n",
       "      <td>1</td>\n",
       "      <td>388</td>\n",
       "      <td>775</td>\n",
       "      <td>454</td>\n",
       "      <td>874</td>\n",
       "      <td>1</td>\n",
       "      <td>0.964154</td>\n",
       "      <td>384.527009</td>\n",
       "      <td>771.206178</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999988</td>\n",
       "      <td>389.058445</td>\n",
       "      <td>775.985013</td>\n",
       "      <td>456.856925</td>\n",
       "      <td>874.412311</td>\n",
       "      <td>98.427298</td>\n",
       "      <td>67.798480</td>\n",
       "      <td>0.688818</td>\n",
       "      <td>6673.221171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>047b864f-0753-448b-9483-f990ae41abaf/0003</td>\n",
       "      <td>1</td>\n",
       "      <td>450</td>\n",
       "      <td>700</td>\n",
       "      <td>488</td>\n",
       "      <td>757</td>\n",
       "      <td>1</td>\n",
       "      <td>0.800901</td>\n",
       "      <td>450.871632</td>\n",
       "      <td>721.615883</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999991</td>\n",
       "      <td>450.100999</td>\n",
       "      <td>700.659639</td>\n",
       "      <td>488.284715</td>\n",
       "      <td>756.692331</td>\n",
       "      <td>56.032692</td>\n",
       "      <td>38.183716</td>\n",
       "      <td>0.681454</td>\n",
       "      <td>2139.536418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>047b864f-0753-448b-9483-f990ae41abaf/0004</td>\n",
       "      <td>1</td>\n",
       "      <td>429</td>\n",
       "      <td>170</td>\n",
       "      <td>666</td>\n",
       "      <td>657</td>\n",
       "      <td>1</td>\n",
       "      <td>0.997712</td>\n",
       "      <td>426.966608</td>\n",
       "      <td>183.737470</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999964</td>\n",
       "      <td>415.386574</td>\n",
       "      <td>181.614908</td>\n",
       "      <td>666.035782</td>\n",
       "      <td>657.711843</td>\n",
       "      <td>476.096935</td>\n",
       "      <td>250.649207</td>\n",
       "      <td>0.526467</td>\n",
       "      <td>119333.319517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  guid/image  gt_label  gt_ymin  gt_xmin  \\\n",
       "0  047b864f-0753-448b-9483-f990ae41abaf/0000         2      336     1530   \n",
       "1  047b864f-0753-448b-9483-f990ae41abaf/0001         1      415      856   \n",
       "2  047b864f-0753-448b-9483-f990ae41abaf/0002         1      388      775   \n",
       "3  047b864f-0753-448b-9483-f990ae41abaf/0003         1      450      700   \n",
       "4  047b864f-0753-448b-9483-f990ae41abaf/0004         1      429      170   \n",
       "\n",
       "   gt_ymax  gt_xmax  cocov1-can1-label  cocov1-can1-score  cocov1-can1-ymin  \\\n",
       "0      406     1716                  2           0.993506        333.420827   \n",
       "1      450      900                  2           0.045906        406.718048   \n",
       "2      454      874                  1           0.964154        384.527009   \n",
       "3      488      757                  1           0.800901        450.871632   \n",
       "4      666      657                  1           0.997712        426.966608   \n",
       "\n",
       "   cocov1-can1-xmin         ...           incep-can1-label  incep-can1-score  \\\n",
       "0       1544.805710         ...                          2          0.993259   \n",
       "1        824.814145         ...                          1          0.999943   \n",
       "2        771.206178         ...                          1          0.999988   \n",
       "3        721.615883         ...                          1          0.999991   \n",
       "4        183.737470         ...                          1          0.999964   \n",
       "\n",
       "   incep-can1-ymin  incep-can1-xmin  incep-can1-ymax  incep-can1-xmax  \\\n",
       "0       335.282574      1523.964975       401.099354      1723.260268   \n",
       "1       416.218501       855.756898       448.187051       901.657055   \n",
       "2       389.058445       775.985013       456.856925       874.412311   \n",
       "3       450.100999       700.659639       488.284715       756.692331   \n",
       "4       415.386574       181.614908       666.035782       657.711843   \n",
       "\n",
       "   incep-can1-box-width  incep-can1-box-height  incep-can1-box-ratio  \\\n",
       "0            199.295294              65.816780              0.330248   \n",
       "1             45.900157              31.968550              0.696480   \n",
       "2             98.427298              67.798480              0.688818   \n",
       "3             56.032692              38.183716              0.681454   \n",
       "4            476.096935             250.649207              0.526467   \n",
       "\n",
       "   incep-can1-box-area  \n",
       "0         13116.974478  \n",
       "1          1467.361447  \n",
       "2          6673.221171  \n",
       "3          2139.536418  \n",
       "4        119333.319517  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gt_cocov1_kitti_incep.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_cocov1_kitti_incep.to_csv('lgb_train.tsv',sep='\\t',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}