Catboost_ipy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a856ec43-def5-4c97-9db7-3f53d1ad1d2a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/ml/.venv/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "from catboost import * \n",
    "from PIL import Image \n",
    "from tqdm import tqdm \n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e3f71d36-8007-4b37-b483-8b00542e6430",
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_age(x):\n",
    "\tif x < 10: return 0\n",
    "\telif 10 <= x < 20: return 1\n",
    "\telif 20 <= x < 30: return 2\n",
    "\telif 30 <= x < 40: return 3\n",
    "\telif 40 <= x < 50: return 4\n",
    "\telif 50 <= x < 60: return 5\n",
    "\telif 60 <= x < 70: return 6\n",
    "\telif 70 <= x < 100: return 7\n",
    "\telse: return 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8cf6e574",
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = 'data/v4/'\n",
    "users = pd.read_csv(os.path.join(dpath,'users.csv'))\n",
    "books = pd.read_csv(os.path.join(dpath,'books.csv'))\n",
    "train = pd.read_csv(os.path.join(dpath,'train_ratings.csv'))\n",
    "test = pd.read_csv(os.path.join(dpath,'test_ratings.csv'))\n",
    "submission = pd.read_csv(os.path.join(dpath,'sample_submission.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f0fd1c1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "users['age'] = users['age'].apply(change_age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "24ffadd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = pd.concat([train['user_id'], submission['user_id']]).unique()\n",
    "isbns = pd.concat([train['isbn'], submission['isbn']]).unique()\n",
    "\n",
    "idx2user = {idx:id for idx, id in enumerate(ids)}\n",
    "idx2isbn = {idx:isbn for idx, isbn in enumerate(isbns)}\n",
    "\n",
    "user2idx = {id:idx for idx, id in idx2user.items()}\n",
    "isbn2idx = {isbn:idx for idx, isbn in idx2isbn.items()}\n",
    "\n",
    "users_ = users.copy()\n",
    "books_ = books.copy()\n",
    "books_ = books_.drop(['img_url','img_path'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dc2621f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.merge(train, users_, on='user_id', how='left')\n",
    "submission = pd.merge(submission, users_, on='user_id', how='left')\n",
    "test = pd.merge(test, users_, on='user_id', how='left')\n",
    "\n",
    "train = pd.merge(train, books_, on='isbn', how='left')\n",
    "submission = pd.merge(submission, books_, on='isbn', how='left')\n",
    "test = pd.merge(test, books_, on='isbn', how='left')\n",
    "\n",
    "train['user_id'] = train['user_id'].map(user2idx)\n",
    "submission['user_id'] = submission['user_id'].map(user2idx)\n",
    "test['user_id'] = test['user_id'].map(user2idx)\n",
    "\n",
    "train['isbn'] = train['isbn'].map(isbn2idx)\n",
    "submission['isbn'] = submission['isbn'].map(isbn2idx)\n",
    "test['isbn'] = test['isbn'].map(isbn2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "41cbd17e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['year_of_publication'] = train['year_of_publication'].astype(int)\n",
    "submission['year_of_publication'] = submission['year_of_publication'].astype(int)\n",
    "test['year_of_publication'] = test['year_of_publication'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b6723ae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.fillna('-1')\n",
    "submission = submission.fillna('-1')\n",
    "test = test.fillna('-1')\n",
    "\n",
    "sub_rating = submission['rating']\n",
    "submission = submission.drop(columns='rating')\n",
    "submission['rating'] = sub_rating\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0a6adaac",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid =  train_test_split(train.drop(['rating'],axis=1), train['rating'], test_size=0.2, random_state=42,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c92282dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(real: list, predict: list) -> float:\n",
    "    pred = np.array(predict)\n",
    "    return np.sqrt(np.mean((real-pred) ** 2))\n",
    "\n",
    "class RMSELoss(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(RMSELoss,self).__init__()\n",
    "        self.eps = 1e-6\n",
    "\n",
    "    def forward(self, x, y):\n",
    "        criterion = nn.MSELoss()\n",
    "        loss = torch.sqrt(criterion(x, y)+self.eps)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a1f0f501",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>isbn</th>\n",
       "      <th>age</th>\n",
       "      <th>location_city</th>\n",
       "      <th>location_state</th>\n",
       "      <th>location_country</th>\n",
       "      <th>rating_count</th>\n",
       "      <th>book_title</th>\n",
       "      <th>book_author</th>\n",
       "      <th>year_of_publication</th>\n",
       "      <th>publisher</th>\n",
       "      <th>language</th>\n",
       "      <th>category</th>\n",
       "      <th>summary</th>\n",
       "      <th>category_high</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>121312</th>\n",
       "      <td>22946</td>\n",
       "      <td>15310</td>\n",
       "      <td>4</td>\n",
       "      <td>arlington</td>\n",
       "      <td>virginia</td>\n",
       "      <td>usa</td>\n",
       "      <td>22</td>\n",
       "      <td>Grendel</td>\n",
       "      <td>John Champlin Gardner</td>\n",
       "      <td>1989</td>\n",
       "      <td>Vintage Books USA</td>\n",
       "      <td>en</td>\n",
       "      <td>fiction</td>\n",
       "      <td>The Beowulf story retold from the monster&amp;#39;...</td>\n",
       "      <td>fiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265089</th>\n",
       "      <td>44452</td>\n",
       "      <td>91405</td>\n",
       "      <td>6</td>\n",
       "      <td>portland</td>\n",
       "      <td>oregon</td>\n",
       "      <td>usa</td>\n",
       "      <td>5</td>\n",
       "      <td>Solomon's Decision</td>\n",
       "      <td>Judith B. Glad</td>\n",
       "      <td>2002</td>\n",
       "      <td>Tokyopop</td>\n",
       "      <td>en</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60236</th>\n",
       "      <td>15913</td>\n",
       "      <td>4800</td>\n",
       "      <td>3</td>\n",
       "      <td>fortcollins</td>\n",
       "      <td>colorado</td>\n",
       "      <td>usa</td>\n",
       "      <td>9</td>\n",
       "      <td>Politically Correct Bedtime Stories: Modern Ta...</td>\n",
       "      <td>James Finn Garner</td>\n",
       "      <td>1994</td>\n",
       "      <td>Scribner</td>\n",
       "      <td>en</td>\n",
       "      <td>humor</td>\n",
       "      <td>A whimsical adaptation of classic fairy tales ...</td>\n",
       "      <td>humor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111218</th>\n",
       "      <td>1339</td>\n",
       "      <td>13043</td>\n",
       "      <td>2</td>\n",
       "      <td>badaxe</td>\n",
       "      <td>michigan</td>\n",
       "      <td>usa</td>\n",
       "      <td>113</td>\n",
       "      <td>Eyes of Prey</td>\n",
       "      <td>John Sandford</td>\n",
       "      <td>2004</td>\n",
       "      <td>Berkley Publishing Group</td>\n",
       "      <td>en</td>\n",
       "      <td>fiction</td>\n",
       "      <td>When a series of gruesome mutilation killings ...</td>\n",
       "      <td>fiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306001</th>\n",
       "      <td>15663</td>\n",
       "      <td>128985</td>\n",
       "      <td>4</td>\n",
       "      <td>amora</td>\n",
       "      <td>estremadura</td>\n",
       "      <td>estremadura</td>\n",
       "      <td>71</td>\n",
       "      <td>Culture Shock!: South Africa</td>\n",
       "      <td>Dee Rissik</td>\n",
       "      <td>1993</td>\n",
       "      <td>Health Communications</td>\n",
       "      <td>en</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119879</th>\n",
       "      <td>2752</td>\n",
       "      <td>14936</td>\n",
       "      <td>3</td>\n",
       "      <td>fpo</td>\n",
       "      <td>ap</td>\n",
       "      <td>usa</td>\n",
       "      <td>174</td>\n",
       "      <td>The Genesis Code</td>\n",
       "      <td>John Case</td>\n",
       "      <td>1998</td>\n",
       "      <td>Ballantine Books</td>\n",
       "      <td>en</td>\n",
       "      <td>fiction</td>\n",
       "      <td>In a suburb of Washington, D.C., Joe Lassiter ...</td>\n",
       "      <td>fiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259178</th>\n",
       "      <td>8108</td>\n",
       "      <td>86566</td>\n",
       "      <td>3</td>\n",
       "      <td>newalbany</td>\n",
       "      <td>indiana</td>\n",
       "      <td>usa</td>\n",
       "      <td>4</td>\n",
       "      <td>The Marriage Dictionary</td>\n",
       "      <td>Tom Carey</td>\n",
       "      <td>1995</td>\n",
       "      <td>Pub Group West</td>\n",
       "      <td>en</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131932</th>\n",
       "      <td>10534</td>\n",
       "      <td>18168</td>\n",
       "      <td>5</td>\n",
       "      <td>westbloomfield</td>\n",
       "      <td>michigan</td>\n",
       "      <td>usa</td>\n",
       "      <td>8</td>\n",
       "      <td>Billy Straight : A Novel</td>\n",
       "      <td>JONATHAN KELLERMAN</td>\n",
       "      <td>1999</td>\n",
       "      <td>Ballantine Books</td>\n",
       "      <td>en</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146867</th>\n",
       "      <td>40246</td>\n",
       "      <td>22419</td>\n",
       "      <td>2</td>\n",
       "      <td>noisylegrand</td>\n",
       "      <td>seinestdenis</td>\n",
       "      <td>france</td>\n",
       "      <td>1</td>\n",
       "      <td>La Vie est ailleurs</td>\n",
       "      <td>Milan Kundera</td>\n",
       "      <td>1976</td>\n",
       "      <td>Gallimard</td>\n",
       "      <td>fr</td>\n",
       "      <td>fiction</td>\n",
       "      <td>Postface, p. 397-405, écrite en 1978 (Lire Kun...</td>\n",
       "      <td>fiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121958</th>\n",
       "      <td>24097</td>\n",
       "      <td>15452</td>\n",
       "      <td>1</td>\n",
       "      <td>wakefield</td>\n",
       "      <td>massachusetts</td>\n",
       "      <td>usa</td>\n",
       "      <td>66</td>\n",
       "      <td>Love Hina (Book 3)</td>\n",
       "      <td>Ken Akamatsu</td>\n",
       "      <td>2002</td>\n",
       "      <td>Tokyopop</td>\n",
       "      <td>en</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>245436 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id    isbn  age   location_city location_state location_country   \n",
       "121312    22946   15310    4       arlington       virginia              usa  \\\n",
       "265089    44452   91405    6        portland         oregon              usa   \n",
       "60236     15913    4800    3     fortcollins       colorado              usa   \n",
       "111218     1339   13043    2          badaxe       michigan              usa   \n",
       "306001    15663  128985    4           amora    estremadura      estremadura   \n",
       "...         ...     ...  ...             ...            ...              ...   \n",
       "119879     2752   14936    3             fpo             ap              usa   \n",
       "259178     8108   86566    3       newalbany        indiana              usa   \n",
       "131932    10534   18168    5  westbloomfield       michigan              usa   \n",
       "146867    40246   22419    2    noisylegrand   seinestdenis           france   \n",
       "121958    24097   15452    1       wakefield  massachusetts              usa   \n",
       "\n",
       "        rating_count                                         book_title   \n",
       "121312            22                                            Grendel  \\\n",
       "265089             5                                 Solomon's Decision   \n",
       "60236              9  Politically Correct Bedtime Stories: Modern Ta...   \n",
       "111218           113                                       Eyes of Prey   \n",
       "306001            71                       Culture Shock!: South Africa   \n",
       "...              ...                                                ...   \n",
       "119879           174                                   The Genesis Code   \n",
       "259178             4                            The Marriage Dictionary   \n",
       "131932             8                           Billy Straight : A Novel   \n",
       "146867             1                                La Vie est ailleurs   \n",
       "121958            66                                 Love Hina (Book 3)   \n",
       "\n",
       "                  book_author  year_of_publication                 publisher   \n",
       "121312  John Champlin Gardner                 1989         Vintage Books USA  \\\n",
       "265089         Judith B. Glad                 2002                  Tokyopop   \n",
       "60236       James Finn Garner                 1994                  Scribner   \n",
       "111218          John Sandford                 2004  Berkley Publishing Group   \n",
       "306001             Dee Rissik                 1993     Health Communications   \n",
       "...                       ...                  ...                       ...   \n",
       "119879              John Case                 1998          Ballantine Books   \n",
       "259178              Tom Carey                 1995            Pub Group West   \n",
       "131932     JONATHAN KELLERMAN                 1999          Ballantine Books   \n",
       "146867          Milan Kundera                 1976                 Gallimard   \n",
       "121958           Ken Akamatsu                 2002                  Tokyopop   \n",
       "\n",
       "       language category                                            summary   \n",
       "121312       en  fiction  The Beowulf story retold from the monster&#39;...  \\\n",
       "265089       en       -1                                                 -1   \n",
       "60236        en    humor  A whimsical adaptation of classic fairy tales ...   \n",
       "111218       en  fiction  When a series of gruesome mutilation killings ...   \n",
       "306001       en       -1                                                 -1   \n",
       "...         ...      ...                                                ...   \n",
       "119879       en  fiction  In a suburb of Washington, D.C., Joe Lassiter ...   \n",
       "259178       en       -1                                                 -1   \n",
       "131932       en       -1                                                 -1   \n",
       "146867       fr  fiction  Postface, p. 397-405, écrite en 1978 (Lire Kun...   \n",
       "121958       en       -1                                                 -1   \n",
       "\n",
       "       category_high  \n",
       "121312       fiction  \n",
       "265089            -1  \n",
       "60236          humor  \n",
       "111218       fiction  \n",
       "306001            -1  \n",
       "...              ...  \n",
       "119879       fiction  \n",
       "259178            -1  \n",
       "131932            -1  \n",
       "146867       fiction  \n",
       "121958            -1  \n",
       "\n",
       "[245436 rows x 15 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "be894fc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "06918cd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 2.4174387\ttest: 2.4137792\tbest: 2.4137792 (0)\ttotal: 105ms\tremaining: 17m 26s\n",
      "50:\tlearn: 2.2219653\ttest: 2.1727156\tbest: 2.1727156 (50)\ttotal: 5.06s\tremaining: 16m 26s\n",
      "100:\tlearn: 2.2066633\ttest: 2.1541488\tbest: 2.1541488 (100)\ttotal: 16.3s\tremaining: 26m 37s\n",
      "150:\tlearn: 2.2010505\ttest: 2.1492847\tbest: 2.1492847 (150)\ttotal: 28.1s\tremaining: 30m 34s\n",
      "200:\tlearn: 2.1973047\ttest: 2.1462854\tbest: 2.1462854 (200)\ttotal: 40.5s\tremaining: 32m 52s\n",
      "250:\tlearn: 2.1944491\ttest: 2.1440590\tbest: 2.1440590 (250)\ttotal: 52.6s\tremaining: 34m 4s\n",
      "300:\tlearn: 2.1916945\ttest: 2.1422863\tbest: 2.1422863 (300)\ttotal: 1m 5s\tremaining: 35m 16s\n",
      "350:\tlearn: 2.1896382\ttest: 2.1410928\tbest: 2.1410928 (350)\ttotal: 1m 17s\tremaining: 35m 36s\n",
      "400:\tlearn: 2.1876262\ttest: 2.1398966\tbest: 2.1398966 (400)\ttotal: 1m 30s\tremaining: 36m 3s\n",
      "450:\tlearn: 2.1857135\ttest: 2.1386556\tbest: 2.1386556 (450)\ttotal: 1m 42s\tremaining: 36m 12s\n",
      "500:\tlearn: 2.1841347\ttest: 2.1378017\tbest: 2.1378014 (499)\ttotal: 1m 55s\tremaining: 36m 21s\n",
      "550:\tlearn: 2.1829096\ttest: 2.1372990\tbest: 2.1372990 (550)\ttotal: 2m 7s\tremaining: 36m 20s\n",
      "600:\tlearn: 2.1816636\ttest: 2.1368359\tbest: 2.1368333 (599)\ttotal: 2m 18s\tremaining: 36m 8s\n",
      "650:\tlearn: 2.1804663\ttest: 2.1364070\tbest: 2.1364070 (650)\ttotal: 2m 30s\tremaining: 36m 1s\n",
      "700:\tlearn: 2.1794667\ttest: 2.1360708\tbest: 2.1360708 (700)\ttotal: 2m 42s\tremaining: 35m 53s\n",
      "750:\tlearn: 2.1782502\ttest: 2.1357098\tbest: 2.1357098 (750)\ttotal: 2m 54s\tremaining: 35m 47s\n",
      "800:\tlearn: 2.1772674\ttest: 2.1353636\tbest: 2.1353636 (800)\ttotal: 3m 7s\tremaining: 35m 47s\n",
      "850:\tlearn: 2.1763429\ttest: 2.1351631\tbest: 2.1351631 (850)\ttotal: 3m 18s\tremaining: 35m 36s\n",
      "900:\tlearn: 2.1755484\ttest: 2.1349966\tbest: 2.1349938 (899)\ttotal: 3m 30s\tremaining: 35m 26s\n",
      "950:\tlearn: 2.1746616\ttest: 2.1348031\tbest: 2.1347996 (942)\ttotal: 3m 42s\tremaining: 35m 13s\n",
      "1000:\tlearn: 2.1737462\ttest: 2.1346306\tbest: 2.1346235 (994)\ttotal: 3m 54s\tremaining: 35m 5s\n",
      "1050:\tlearn: 2.1727814\ttest: 2.1343968\tbest: 2.1343968 (1050)\ttotal: 4m 5s\tremaining: 34m 49s\n",
      "1100:\tlearn: 2.1719371\ttest: 2.1341826\tbest: 2.1341664 (1091)\ttotal: 4m 17s\tremaining: 34m 40s\n",
      "1150:\tlearn: 2.1709236\ttest: 2.1340245\tbest: 2.1340245 (1150)\ttotal: 4m 29s\tremaining: 34m 33s\n",
      "1200:\tlearn: 2.1700412\ttest: 2.1338263\tbest: 2.1338163 (1198)\ttotal: 4m 41s\tremaining: 34m 22s\n",
      "1250:\tlearn: 2.1691213\ttest: 2.1337503\tbest: 2.1337503 (1250)\ttotal: 4m 53s\tremaining: 34m 11s\n",
      "1300:\tlearn: 2.1682060\ttest: 2.1336143\tbest: 2.1336085 (1292)\ttotal: 5m 5s\tremaining: 34m 2s\n",
      "1350:\tlearn: 2.1673561\ttest: 2.1335242\tbest: 2.1335242 (1350)\ttotal: 5m 16s\tremaining: 33m 46s\n",
      "1400:\tlearn: 2.1666216\ttest: 2.1334484\tbest: 2.1334365 (1393)\ttotal: 5m 28s\tremaining: 33m 34s\n",
      "1450:\tlearn: 2.1658590\ttest: 2.1333729\tbest: 2.1333653 (1444)\ttotal: 5m 40s\tremaining: 33m 26s\n",
      "1500:\tlearn: 2.1650837\ttest: 2.1333182\tbest: 2.1332997 (1470)\ttotal: 5m 52s\tremaining: 33m 17s\n",
      "1550:\tlearn: 2.1642441\ttest: 2.1331878\tbest: 2.1331734 (1548)\ttotal: 6m 5s\tremaining: 33m 9s\n",
      "1600:\tlearn: 2.1635469\ttest: 2.1331522\tbest: 2.1331403 (1572)\ttotal: 6m 17s\tremaining: 32m 57s\n",
      "1650:\tlearn: 2.1628155\ttest: 2.1330566\tbest: 2.1330566 (1650)\ttotal: 6m 29s\tremaining: 32m 51s\n",
      "1700:\tlearn: 2.1620780\ttest: 2.1329842\tbest: 2.1329669 (1694)\ttotal: 6m 41s\tremaining: 32m 39s\n",
      "1750:\tlearn: 2.1613121\ttest: 2.1328762\tbest: 2.1328752 (1749)\ttotal: 6m 53s\tremaining: 32m 29s\n",
      "1800:\tlearn: 2.1605789\ttest: 2.1327864\tbest: 2.1327851 (1797)\ttotal: 7m 6s\tremaining: 32m 20s\n",
      "1850:\tlearn: 2.1596476\ttest: 2.1326297\tbest: 2.1326268 (1848)\ttotal: 7m 18s\tremaining: 32m 10s\n",
      "1900:\tlearn: 2.1589090\ttest: 2.1325718\tbest: 2.1325644 (1895)\ttotal: 7m 30s\tremaining: 32m\n",
      "1950:\tlearn: 2.1581393\ttest: 2.1325125\tbest: 2.1325125 (1950)\ttotal: 7m 42s\tremaining: 31m 48s\n",
      "2000:\tlearn: 2.1573706\ttest: 2.1324233\tbest: 2.1324217 (1995)\ttotal: 7m 54s\tremaining: 31m 36s\n",
      "2050:\tlearn: 2.1566262\ttest: 2.1323656\tbest: 2.1323591 (2047)\ttotal: 8m 6s\tremaining: 31m 26s\n",
      "2100:\tlearn: 2.1559670\ttest: 2.1322754\tbest: 2.1322600 (2098)\ttotal: 8m 20s\tremaining: 31m 21s\n",
      "2150:\tlearn: 2.1552396\ttest: 2.1321969\tbest: 2.1321845 (2142)\ttotal: 8m 32s\tremaining: 31m 10s\n",
      "2200:\tlearn: 2.1544958\ttest: 2.1321374\tbest: 2.1321344 (2196)\ttotal: 8m 45s\tremaining: 31m 2s\n",
      "2250:\tlearn: 2.1538609\ttest: 2.1320657\tbest: 2.1320657 (2250)\ttotal: 8m 59s\tremaining: 30m 55s\n",
      "2300:\tlearn: 2.1531366\ttest: 2.1320305\tbest: 2.1320305 (2300)\ttotal: 9m 11s\tremaining: 30m 45s\n",
      "2350:\tlearn: 2.1523984\ttest: 2.1319985\tbest: 2.1319825 (2346)\ttotal: 9m 24s\tremaining: 30m 35s\n",
      "2400:\tlearn: 2.1517448\ttest: 2.1319758\tbest: 2.1319713 (2399)\ttotal: 9m 36s\tremaining: 30m 23s\n",
      "2450:\tlearn: 2.1510563\ttest: 2.1319073\tbest: 2.1318847 (2439)\ttotal: 9m 48s\tremaining: 30m 12s\n",
      "2500:\tlearn: 2.1504252\ttest: 2.1318217\tbest: 2.1318192 (2499)\ttotal: 10m 1s\tremaining: 30m 2s\n",
      "2550:\tlearn: 2.1497449\ttest: 2.1317741\tbest: 2.1317667 (2548)\ttotal: 10m 13s\tremaining: 29m 52s\n",
      "2600:\tlearn: 2.1490905\ttest: 2.1317422\tbest: 2.1317310 (2568)\ttotal: 10m 26s\tremaining: 29m 42s\n",
      "2650:\tlearn: 2.1483994\ttest: 2.1316508\tbest: 2.1316459 (2638)\ttotal: 10m 39s\tremaining: 29m 31s\n",
      "2700:\tlearn: 2.1476446\ttest: 2.1315357\tbest: 2.1315300 (2693)\ttotal: 10m 51s\tremaining: 29m 20s\n",
      "2750:\tlearn: 2.1470186\ttest: 2.1315102\tbest: 2.1314943 (2729)\ttotal: 11m 3s\tremaining: 29m 7s\n",
      "2800:\tlearn: 2.1464277\ttest: 2.1314781\tbest: 2.1314676 (2797)\ttotal: 11m 16s\tremaining: 28m 57s\n",
      "2850:\tlearn: 2.1458665\ttest: 2.1314460\tbest: 2.1314310 (2835)\ttotal: 11m 28s\tremaining: 28m 46s\n",
      "2900:\tlearn: 2.1452933\ttest: 2.1314047\tbest: 2.1313822 (2884)\ttotal: 11m 41s\tremaining: 28m 36s\n",
      "2950:\tlearn: 2.1447145\ttest: 2.1313822\tbest: 2.1313703 (2908)\ttotal: 11m 53s\tremaining: 28m 24s\n",
      "3000:\tlearn: 2.1439740\ttest: 2.1313848\tbest: 2.1313435 (2981)\ttotal: 12m 6s\tremaining: 28m 13s\n",
      "3050:\tlearn: 2.1434272\ttest: 2.1313533\tbest: 2.1313435 (2981)\ttotal: 12m 18s\tremaining: 28m 2s\n",
      "3100:\tlearn: 2.1428350\ttest: 2.1313339\tbest: 2.1313297 (3082)\ttotal: 12m 31s\tremaining: 27m 51s\n",
      "3150:\tlearn: 2.1422045\ttest: 2.1312915\tbest: 2.1312915 (3150)\ttotal: 12m 43s\tremaining: 27m 39s\n",
      "3200:\tlearn: 2.1415497\ttest: 2.1312333\tbest: 2.1312333 (3200)\ttotal: 12m 56s\tremaining: 27m 29s\n",
      "3250:\tlearn: 2.1409136\ttest: 2.1312045\tbest: 2.1312045 (3248)\ttotal: 13m 9s\tremaining: 27m 18s\n",
      "3300:\tlearn: 2.1403614\ttest: 2.1312780\tbest: 2.1311993 (3256)\ttotal: 13m 22s\tremaining: 27m 7s\n",
      "3350:\tlearn: 2.1397214\ttest: 2.1312464\tbest: 2.1311993 (3256)\ttotal: 13m 34s\tremaining: 26m 55s\n",
      "3400:\tlearn: 2.1391767\ttest: 2.1312342\tbest: 2.1311993 (3256)\ttotal: 13m 46s\tremaining: 26m 43s\n",
      "3450:\tlearn: 2.1385798\ttest: 2.1312484\tbest: 2.1311993 (3256)\ttotal: 13m 59s\tremaining: 26m 32s\n",
      "3500:\tlearn: 2.1379844\ttest: 2.1312487\tbest: 2.1311993 (3256)\ttotal: 14m 11s\tremaining: 26m 20s\n",
      "3550:\tlearn: 2.1373023\ttest: 2.1313008\tbest: 2.1311993 (3256)\ttotal: 14m 23s\tremaining: 26m 8s\n",
      "3600:\tlearn: 2.1367070\ttest: 2.1312865\tbest: 2.1311993 (3256)\ttotal: 14m 36s\tremaining: 25m 56s\n",
      "3650:\tlearn: 2.1360636\ttest: 2.1312839\tbest: 2.1311993 (3256)\ttotal: 14m 48s\tremaining: 25m 45s\n",
      "3700:\tlearn: 2.1353572\ttest: 2.1312172\tbest: 2.1311941 (3692)\ttotal: 15m 1s\tremaining: 25m 34s\n",
      "3750:\tlearn: 2.1346983\ttest: 2.1311848\tbest: 2.1311807 (3744)\ttotal: 15m 14s\tremaining: 25m 23s\n",
      "3800:\tlearn: 2.1340001\ttest: 2.1310841\tbest: 2.1310802 (3799)\ttotal: 15m 27s\tremaining: 25m 12s\n",
      "3850:\tlearn: 2.1333723\ttest: 2.1310594\tbest: 2.1310594 (3850)\ttotal: 15m 39s\tremaining: 25m\n",
      "3900:\tlearn: 2.1327812\ttest: 2.1310914\tbest: 2.1310594 (3850)\ttotal: 15m 51s\tremaining: 24m 47s\n",
      "3950:\tlearn: 2.1322909\ttest: 2.1310974\tbest: 2.1310594 (3850)\ttotal: 16m 3s\tremaining: 24m 35s\n",
      "4000:\tlearn: 2.1316523\ttest: 2.1311345\tbest: 2.1310594 (3850)\ttotal: 16m 15s\tremaining: 24m 22s\n",
      "4050:\tlearn: 2.1310250\ttest: 2.1311171\tbest: 2.1310594 (3850)\ttotal: 16m 28s\tremaining: 24m 11s\n",
      "4100:\tlearn: 2.1303772\ttest: 2.1311874\tbest: 2.1310594 (3850)\ttotal: 16m 40s\tremaining: 23m 59s\n",
      "4150:\tlearn: 2.1298127\ttest: 2.1311359\tbest: 2.1310594 (3850)\ttotal: 16m 53s\tremaining: 23m 48s\n",
      "4200:\tlearn: 2.1292350\ttest: 2.1311022\tbest: 2.1310594 (3850)\ttotal: 17m 6s\tremaining: 23m 36s\n",
      "4250:\tlearn: 2.1286237\ttest: 2.1310878\tbest: 2.1310594 (3850)\ttotal: 17m 19s\tremaining: 23m 25s\n",
      "4300:\tlearn: 2.1280889\ttest: 2.1310936\tbest: 2.1310594 (3850)\ttotal: 17m 32s\tremaining: 23m 14s\n",
      "4350:\tlearn: 2.1275312\ttest: 2.1310629\tbest: 2.1310297 (4325)\ttotal: 17m 44s\tremaining: 23m 2s\n",
      "4400:\tlearn: 2.1269476\ttest: 2.1310409\tbest: 2.1310234 (4377)\ttotal: 17m 56s\tremaining: 22m 49s\n",
      "4450:\tlearn: 2.1263191\ttest: 2.1310581\tbest: 2.1310138 (4431)\ttotal: 18m 9s\tremaining: 22m 38s\n",
      "4500:\tlearn: 2.1257411\ttest: 2.1310474\tbest: 2.1310138 (4431)\ttotal: 18m 21s\tremaining: 22m 26s\n",
      "4550:\tlearn: 2.1251464\ttest: 2.1310525\tbest: 2.1310094 (4519)\ttotal: 18m 34s\tremaining: 22m 14s\n",
      "4600:\tlearn: 2.1246420\ttest: 2.1309961\tbest: 2.1309909 (4590)\ttotal: 18m 47s\tremaining: 22m 3s\n",
      "4650:\tlearn: 2.1240537\ttest: 2.1309849\tbest: 2.1309677 (4608)\ttotal: 19m\tremaining: 21m 51s\n",
      "4700:\tlearn: 2.1234990\ttest: 2.1310075\tbest: 2.1309677 (4608)\ttotal: 19m 13s\tremaining: 21m 39s\n",
      "4750:\tlearn: 2.1228704\ttest: 2.1310199\tbest: 2.1309677 (4608)\ttotal: 19m 26s\tremaining: 21m 28s\n",
      "4800:\tlearn: 2.1223005\ttest: 2.1309891\tbest: 2.1309677 (4608)\ttotal: 19m 38s\tremaining: 21m 16s\n",
      "4850:\tlearn: 2.1217194\ttest: 2.1310339\tbest: 2.1309677 (4608)\ttotal: 19m 51s\tremaining: 21m 4s\n",
      "4900:\tlearn: 2.1211727\ttest: 2.1310169\tbest: 2.1309677 (4608)\ttotal: 20m 4s\tremaining: 20m 52s\n",
      "4950:\tlearn: 2.1205924\ttest: 2.1310039\tbest: 2.1309677 (4608)\ttotal: 20m 17s\tremaining: 20m 41s\n",
      "5000:\tlearn: 2.1200771\ttest: 2.1309748\tbest: 2.1309647 (4966)\ttotal: 20m 29s\tremaining: 20m 29s\n",
      "5050:\tlearn: 2.1194471\ttest: 2.1309224\tbest: 2.1309090 (5041)\ttotal: 20m 43s\tremaining: 20m 17s\n",
      "5100:\tlearn: 2.1188654\ttest: 2.1308631\tbest: 2.1308573 (5099)\ttotal: 20m 56s\tremaining: 20m 6s\n",
      "5150:\tlearn: 2.1182962\ttest: 2.1308261\tbest: 2.1308063 (5124)\ttotal: 21m 8s\tremaining: 19m 53s\n",
      "5200:\tlearn: 2.1176900\ttest: 2.1308124\tbest: 2.1307978 (5190)\ttotal: 21m 20s\tremaining: 19m 41s\n",
      "5250:\tlearn: 2.1171551\ttest: 2.1308516\tbest: 2.1307900 (5211)\ttotal: 21m 32s\tremaining: 19m 29s\n",
      "5300:\tlearn: 2.1165879\ttest: 2.1307990\tbest: 2.1307900 (5211)\ttotal: 21m 45s\tremaining: 19m 16s\n",
      "5350:\tlearn: 2.1161016\ttest: 2.1308267\tbest: 2.1307661 (5310)\ttotal: 21m 57s\tremaining: 19m 4s\n",
      "5400:\tlearn: 2.1156193\ttest: 2.1308942\tbest: 2.1307661 (5310)\ttotal: 22m 9s\tremaining: 18m 52s\n",
      "5450:\tlearn: 2.1150772\ttest: 2.1309120\tbest: 2.1307661 (5310)\ttotal: 22m 22s\tremaining: 18m 40s\n",
      "5500:\tlearn: 2.1145447\ttest: 2.1309105\tbest: 2.1307661 (5310)\ttotal: 22m 35s\tremaining: 18m 28s\n",
      "5550:\tlearn: 2.1140566\ttest: 2.1309420\tbest: 2.1307661 (5310)\ttotal: 22m 47s\tremaining: 18m 16s\n",
      "5600:\tlearn: 2.1134635\ttest: 2.1309311\tbest: 2.1307661 (5310)\ttotal: 22m 59s\tremaining: 18m 3s\n",
      "5650:\tlearn: 2.1128961\ttest: 2.1308780\tbest: 2.1307661 (5310)\ttotal: 23m 12s\tremaining: 17m 51s\n",
      "5700:\tlearn: 2.1123296\ttest: 2.1308437\tbest: 2.1307661 (5310)\ttotal: 23m 26s\tremaining: 17m 40s\n",
      "5750:\tlearn: 2.1117846\ttest: 2.1308375\tbest: 2.1307661 (5310)\ttotal: 23m 38s\tremaining: 17m 28s\n",
      "5800:\tlearn: 2.1112476\ttest: 2.1308338\tbest: 2.1307661 (5310)\ttotal: 23m 51s\tremaining: 17m 16s\n",
      "5850:\tlearn: 2.1106861\ttest: 2.1308074\tbest: 2.1307661 (5310)\ttotal: 24m 4s\tremaining: 17m 4s\n",
      "5900:\tlearn: 2.1102267\ttest: 2.1308897\tbest: 2.1307661 (5310)\ttotal: 24m 16s\tremaining: 16m 51s\n",
      "5950:\tlearn: 2.1097043\ttest: 2.1309369\tbest: 2.1307661 (5310)\ttotal: 24m 28s\tremaining: 16m 39s\n",
      "6000:\tlearn: 2.1091844\ttest: 2.1309695\tbest: 2.1307661 (5310)\ttotal: 24m 41s\tremaining: 16m 27s\n",
      "6050:\tlearn: 2.1086328\ttest: 2.1309510\tbest: 2.1307661 (5310)\ttotal: 24m 54s\tremaining: 16m 15s\n",
      "6100:\tlearn: 2.1081142\ttest: 2.1309461\tbest: 2.1307661 (5310)\ttotal: 25m 6s\tremaining: 16m 2s\n",
      "6150:\tlearn: 2.1075817\ttest: 2.1309733\tbest: 2.1307661 (5310)\ttotal: 25m 19s\tremaining: 15m 50s\n",
      "6200:\tlearn: 2.1070952\ttest: 2.1309938\tbest: 2.1307661 (5310)\ttotal: 25m 31s\tremaining: 15m 38s\n",
      "6250:\tlearn: 2.1065999\ttest: 2.1309854\tbest: 2.1307661 (5310)\ttotal: 25m 44s\tremaining: 15m 26s\n",
      "6300:\tlearn: 2.1060016\ttest: 2.1309721\tbest: 2.1307661 (5310)\ttotal: 25m 56s\tremaining: 15m 13s\n",
      "6350:\tlearn: 2.1054564\ttest: 2.1310169\tbest: 2.1307661 (5310)\ttotal: 26m 9s\tremaining: 15m 1s\n",
      "6400:\tlearn: 2.1049123\ttest: 2.1310487\tbest: 2.1307661 (5310)\ttotal: 26m 22s\tremaining: 14m 49s\n",
      "6450:\tlearn: 2.1043162\ttest: 2.1310272\tbest: 2.1307661 (5310)\ttotal: 26m 35s\tremaining: 14m 37s\n",
      "6500:\tlearn: 2.1038594\ttest: 2.1310535\tbest: 2.1307661 (5310)\ttotal: 26m 47s\tremaining: 14m 25s\n",
      "6550:\tlearn: 2.1033275\ttest: 2.1310692\tbest: 2.1307661 (5310)\ttotal: 27m\tremaining: 14m 13s\n",
      "6600:\tlearn: 2.1027898\ttest: 2.1311132\tbest: 2.1307661 (5310)\ttotal: 27m 12s\tremaining: 14m\n",
      "6650:\tlearn: 2.1022703\ttest: 2.1311048\tbest: 2.1307661 (5310)\ttotal: 27m 25s\tremaining: 13m 48s\n",
      "6700:\tlearn: 2.1017519\ttest: 2.1311484\tbest: 2.1307661 (5310)\ttotal: 27m 38s\tremaining: 13m 36s\n",
      "6750:\tlearn: 2.1012524\ttest: 2.1311375\tbest: 2.1307661 (5310)\ttotal: 27m 50s\tremaining: 13m 24s\n",
      "6800:\tlearn: 2.1007152\ttest: 2.1311649\tbest: 2.1307661 (5310)\ttotal: 28m 3s\tremaining: 13m 11s\n",
      "6850:\tlearn: 2.1002415\ttest: 2.1311553\tbest: 2.1307661 (5310)\ttotal: 28m 15s\tremaining: 12m 59s\n",
      "6900:\tlearn: 2.0997517\ttest: 2.1311982\tbest: 2.1307661 (5310)\ttotal: 28m 28s\tremaining: 12m 47s\n",
      "6950:\tlearn: 2.0992969\ttest: 2.1312409\tbest: 2.1307661 (5310)\ttotal: 28m 41s\tremaining: 12m 34s\n",
      "7000:\tlearn: 2.0987847\ttest: 2.1312370\tbest: 2.1307661 (5310)\ttotal: 28m 53s\tremaining: 12m 22s\n",
      "7050:\tlearn: 2.0983257\ttest: 2.1312347\tbest: 2.1307661 (5310)\ttotal: 29m 6s\tremaining: 12m 10s\n",
      "7100:\tlearn: 2.0978773\ttest: 2.1312203\tbest: 2.1307661 (5310)\ttotal: 29m 18s\tremaining: 11m 57s\n",
      "7150:\tlearn: 2.0973921\ttest: 2.1312645\tbest: 2.1307661 (5310)\ttotal: 29m 31s\tremaining: 11m 45s\n",
      "7200:\tlearn: 2.0969044\ttest: 2.1313105\tbest: 2.1307661 (5310)\ttotal: 29m 43s\tremaining: 11m 33s\n",
      "7250:\tlearn: 2.0964241\ttest: 2.1313494\tbest: 2.1307661 (5310)\ttotal: 29m 56s\tremaining: 11m 20s\n",
      "7300:\tlearn: 2.0959044\ttest: 2.1313025\tbest: 2.1307661 (5310)\ttotal: 30m 8s\tremaining: 11m 8s\n",
      "7350:\tlearn: 2.0954368\ttest: 2.1313650\tbest: 2.1307661 (5310)\ttotal: 30m 21s\tremaining: 10m 56s\n",
      "7400:\tlearn: 2.0949389\ttest: 2.1313848\tbest: 2.1307661 (5310)\ttotal: 30m 33s\tremaining: 10m 43s\n",
      "7450:\tlearn: 2.0944417\ttest: 2.1313710\tbest: 2.1307661 (5310)\ttotal: 30m 45s\tremaining: 10m 31s\n",
      "7500:\tlearn: 2.0939577\ttest: 2.1313886\tbest: 2.1307661 (5310)\ttotal: 30m 58s\tremaining: 10m 19s\n",
      "7550:\tlearn: 2.0934730\ttest: 2.1314286\tbest: 2.1307661 (5310)\ttotal: 31m 11s\tremaining: 10m 6s\n",
      "7600:\tlearn: 2.0930260\ttest: 2.1314082\tbest: 2.1307661 (5310)\ttotal: 31m 23s\tremaining: 9m 54s\n",
      "7650:\tlearn: 2.0925831\ttest: 2.1314391\tbest: 2.1307661 (5310)\ttotal: 31m 35s\tremaining: 9m 42s\n",
      "7700:\tlearn: 2.0921678\ttest: 2.1314097\tbest: 2.1307661 (5310)\ttotal: 31m 47s\tremaining: 9m 29s\n",
      "7750:\tlearn: 2.0916404\ttest: 2.1314427\tbest: 2.1307661 (5310)\ttotal: 31m 59s\tremaining: 9m 16s\n",
      "7800:\tlearn: 2.0911795\ttest: 2.1314737\tbest: 2.1307661 (5310)\ttotal: 32m 12s\tremaining: 9m 4s\n",
      "7850:\tlearn: 2.0907149\ttest: 2.1315091\tbest: 2.1307661 (5310)\ttotal: 32m 24s\tremaining: 8m 52s\n",
      "7900:\tlearn: 2.0902514\ttest: 2.1315223\tbest: 2.1307661 (5310)\ttotal: 32m 37s\tremaining: 8m 40s\n",
      "7950:\tlearn: 2.0897778\ttest: 2.1315717\tbest: 2.1307661 (5310)\ttotal: 32m 50s\tremaining: 8m 27s\n",
      "8000:\tlearn: 2.0892677\ttest: 2.1316316\tbest: 2.1307661 (5310)\ttotal: 33m 2s\tremaining: 8m 15s\n",
      "8050:\tlearn: 2.0887699\ttest: 2.1316557\tbest: 2.1307661 (5310)\ttotal: 33m 15s\tremaining: 8m 3s\n",
      "8100:\tlearn: 2.0883011\ttest: 2.1316854\tbest: 2.1307661 (5310)\ttotal: 33m 28s\tremaining: 7m 50s\n",
      "8150:\tlearn: 2.0878344\ttest: 2.1317527\tbest: 2.1307661 (5310)\ttotal: 33m 41s\tremaining: 7m 38s\n",
      "8200:\tlearn: 2.0873560\ttest: 2.1317163\tbest: 2.1307661 (5310)\ttotal: 33m 53s\tremaining: 7m 26s\n",
      "8250:\tlearn: 2.0868332\ttest: 2.1317093\tbest: 2.1307661 (5310)\ttotal: 34m 6s\tremaining: 7m 13s\n",
      "8300:\tlearn: 2.0864029\ttest: 2.1316851\tbest: 2.1307661 (5310)\ttotal: 34m 19s\tremaining: 7m 1s\n",
      "8350:\tlearn: 2.0859272\ttest: 2.1316907\tbest: 2.1307661 (5310)\ttotal: 34m 32s\tremaining: 6m 49s\n",
      "8400:\tlearn: 2.0853610\ttest: 2.1316709\tbest: 2.1307661 (5310)\ttotal: 34m 44s\tremaining: 6m 36s\n",
      "8450:\tlearn: 2.0848552\ttest: 2.1316590\tbest: 2.1307661 (5310)\ttotal: 34m 57s\tremaining: 6m 24s\n",
      "8500:\tlearn: 2.0843880\ttest: 2.1317250\tbest: 2.1307661 (5310)\ttotal: 35m 9s\tremaining: 6m 12s\n",
      "8550:\tlearn: 2.0839302\ttest: 2.1317602\tbest: 2.1307661 (5310)\ttotal: 35m 22s\tremaining: 5m 59s\n",
      "8600:\tlearn: 2.0834679\ttest: 2.1318007\tbest: 2.1307661 (5310)\ttotal: 35m 35s\tremaining: 5m 47s\n",
      "8650:\tlearn: 2.0829562\ttest: 2.1317798\tbest: 2.1307661 (5310)\ttotal: 35m 47s\tremaining: 5m 34s\n",
      "8700:\tlearn: 2.0824446\ttest: 2.1318445\tbest: 2.1307661 (5310)\ttotal: 36m 1s\tremaining: 5m 22s\n",
      "8750:\tlearn: 2.0819755\ttest: 2.1318997\tbest: 2.1307661 (5310)\ttotal: 36m 14s\tremaining: 5m 10s\n",
      "8800:\tlearn: 2.0814997\ttest: 2.1319080\tbest: 2.1307661 (5310)\ttotal: 36m 26s\tremaining: 4m 57s\n",
      "8850:\tlearn: 2.0810387\ttest: 2.1319158\tbest: 2.1307661 (5310)\ttotal: 36m 39s\tremaining: 4m 45s\n",
      "8900:\tlearn: 2.0806463\ttest: 2.1319118\tbest: 2.1307661 (5310)\ttotal: 36m 53s\tremaining: 4m 33s\n",
      "8950:\tlearn: 2.0801750\ttest: 2.1319485\tbest: 2.1307661 (5310)\ttotal: 37m 5s\tremaining: 4m 20s\n",
      "9000:\tlearn: 2.0797285\ttest: 2.1319690\tbest: 2.1307661 (5310)\ttotal: 37m 18s\tremaining: 4m 8s\n",
      "9050:\tlearn: 2.0792833\ttest: 2.1319527\tbest: 2.1307661 (5310)\ttotal: 37m 30s\tremaining: 3m 56s\n",
      "9100:\tlearn: 2.0788083\ttest: 2.1319978\tbest: 2.1307661 (5310)\ttotal: 37m 43s\tremaining: 3m 43s\n",
      "9150:\tlearn: 2.0783345\ttest: 2.1320218\tbest: 2.1307661 (5310)\ttotal: 37m 55s\tremaining: 3m 31s\n",
      "9200:\tlearn: 2.0777755\ttest: 2.1320261\tbest: 2.1307661 (5310)\ttotal: 38m 8s\tremaining: 3m 18s\n",
      "9250:\tlearn: 2.0772944\ttest: 2.1320609\tbest: 2.1307661 (5310)\ttotal: 38m 21s\tremaining: 3m 6s\n",
      "9300:\tlearn: 2.0767983\ttest: 2.1320395\tbest: 2.1307661 (5310)\ttotal: 38m 34s\tremaining: 2m 53s\n",
      "9350:\tlearn: 2.0762959\ttest: 2.1320518\tbest: 2.1307661 (5310)\ttotal: 38m 46s\tremaining: 2m 41s\n",
      "9400:\tlearn: 2.0758175\ttest: 2.1320614\tbest: 2.1307661 (5310)\ttotal: 39m\tremaining: 2m 29s\n",
      "9450:\tlearn: 2.0753588\ttest: 2.1321133\tbest: 2.1307661 (5310)\ttotal: 39m 13s\tremaining: 2m 16s\n",
      "9500:\tlearn: 2.0748733\ttest: 2.1320826\tbest: 2.1307661 (5310)\ttotal: 39m 26s\tremaining: 2m 4s\n",
      "9550:\tlearn: 2.0743920\ttest: 2.1320517\tbest: 2.1307661 (5310)\ttotal: 39m 38s\tremaining: 1m 51s\n",
      "9600:\tlearn: 2.0739720\ttest: 2.1320532\tbest: 2.1307661 (5310)\ttotal: 39m 51s\tremaining: 1m 39s\n",
      "9650:\tlearn: 2.0735219\ttest: 2.1319868\tbest: 2.1307661 (5310)\ttotal: 40m 4s\tremaining: 1m 26s\n",
      "9700:\tlearn: 2.0730890\ttest: 2.1319752\tbest: 2.1307661 (5310)\ttotal: 40m 16s\tremaining: 1m 14s\n",
      "9750:\tlearn: 2.0726153\ttest: 2.1320567\tbest: 2.1307661 (5310)\ttotal: 40m 29s\tremaining: 1m 2s\n",
      "9800:\tlearn: 2.0721693\ttest: 2.1320805\tbest: 2.1307661 (5310)\ttotal: 40m 41s\tremaining: 49.6s\n",
      "9850:\tlearn: 2.0717211\ttest: 2.1320686\tbest: 2.1307661 (5310)\ttotal: 40m 53s\tremaining: 37.1s\n",
      "9900:\tlearn: 2.0712629\ttest: 2.1321101\tbest: 2.1307661 (5310)\ttotal: 41m 6s\tremaining: 24.7s\n",
      "9950:\tlearn: 2.0707746\ttest: 2.1321152\tbest: 2.1307661 (5310)\ttotal: 41m 19s\tremaining: 12.2s\n",
      "9999:\tlearn: 2.0703128\ttest: 2.1321536\tbest: 2.1307661 (5310)\ttotal: 41m 31s\tremaining: 0us\n",
      "\n",
      "bestTest = 2.130766081\n",
      "bestIteration = 5310\n",
      "\n",
      "Shrink model to first 5311 iterations.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostRegressor at 0x7fb6fa367130>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_features = [0,1,2,3,4,5,7,8,9,10,11,12,13,14]\n",
    "model = CatBoostRegressor(iterations=10000, depth=6, learning_rate=0.05, random_seed=42, verbose=50)\n",
    "model.fit(X_train,\n",
    "          y_train,\n",
    "          cat_features = cat_features,\n",
    "          eval_set = (X_valid,y_valid),\n",
    "          use_best_model=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2e83f1f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'learn': {'RMSE': 2.1737556982842934},\n",
       " 'validation': {'RMSE': 2.134623484111431}}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_best_score()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b287286a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76699"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model.predict(submission.drop(['rating'],axis=1))\n",
    "len(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3c22779e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76699"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission = pd.read_csv(os.path.join(dpath,'sample_submission.csv'))\n",
    "len(sample_submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "32b97612",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_submission = submission.copy()\n",
    "pred_submission['rating'] = pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6c26f7c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_submission['rating'] = pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "6f7a32bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/opt/ml'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7c773a52",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_submission.to_csv('code/submit/catboost_v4_1_submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "68b94a16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         11676\n",
       "1        116866\n",
       "2        152827\n",
       "3        157969\n",
       "4         67958\n",
       "          ...  \n",
       "76694    278543\n",
       "76695    278563\n",
       "76696    278633\n",
       "76697    278668\n",
       "76698    278851\n",
       "Name: user_id, Length: 76699, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_submission['user_id'].map(idx2user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "e3c21ebf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        0002005018\n",
       "1        0002005018\n",
       "2        0060973129\n",
       "3        0374157065\n",
       "4        0399135782\n",
       "            ...    \n",
       "76694    1576734218\n",
       "76695    3492223710\n",
       "76696    1896095186\n",
       "76697    8408044079\n",
       "76698    0767907566\n",
       "Name: isbn, Length: 76699, dtype: object"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_submission['isbn'].map(idx2isbn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "ca4bfe8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6.03397012e+01, 4.42881231e-02, 1.87064969e+00, 2.12270226e+00,\n",
       "       7.04342400e+00, 1.60583707e+01, 2.03785915e+00, 7.26898546e+00,\n",
       "       6.49503360e-01, 2.39280077e+00, 1.71715328e-01])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "197b1fc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'isbn', 'location', 'age', 'book_title', 'book_author',\n",
       "       'year_of_publication', 'publisher', 'language', 'category', 'summary'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "6d66ebc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>user_id</td>\n",
       "      <td>60.339701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>book_author</td>\n",
       "      <td>16.058371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>publisher</td>\n",
       "      <td>7.268985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>book_title</td>\n",
       "      <td>7.043424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>category</td>\n",
       "      <td>2.392801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>age</td>\n",
       "      <td>2.122702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>year_of_publication</td>\n",
       "      <td>2.037859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>location</td>\n",
       "      <td>1.870650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>language</td>\n",
       "      <td>0.649503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>summary</td>\n",
       "      <td>0.171715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>isbn</td>\n",
       "      <td>0.044288</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                feature  importance\n",
       "0               user_id   60.339701\n",
       "5           book_author   16.058371\n",
       "7             publisher    7.268985\n",
       "4            book_title    7.043424\n",
       "9              category    2.392801\n",
       "3                   age    2.122702\n",
       "6   year_of_publication    2.037859\n",
       "2              location    1.870650\n",
       "8              language    0.649503\n",
       "10              summary    0.171715\n",
       "1                  isbn    0.044288"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'feature':X_train.columns,'importance':model.feature_importances_}).sort_values(by='importance',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "bdaab3e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "149570"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(os.listdir('data/images/'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c7a1e27",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}