Day_022_HW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# (Kaggle)房價預測\n",
    "以下用房價預測資料, 觀查標籤編碼與獨編碼熱的影響"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>...</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities LotConfig      ...       ScreenPorch PoolArea PoolQC  \\\n",
       "0         Lvl    AllPub    Inside      ...                 0        0    NaN   \n",
       "1         Lvl    AllPub       FR2      ...                 0        0    NaN   \n",
       "2         Lvl    AllPub    Inside      ...                 0        0    NaN   \n",
       "3         Lvl    AllPub    Corner      ...                 0        0    NaN   \n",
       "4         Lvl    AllPub       FR2      ...                 0        0    NaN   \n",
       "\n",
       "  Fence MiscFeature MiscVal  MoSold  YrSold  SaleType  SaleCondition  \n",
       "0   NaN         NaN       0       2    2008        WD         Normal  \n",
       "1   NaN         NaN       0       5    2007        WD         Normal  \n",
       "2   NaN         NaN       0       9    2008        WD         Normal  \n",
       "3   NaN         NaN       0       2    2006        WD        Abnorml  \n",
       "4   NaN         NaN       0      12    2008        WD         Normal  \n",
       "\n",
       "[5 rows x 79 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 做完特徵工程前的所有準備\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import copy, time\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "\n",
    "data_path = 'data/'\n",
    "df_train = pd.read_csv(data_path + 'house_train.csv')\n",
    "df_test = pd.read_csv(data_path + 'house_test.csv')\n",
    "\n",
    "train_Y = np.log1p(df_train['SalePrice'])\n",
    "ids = df_test['Id']\n",
    "df_train = df_train.drop(['Id', 'SalePrice'] , axis=1)\n",
    "df_test = df_test.drop(['Id'] , axis=1)\n",
    "df = pd.concat([df_train,df_test])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 Object Features : ['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual', 'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition']\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>LandSlope</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Condition1</th>\n",
       "      <th>...</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "      <th>PavedDrive</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RL</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>CollgCr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>...</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RL</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Veenker</td>\n",
       "      <td>Feedr</td>\n",
       "      <td>...</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RL</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>CollgCr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>...</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RL</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Crawfor</td>\n",
       "      <td>Norm</td>\n",
       "      <td>...</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>Unf</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RL</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>NoRidge</td>\n",
       "      <td>Norm</td>\n",
       "      <td>...</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  MSZoning Street Alley LotShape LandContour Utilities LotConfig LandSlope  \\\n",
       "0       RL   Pave  None      Reg         Lvl    AllPub    Inside       Gtl   \n",
       "1       RL   Pave  None      Reg         Lvl    AllPub       FR2       Gtl   \n",
       "2       RL   Pave  None      IR1         Lvl    AllPub    Inside       Gtl   \n",
       "3       RL   Pave  None      IR1         Lvl    AllPub    Corner       Gtl   \n",
       "4       RL   Pave  None      IR1         Lvl    AllPub       FR2       Gtl   \n",
       "\n",
       "  Neighborhood Condition1      ...      GarageType GarageFinish GarageQual  \\\n",
       "0      CollgCr       Norm      ...          Attchd          RFn         TA   \n",
       "1      Veenker      Feedr      ...          Attchd          RFn         TA   \n",
       "2      CollgCr       Norm      ...          Attchd          RFn         TA   \n",
       "3      Crawfor       Norm      ...          Detchd          Unf         TA   \n",
       "4      NoRidge       Norm      ...          Attchd          RFn         TA   \n",
       "\n",
       "  GarageCond PavedDrive PoolQC Fence MiscFeature SaleType SaleCondition  \n",
       "0         TA          Y   None  None        None       WD        Normal  \n",
       "1         TA          Y   None  None        None       WD        Normal  \n",
       "2         TA          Y   None  None        None       WD        Normal  \n",
       "3         TA          Y   None  None        None       WD       Abnorml  \n",
       "4         TA          Y   None  None        None       WD        Normal  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 僅選取類別型欄位\n",
    "object_features = []\n",
    "for dtype, feature in zip(df.dtypes, df.columns):\n",
    "    if dtype == 'object':\n",
    "        object_features.append(feature)\n",
    "print(f'{len(object_features)} Object Features : {object_features}\\n')\n",
    "\n",
    "# 只留類別型欄位\n",
    "df = df[object_features]\n",
    "df = df.fillna('None')\n",
    "train_num = train_Y.shape[0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (1460, 43)\n",
      "score : 0.6615606866851302\n",
      "time : 0.018062591552734375 sec\n"
     ]
    }
   ],
   "source": [
    "# 標籤編碼 + 線性回歸\n",
    "df_label = pd.DataFrame()\n",
    "for i in df.columns:\n",
    "    df_label[i] = LabelEncoder().fit_transform(df[i])\n",
    "train_X = df_label[:train_num]\n",
    "estimator = LinearRegression()\n",
    "start = time.time()\n",
    "print(f'shape : {train_X.shape}')\n",
    "print(f'score : {cross_val_score(estimator, train_X, train_Y, cv=5).mean()}')\n",
    "print(f'time : {time.time() - start} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>264</th>\n",
       "      <th>265</th>\n",
       "      <th>266</th>\n",
       "      <th>267</th>\n",
       "      <th>268</th>\n",
       "      <th>269</th>\n",
       "      <th>270</th>\n",
       "      <th>271</th>\n",
       "      <th>272</th>\n",
       "      <th>273</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 274 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0    1    2    3    4    5    6    7    8    9   ...   264  265  266  267  \\\n",
       "0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0 ...   0.0  0.0  0.0  1.0   \n",
       "1  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0 ...   0.0  0.0  0.0  1.0   \n",
       "2  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0 ...   0.0  0.0  0.0  1.0   \n",
       "3  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0 ...   0.0  0.0  0.0  1.0   \n",
       "4  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0 ...   0.0  0.0  0.0  1.0   \n",
       "\n",
       "   268  269  270  271  272  273  \n",
       "0  0.0  0.0  0.0  0.0  1.0  0.0  \n",
       "1  0.0  0.0  0.0  0.0  1.0  0.0  \n",
       "2  0.0  0.0  0.0  0.0  1.0  0.0  \n",
       "3  1.0  0.0  0.0  0.0  0.0  0.0  \n",
       "4  0.0  0.0  0.0  0.0  1.0  0.0  \n",
       "\n",
       "[5 rows x 274 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 獨熱編碼 + 線性迴歸\n",
    "df_onehot = df_label.values\n",
    "df_onehot = OneHotEncoder().fit_transform(df_onehot).toarray()\n",
    "df_onehot = pd.DataFrame(df_onehot)\n",
    "df_onehot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (1460, 274)\n",
      "score : -7.147129671268858e+22\n",
      "time : 0.1755831241607666 sec\n"
     ]
    }
   ],
   "source": [
    "# 獨熱編碼 + 線性迴歸\n",
    "train_X = df_onehot[:train_num]\n",
    "estimator = LinearRegression()\n",
    "start = time.time()\n",
    "print(f'shape : {train_X.shape}')\n",
    "print(f'score : {cross_val_score(estimator, train_X, train_Y, cv=5).mean()}')\n",
    "print(f'time : {time.time() - start} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (1460, 43)\n",
      "score : 0.7764044277733078\n",
      "time : 0.762847900390625 sec\n"
     ]
    }
   ],
   "source": [
    "# 標籤編碼 + 梯度提升樹\n",
    "train_X = df_label[:train_num]\n",
    "estimator = GradientBoostingRegressor()\n",
    "start = time.time()\n",
    "print(f'shape : {train_X.shape}')\n",
    "print(f'score : {cross_val_score(estimator, train_X, train_Y, cv=5).mean()}')\n",
    "print(f'time : {time.time() - start} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (1460, 274)\n",
      "score : 0.7910629872425211\n",
      "time : 3.242842435836792 sec\n"
     ]
    }
   ],
   "source": [
    "# 獨熱編碼 + 梯度提升樹\n",
    "train_X = df_onehot[:train_num]\n",
    "estimator = GradientBoostingRegressor()\n",
    "start = time.time()\n",
    "print(f'shape : {train_X.shape}')\n",
    "print(f'score : {cross_val_score(estimator, train_X, train_Y, cv=5).mean()}')\n",
    "print(f'time : {time.time() - start} sec')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# [鐵達尼生存預測(Kaggle)](https://www.kaggle.com/c/titanic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 作業1\n",
    "* 觀察範例，在房價預測中調整標籤編碼(Label Encoder) / 獨熱編碼 (One Hot Encoder) 方式，  \n",
    "對於線性迴歸以及梯度提升樹兩種模型，何者影響比較大?\n",
    "對於線性迴規模型來說，label Encoder的表現較佳\n",
    "對於GradientBoostingRegressor模型來說OneHotEncoder的表現較佳。\n",
    "線性模型使用OneHotEncoder方法表現較差的可能原因為尚未產生出的大量變數已經違反多重共線性的規範。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass                                               Name     Sex   Age  \\\n",
       "0       3                            Braund, Mr. Owen Harris    male  22.0   \n",
       "1       1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0   \n",
       "2       3                             Heikkinen, Miss. Laina  female  26.0   \n",
       "3       1       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0   \n",
       "4       3                           Allen, Mr. William Henry    male  35.0   \n",
       "\n",
       "   SibSp  Parch            Ticket     Fare Cabin Embarked  \n",
       "0      1      0         A/5 21171   7.2500   NaN        S  \n",
       "1      1      0          PC 17599  71.2833   C85        C  \n",
       "2      0      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      1      0            113803  53.1000  C123        S  \n",
       "4      0      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 做完特徵工程前的所有準備 (與前範例相同)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import copy, time\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "data_path = 'data/'\n",
    "df_train = pd.read_csv(data_path + 'titanic_train.csv')\n",
    "df_test = pd.read_csv(data_path + 'titanic_test.csv')\n",
    "\n",
    "train_Y = df_train['Survived']\n",
    "ids = df_test['PassengerId']\n",
    "df_train = df_train.drop(['PassengerId', 'Survived'] , axis=1)\n",
    "df_test = df_test.drop(['PassengerId'] , axis=1)\n",
    "df = pd.concat([df_train,df_test])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 Numeric Features : ['Name', 'Sex', 'Ticket', 'Cabin', 'Embarked']\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>113803</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>373450</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Name     Sex  \\\n",
       "0                            Braund, Mr. Owen Harris    male   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female   \n",
       "2                             Heikkinen, Miss. Laina  female   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female   \n",
       "4                           Allen, Mr. William Henry    male   \n",
       "\n",
       "             Ticket Cabin Embarked  \n",
       "0         A/5 21171  None        S  \n",
       "1          PC 17599   C85        C  \n",
       "2  STON/O2. 3101282  None        S  \n",
       "3            113803  C123        S  \n",
       "4            373450  None        S  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#只取類別值 (object) 型欄位, 存於 object_features 中\n",
    "object_features = []\n",
    "for dtype, feature in zip(df.dtypes, df.columns):\n",
    "    if dtype == 'object':\n",
    "        object_features.append(feature)\n",
    "print(f'{len(object_features)} Numeric Features : {object_features}\\n')\n",
    "\n",
    "# 只留類別型欄位\n",
    "df = df[object_features]\n",
    "df = df.fillna('None')\n",
    "train_num = train_Y.shape[0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 作業2\n",
    "* 鐵達尼號例題中，標籤編碼 / 獨熱編碼又分別對預測結果有何影響? (Hint : 參考今日範例)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (891, 5)\n",
      "score :0.780004837244799\n",
      "time : 0.04017472267150879sec\n"
     ]
    }
   ],
   "source": [
    "# 標籤編碼 + 羅吉斯迴歸\n",
    "df_le = pd.DataFrame()\n",
    "for i in df.columns:\n",
    "    df_le[i] = LabelEncoder().fit_transform(df[i])\n",
    "train_X = df_le[:train_num]\n",
    "estimator = LogisticRegression()\n",
    "start = time.time()\n",
    "print(f\"shape : {train_X.shape}\")\n",
    "print(f\"score :{cross_val_score(estimator, train_X, train_Y, cv=5).mean()}\")\n",
    "print(f\"time : {time.time()-start}sec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (891, 2429)\n",
      "score : 0.8013346043513216\n",
      "time : 0.11060333251953125 sec\n"
     ]
    }
   ],
   "source": [
    "# 獨熱編碼 + 羅吉斯迴歸\n",
    "df_enc = df_le.values\n",
    "df_enc = OneHotEncoder().fit_transform(df_enc).toarray()\n",
    "df_enc = pd.DataFrame(df_enc)\n",
    "train_X = df_enc[:train_num]\n",
    "estimator = LogisticRegression()\n",
    "start = time.time()\n",
    "print(f'shape : {train_X.shape}')\n",
    "print(f'score : {cross_val_score(estimator, train_X, train_Y, cv=5).mean()}')\n",
    "print(f'time : {time.time() - start} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (891, 5)\n",
      "score :0.7991313528291322\n",
      "time : 0.3745126724243164sec\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "# label encodinf + gradiantboosting\n",
    "train_X = df_le[:train_num]\n",
    "estimator = GradientBoostingClassifier()\n",
    "start = time.time()\n",
    "print(f\"shape : {train_X.shape}\")\n",
    "print(f\"score :{cross_val_score(estimator, train_X, train_Y, cv=5).mean()}\")\n",
    "print(f\"time : {time.time()-start}sec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape : (891, 2429)\n",
      "score :0.8013030771473163\n",
      "time : 23.197394609451294sec\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "# label encodinf + gradiantboosting\n",
    "train_X = df_enc[:train_num]\n",
    "estimator = GradientBoostingClassifier()\n",
    "start = time.time()\n",
    "print(f\"shape : {train_X.shape}\")\n",
    "print(f\"score :{cross_val_score(estimator, train_X, train_Y, cv=5).mean()}\")\n",
    "print(f\"time : {time.time()-start}sec\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}