02_manual_pre_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loan Prediction - 02 - Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the conclusion of the Exploratory Data Analysis, we fill in some missing data assuming the following hypothesis:\n",
    "\n",
    "- Fill in the CoapplicantIncome with 0.\n",
    "- Fill in the LoanAmount_Terms with 360 for clients who have this value missing.\n",
    "- Fill in the Self_Employed with 'No'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('utils')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import math\n",
    "plt.style.use('seaborn')\n",
    "\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "import preprocess_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(614, 14)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_import = pd.read_csv('dataset/train_loan_new_variables.csv')\n",
    "df_import.drop(columns=['Loan_ID'],inplace = True)\n",
    "df_import.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filling Missing Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fill = df_import.copy()\n",
    "df_fill.loc[df_fill['Loan_Amount_Term'].isnull(),'Loan_Amount_Term'] = 360\n",
    "df_fill.loc[df_fill['Self_Employed'].isnull(),'Self_Employed'] = 'No'\n",
    "df_fill.loc[df_fill['CoapplicantIncome'].isnull(),'CoapplicantIncome'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recalculate Base_Loan_Installment and Remaining_Income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_loan_installment = df_fill['LoanAmount'] * 1000 / df_fill['Loan_Amount_Term']\n",
    "\n",
    "total_income = df_fill['ApplicantIncome'] + df_fill['CoapplicantIncome']\n",
    "remaining_income = (total_income - base_loan_installment) / total_income\n",
    "\n",
    "# married = df_import['Married'].copy()\n",
    "# married[married == 'Yes'] = 1\n",
    "# married[married == 'No'] = 0\n",
    "# dependents = df_import['Dependents'].copy()\n",
    "# dependents[dependents == '0'] = 0\n",
    "# dependents[dependents == '1'] = 1\n",
    "# dependents[dependents == '2'] = 2\n",
    "# dependents[dependents == '3+'] = 3\n",
    "# people_in_house = dependents + married + 1\n",
    "# household_percapita = (total_income / people_in_house).astype(float)\n",
    "# household_percapita.describe()\n",
    "\n",
    "\n",
    "df_fill['Base_Loan_Installment'] = base_loan_installment\n",
    "df_fill['Remaining_Income'] = remaining_income\n",
    "# df_fill['HouseHold_Percapita'] = household_percapita"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing NaNs and Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gender                   518\n",
       "Married                  518\n",
       "Dependents               518\n",
       "Education                518\n",
       "Self_Employed            518\n",
       "ApplicantIncome          518\n",
       "CoapplicantIncome        518\n",
       "LoanAmount               518\n",
       "Loan_Amount_Term         518\n",
       "Credit_History           518\n",
       "Property_Area            518\n",
       "Loan_Status              518\n",
       "Base_Loan_Installment    518\n",
       "Remaining_Income         518\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fill.dropna(inplace = True)\n",
    "df_fill.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(491, 14)\n"
     ]
    }
   ],
   "source": [
    "numerical_columns = ['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount','Base_Loan_Installment','Remaining_Income']\n",
    "df_fill_no_outlier = preprocess_utils.remove_outliers(df_fill,numerical_columns,threshold = 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoding Categorical Variables to Numeric Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_labels(df_input,show_encoding = True, ordinal_encoder = None):\n",
    "    df = df_input.copy()\n",
    "    ordinal_encoder = OrdinalEncoder()\n",
    "    ordinal_encoder.fit(df)\n",
    "    df = ordinal_encoder.transform(df)\n",
    "    \n",
    "    if show_encoding:\n",
    "        list(map(lambda x: print('Labels:',x,'| Ecoding:',np.array(range(x.shape[0]))),ordinal_encoder.categories_))\n",
    "    return df,ordinal_encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels: ['Female' 'Male'] | Ecoding: [0 1]\n",
      "Labels: ['No' 'Yes'] | Ecoding: [0 1]\n",
      "Labels: ['0' '1' '2' '3+'] | Ecoding: [0 1 2 3]\n",
      "Labels: ['Graduate' 'Not Graduate'] | Ecoding: [0 1]\n",
      "Labels: ['No' 'Yes'] | Ecoding: [0 1]\n",
      "Labels: ['Rural' 'Semiurban' 'Urban'] | Ecoding: [0 1 2]\n",
      "Labels: ['N' 'Y'] | Ecoding: [0 1]\n"
     ]
    }
   ],
   "source": [
    "categorical_columns = ['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed', 'Property_Area', 'Loan_Status']\n",
    "df_encoded = df_fill_no_outlier.copy()\n",
    "df_encoded[categorical_columns],ordinal_encoder = encode_labels(df_fill_no_outlier[categorical_columns])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us save the original encoder, in order to reuse it later on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('saves/variable_encoder_categories.npy',ordinal_encoder.categories_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "      <th>Base_Loan_Installment</th>\n",
       "      <th>Remaining_Income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>355.555556</td>\n",
       "      <td>0.941626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>183.333333</td>\n",
       "      <td>0.938889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>333.333333</td>\n",
       "      <td>0.932537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>391.666667</td>\n",
       "      <td>0.934722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5417</td>\n",
       "      <td>4196.0</td>\n",
       "      <td>267.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>741.666667</td>\n",
       "      <td>0.922848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2333</td>\n",
       "      <td>1516.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>263.888889</td>\n",
       "      <td>0.931440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3036</td>\n",
       "      <td>2504.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>438.888889</td>\n",
       "      <td>0.920778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4006</td>\n",
       "      <td>1526.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>466.666667</td>\n",
       "      <td>0.915642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3200</td>\n",
       "      <td>700.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>194.444444</td>\n",
       "      <td>0.950142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2500</td>\n",
       "      <td>1840.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>302.777778</td>\n",
       "      <td>0.930236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Gender  Married  Dependents  Education  Self_Employed  ApplicantIncome  \\\n",
       "0     1.0      1.0         1.0        0.0            0.0             4583   \n",
       "1     1.0      1.0         0.0        0.0            1.0             3000   \n",
       "2     1.0      1.0         0.0        1.0            0.0             2583   \n",
       "3     1.0      0.0         0.0        0.0            0.0             6000   \n",
       "4     1.0      1.0         2.0        0.0            1.0             5417   \n",
       "5     1.0      1.0         0.0        1.0            0.0             2333   \n",
       "6     1.0      1.0         3.0        0.0            0.0             3036   \n",
       "7     1.0      1.0         2.0        0.0            0.0             4006   \n",
       "8     1.0      1.0         2.0        0.0            0.0             3200   \n",
       "9     1.0      1.0         2.0        0.0            0.0             2500   \n",
       "\n",
       "   CoapplicantIncome  LoanAmount  Loan_Amount_Term  Credit_History  \\\n",
       "0             1508.0       128.0             360.0             1.0   \n",
       "1                0.0        66.0             360.0             1.0   \n",
       "2             2358.0       120.0             360.0             1.0   \n",
       "3                0.0       141.0             360.0             1.0   \n",
       "4             4196.0       267.0             360.0             1.0   \n",
       "5             1516.0        95.0             360.0             1.0   \n",
       "6             2504.0       158.0             360.0             0.0   \n",
       "7             1526.0       168.0             360.0             1.0   \n",
       "8              700.0        70.0             360.0             1.0   \n",
       "9             1840.0       109.0             360.0             1.0   \n",
       "\n",
       "   Property_Area  Loan_Status  Base_Loan_Installment  Remaining_Income  \n",
       "0            0.0          0.0             355.555556          0.941626  \n",
       "1            2.0          1.0             183.333333          0.938889  \n",
       "2            2.0          1.0             333.333333          0.932537  \n",
       "3            2.0          1.0             391.666667          0.934722  \n",
       "4            2.0          1.0             741.666667          0.922848  \n",
       "5            2.0          1.0             263.888889          0.931440  \n",
       "6            1.0          0.0             438.888889          0.920778  \n",
       "7            2.0          1.0             466.666667          0.915642  \n",
       "8            2.0          1.0             194.444444          0.950142  \n",
       "9            2.0          1.0             302.777778          0.930236  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_encoded.reset_index(inplace = True)\n",
    "df_encoded.drop(columns='index',inplace = True)\n",
    "df_encoded.to_csv('dataset/train_loan_preprocessed.csv',index = False)\n",
    "df_encoded.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We were able to recover some of the missing data by manually filling in some variables. \n",
    "\n",
    "Also, since we encoded the categorical variables, we can treat them as numerical from now on."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}