data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Dataset Creation\n",
    "\n",
    "Converting `cupid_scientists_original.csv` to `dataset_adjusted.csv`. This section of code takes about 10 minutes to run, and can be skipped if you want, as we've included the `dataset_adjusted.csv` file.\n",
    "\n",
    "All data is included because altogether it is only 11 MB. However, you may download our original and pre-processed datasets externally [here](https://drive.google.com/drive/folders/1VWyO-fxJr0Z3zxB8tg4ejY3ulmz74eYB?usp=sharing)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# from geopy.geocoders import Nominatim\n",
    "# import math\n",
    "import numpy as np\n",
    "\n",
    "cols = ['match', 'exphappy', 'samerace', 'hobby_diff_phys', 'hobby_diff_out', 'hobby_diff_in', 'same_goal', 'attr_diff', 'sinc_diff', 'intel_diff', 'fun_diff', 'amb_diff', 'income_diff', 'age_diff', 'same_career', 'confidence', 'imprace', 'date_freq', 'out_freq']\n",
    "raw_df = pd.read_csv('data/cupid_scientists_original.csv', encoding='latin-1')\n",
    "df = pd.DataFrame(columns=cols)\n",
    "\n",
    "hobbies = ['sports', 'tvsports', 'exercise', 'dining', 'museums', 'art', 'hiking', 'gaming', 'clubbing', 'reading', 'tv', 'theater', 'movies', 'concerts', 'music', 'shopping', 'yoga']\n",
    "\n",
    "hobbies_phys = ['sports', 'exercise', 'hiking', 'yoga']\n",
    "hobbies_out = ['dining', 'museums', 'concerts', 'clubbing', 'theater', 'movies', 'shopping']\n",
    "hobbies_in = ['tvsports', 'art', 'gaming', 'reading', 'tv', 'music']\n",
    "\n",
    "atts = ['attr', 'sinc', 'intel', 'fun', 'amb']\n",
    "lookingfor = ['attr1_1', 'sinc1_1', 'intel1_1', 'fun1_1', 'amb1_1']\n",
    "selfrate_a = ['attr3_1', 'sinc3_1', 'intel3_1', 'fun3_1', 'amb3_1']\n",
    "selfrate_b = ['attr5_1', 'sinc5_1', 'intel5_1', 'fun5_1', 'amb5_1']\n",
    "\n",
    "# Difference between totals for hobby attribute groups\n",
    "def sum_diff(attrs,p_1,p_2):\n",
    "    hobby_diff = 0\n",
    "    for h in attrs:\n",
    "        hobby_diff += abs(p_1[h] - p_2[h])\n",
    "    return hobby_diff\n",
    "\n",
    "# Difference between how much each person values a particular quality vs. the other's self-rating of that quality\n",
    "def att_diff(p_1, p_2):\n",
    "    all_diff = {}\n",
    "    for i in range(5):\n",
    "        diff = 0\n",
    "        selfrate_a1 = p_1[selfrate_a[i]]\n",
    "        selfrate_b1 = p_1[selfrate_b[i]]\n",
    "        selfrate_a2 = p_2[selfrate_a[i]]\n",
    "        selfrate_b2 = p_2[selfrate_b[i]]\n",
    "\n",
    "        # if pd.isna(p_1[selfrate_a[i]]) and pd.isna(p_1[selfrate_b[i]]):\n",
    "        #     selfrate_a1 = raw_df[selfrate_a[i]].mean()\n",
    "        #     selfrate_b1 = raw_df[selfrate_b[i]].mean()\n",
    "        if pd.isna(p_1[selfrate_a[i]]):\n",
    "            selfrate_a1 = p_1[selfrate_b[i]]\n",
    "        elif pd.isna(p_1[selfrate_b[i]]):\n",
    "            selfrate_b1 = p_1[selfrate_a[i]]\n",
    "        # if pd.isna(p_2[selfrate_a[i]]) and pd.isna(p_2[selfrate_b[i]]):\n",
    "        #     selfrate_a2 = raw_df[selfrate_a[i]].mean()\n",
    "        #     selfrate_b2 = raw_df[selfrate_b[i]].mean()\n",
    "        if pd.isna(p_2[selfrate_a[i]]):\n",
    "            selfrate_a2 = p_2[selfrate_b[i]]\n",
    "        elif pd.isna(p_2[selfrate_b[i]]):\n",
    "            selfrate_b2 = p_2[selfrate_a[i]]\n",
    "\n",
    "        want_1 = p_1[lookingfor[i]]\n",
    "        want_2 = p_2[lookingfor[i]]\n",
    "        selfrate_1 = (selfrate_a1 + selfrate_b1) / 2\n",
    "        selfrate_2 = (selfrate_a2 + selfrate_b2) / 2\n",
    "        diff += abs(selfrate_1 - want_2)\n",
    "        diff += abs(selfrate_2 - want_1)\n",
    "        if pd.isna(p_1[lookingfor[i]]) and pd.isna(p_2[lookingfor[i]]): print('W')\n",
    "        all_diff[atts[i]] = diff\n",
    "    return all_diff\n",
    "\n",
    "# Flat difference (for income and age)\n",
    "def plain_diff(key, p_1, p_2):\n",
    "    item_1 = p_1[key]\n",
    "    item_2 = p_2[key]\n",
    "    if isinstance(item_1,str):\n",
    "        item_1 = float(item_1.replace(',',''))\n",
    "    if isinstance(item_2,str):\n",
    "        item_2 = float(item_2.replace(',',''))\n",
    "    if pd.isna(p_1[key]) or pd.isna(p_2[key]):\n",
    "        return np.nan\n",
    "    return abs(item_1 - item_2)\n",
    "\n",
    "# Make sure there exists a date instance between a given pair of IDs\n",
    "def dated(iid, pid):\n",
    "    return len(raw_df.loc[(raw_df['iid'] == iid) & (raw_df['pid'] == pid)]) != 0\n",
    "\n",
    "\n",
    "## We were going to include euclidean distance between where each person grew up, but the required APIs kept giving rate limits and we did not have time to rectify this.\n",
    "# def get_dist(p_1,p_2):\n",
    "#     if isinstance(p_1['zipcode'], str):\n",
    "#         zip_1 = int(p_1['zipcode'].replace(',',''))\n",
    "#     else:\n",
    "#         zip_1 = p_1['zipcode']\n",
    "#     if isinstance(p_2['zipcode'], str):\n",
    "#         zip_2 = int(p_2['zipcode'].replace(',',''))\n",
    "#     else:\n",
    "#         zip_2 = p_2['zipcode']\n",
    "#     if pd.isna(zip_1) or pd.isna(zip_2) or (zip_1 == 0) or (zip_2 == 0):\n",
    "#         return np.nan\n",
    "#     geolocator = Nominatim(user_agent=\"geoapiExercises\")\n",
    "#     l_1 = geolocator.geocode(zip_1)\n",
    "#     l_2 = geolocator.geocode(zip_2)\n",
    "#     d = math.sqrt(((l_1.latitude - l_2.latitude) ** 2) + ((l_1.longitude - l_2.longitude) ** 2))\n",
    "#     return d\n",
    "\n",
    "# \n",
    "def create_df():\n",
    "    for iid in range(1,len(raw_df['iid'].unique())+1):\n",
    "        for pid in raw_df.loc[raw_df['iid'] == iid]['pid'].unique():\n",
    "            if dated(iid,pid):\n",
    "                date = raw_df.loc[(raw_df['iid'] == iid) & (raw_df['pid'] == pid)].to_dict('records')[0]\n",
    "                p_1 = raw_df.loc[raw_df['iid'] == iid].to_dict('records')[0]\n",
    "                p_2 = raw_df.loc[raw_df['iid'] == pid].to_dict('records')[0]\n",
    "                att_diffs = att_diff(p_1, p_2)\n",
    "                # print(date['wave'],att_diffs)\n",
    "\n",
    "                new_row = {\n",
    "                    'match': date['match'],\n",
    "                    'exphappy': date['exphappy'],\n",
    "                    'samerace': date['samerace'],\n",
    "                    'hobby_diff_phys': sum_diff(hobbies_phys,p_1,p_2),\n",
    "                    'hobby_diff_out': sum_diff(hobbies_out, p_1, p_2),\n",
    "                    'hobby_diff_in': sum_diff(hobbies_in, p_1, p_2),\n",
    "                    'same_goal': int(p_1['goal'] == p_2['goal']),\n",
    "                    'attr_diff': att_diffs['attr'],\n",
    "                    'sinc_diff': att_diffs['sinc'],\n",
    "                    'intel_diff': att_diffs['intel'],\n",
    "                    'fun_diff': att_diffs['fun'],\n",
    "                    'amb_diff': att_diffs['amb'],\n",
    "                    'income_diff': plain_diff('income', p_1, p_2),\n",
    "                    'age_diff': plain_diff('age', p_1, p_2),\n",
    "                    'same_career': int(p_1['career_c'] == p_2['career_c']),\n",
    "                    'confidence': ((((int(p_1['expnum']) / 20) + (int(p_2['expnum']) / 20)) / 2) if not (pd.isna(p_1['expnum']) or pd.isna(p_2['expnum'])) else p_1['expnum']),\n",
    "                    'imprace': (((p_1['imprace'] + p_2['imprace']) / 2) if not (pd.isna(p_1['imprace']) or pd.isna(p_2['imprace'])) else p_1['imprace']),\n",
    "                    'date_freq': p_1['date'],\n",
    "                    'out_freq': p_1['go_out']\n",
    "                }\n",
    "                # print(new_row)\n",
    "                df.loc[len(df)] = new_row\n",
    "    return df\n",
    "\n",
    "df = create_df()\n",
    "df.to_csv('data/dataset_adjusted.csv', index=False)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Data Quality Report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DQR: Continuous Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature</th>\n",
       "      <th>Desc.</th>\n",
       "      <th>Count</th>\n",
       "      <th>% of Missing</th>\n",
       "      <th>Card.</th>\n",
       "      <th>Min.</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Median</th>\n",
       "      <th>Q3</th>\n",
       "      <th>Max.</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Std. Dev.</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hobby_diff_phys</td>\n",
       "      <td>sum of difference between hobby/interest value...</td>\n",
       "      <td>8188</td>\n",
       "      <td>0.02</td>\n",
       "      <td>36</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.00</td>\n",
       "      <td>12.00</td>\n",
       "      <td>15.00</td>\n",
       "      <td>35.0</td>\n",
       "      <td>12.01</td>\n",
       "      <td>4.89</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>hobby_diff_out</td>\n",
       "      <td>sum of difference between hobby/interest value...</td>\n",
       "      <td>8188</td>\n",
       "      <td>0.02</td>\n",
       "      <td>52</td>\n",
       "      <td>3.00</td>\n",
       "      <td>12.00</td>\n",
       "      <td>16.00</td>\n",
       "      <td>21.00</td>\n",
       "      <td>58.0</td>\n",
       "      <td>17.23</td>\n",
       "      <td>6.77</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hobby_diff_in</td>\n",
       "      <td>sum of difference between hobby/interest value...</td>\n",
       "      <td>8188</td>\n",
       "      <td>0.02</td>\n",
       "      <td>38</td>\n",
       "      <td>2.00</td>\n",
       "      <td>12.00</td>\n",
       "      <td>16.00</td>\n",
       "      <td>19.00</td>\n",
       "      <td>41.0</td>\n",
       "      <td>15.95</td>\n",
       "      <td>5.27</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>attr_diff</td>\n",
       "      <td>difference in self-rated amount vs partner's p...</td>\n",
       "      <td>8136</td>\n",
       "      <td>0.03</td>\n",
       "      <td>758</td>\n",
       "      <td>0.67</td>\n",
       "      <td>19.00</td>\n",
       "      <td>26.50</td>\n",
       "      <td>37.00</td>\n",
       "      <td>131.0</td>\n",
       "      <td>30.56</td>\n",
       "      <td>16.15</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sinc_diff</td>\n",
       "      <td>difference in self-rated amount vs partner's p...</td>\n",
       "      <td>8136</td>\n",
       "      <td>0.03</td>\n",
       "      <td>726</td>\n",
       "      <td>1.00</td>\n",
       "      <td>15.00</td>\n",
       "      <td>19.48</td>\n",
       "      <td>24.00</td>\n",
       "      <td>62.0</td>\n",
       "      <td>20.00</td>\n",
       "      <td>8.02</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>intel_diff</td>\n",
       "      <td>difference in self-rated amount vs partner's p...</td>\n",
       "      <td>8136</td>\n",
       "      <td>0.03</td>\n",
       "      <td>633</td>\n",
       "      <td>0.00</td>\n",
       "      <td>18.98</td>\n",
       "      <td>23.00</td>\n",
       "      <td>29.00</td>\n",
       "      <td>69.0</td>\n",
       "      <td>24.39</td>\n",
       "      <td>8.93</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>amb_diff</td>\n",
       "      <td>difference in self-rated amount vs partner's p...</td>\n",
       "      <td>8100</td>\n",
       "      <td>0.03</td>\n",
       "      <td>702</td>\n",
       "      <td>0.00</td>\n",
       "      <td>7.50</td>\n",
       "      <td>11.00</td>\n",
       "      <td>15.00</td>\n",
       "      <td>57.0</td>\n",
       "      <td>11.55</td>\n",
       "      <td>5.57</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>fun_diff</td>\n",
       "      <td>difference in self-rated amount vs partner's p...</td>\n",
       "      <td>8118</td>\n",
       "      <td>0.03</td>\n",
       "      <td>640</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.50</td>\n",
       "      <td>20.00</td>\n",
       "      <td>24.00</td>\n",
       "      <td>61.5</td>\n",
       "      <td>20.24</td>\n",
       "      <td>7.65</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>income_diff</td>\n",
       "      <td>difference between incomes</td>\n",
       "      <td>2178</td>\n",
       "      <td>0.74</td>\n",
       "      <td>1061</td>\n",
       "      <td>8.00</td>\n",
       "      <td>6591.00</td>\n",
       "      <td>14997.00</td>\n",
       "      <td>26150.00</td>\n",
       "      <td>85670.0</td>\n",
       "      <td>18447.40</td>\n",
       "      <td>15078.11</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>age_diff</td>\n",
       "      <td>difference between ages</td>\n",
       "      <td>8159</td>\n",
       "      <td>0.02</td>\n",
       "      <td>25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>3.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>32.0</td>\n",
       "      <td>3.66</td>\n",
       "      <td>3.06</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>confidence</td>\n",
       "      <td>percentage of people each person dating expect...</td>\n",
       "      <td>1790</td>\n",
       "      <td>0.79</td>\n",
       "      <td>50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.38</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.93</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>exphappy</td>\n",
       "      <td>user expectation of happiness with speed datin...</td>\n",
       "      <td>8245</td>\n",
       "      <td>0.01</td>\n",
       "      <td>11</td>\n",
       "      <td>1.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>6.00</td>\n",
       "      <td>7.00</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.52</td>\n",
       "      <td>1.72</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>out_freq</td>\n",
       "      <td>rating of how often user goes out (not necessa...</td>\n",
       "      <td>8267</td>\n",
       "      <td>0.01</td>\n",
       "      <td>8</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>3.00</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.16</td>\n",
       "      <td>1.11</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>date_freq</td>\n",
       "      <td>rating of how often user goes on dates</td>\n",
       "      <td>8249</td>\n",
       "      <td>0.01</td>\n",
       "      <td>8</td>\n",
       "      <td>1.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>6.00</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.02</td>\n",
       "      <td>1.44</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>imprace</td>\n",
       "      <td>importance of having same racial/ethnic backgr...</td>\n",
       "      <td>8267</td>\n",
       "      <td>0.01</td>\n",
       "      <td>21</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2.00</td>\n",
       "      <td>3.50</td>\n",
       "      <td>5.00</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.79</td>\n",
       "      <td>2.04</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Feature                                              Desc.  Count  \\\n",
       "0   hobby_diff_phys  sum of difference between hobby/interest value...   8188   \n",
       "1    hobby_diff_out  sum of difference between hobby/interest value...   8188   \n",
       "2     hobby_diff_in  sum of difference between hobby/interest value...   8188   \n",
       "3         attr_diff  difference in self-rated amount vs partner's p...   8136   \n",
       "4         sinc_diff  difference in self-rated amount vs partner's p...   8136   \n",
       "5        intel_diff  difference in self-rated amount vs partner's p...   8136   \n",
       "6          amb_diff  difference in self-rated amount vs partner's p...   8100   \n",
       "7          fun_diff  difference in self-rated amount vs partner's p...   8118   \n",
       "8       income_diff                         difference between incomes   2178   \n",
       "9          age_diff                            difference between ages   8159   \n",
       "10       confidence  percentage of people each person dating expect...   1790   \n",
       "11         exphappy  user expectation of happiness with speed datin...   8245   \n",
       "12         out_freq  rating of how often user goes out (not necessa...   8267   \n",
       "13        date_freq             rating of how often user goes on dates   8249   \n",
       "14          imprace  importance of having same racial/ethnic backgr...   8267   \n",
       "\n",
       "    % of Missing  Card.  Min.       Q1    Median        Q3     Max.      Mean  \\\n",
       "0           0.02     36  0.00     8.00     12.00     15.00     35.0     12.01   \n",
       "1           0.02     52  3.00    12.00     16.00     21.00     58.0     17.23   \n",
       "2           0.02     38  2.00    12.00     16.00     19.00     41.0     15.95   \n",
       "3           0.03    758  0.67    19.00     26.50     37.00    131.0     30.56   \n",
       "4           0.03    726  1.00    15.00     19.48     24.00     62.0     20.00   \n",
       "5           0.03    633  0.00    18.98     23.00     29.00     69.0     24.39   \n",
       "6           0.03    702  0.00     7.50     11.00     15.00     57.0     11.55   \n",
       "7           0.03    640  0.00    15.50     20.00     24.00     61.5     20.24   \n",
       "8           0.74   1061  8.00  6591.00  14997.00  26150.00  85670.0  18447.40   \n",
       "9           0.02     25  0.00     1.00      3.00      5.00     32.0      3.66   \n",
       "10          0.79     50  0.00     0.15      0.25      0.38     20.0      0.39   \n",
       "11          0.01     11  1.00     5.00      6.00      7.00     10.0      5.52   \n",
       "12          0.01      8  1.00     1.00      2.00      3.00      7.0      2.16   \n",
       "13          0.01      8  1.00     4.00      5.00      6.00      7.0      5.02   \n",
       "14          0.01     21  0.50     2.00      3.50      5.00     10.0      3.79   \n",
       "\n",
       "    Std. Dev. Notes  \n",
       "0        4.89        \n",
       "1        6.77        \n",
       "2        5.27        \n",
       "3       16.15        \n",
       "4        8.02        \n",
       "5        8.93        \n",
       "6        5.57        \n",
       "7        7.65        \n",
       "8    15078.11        \n",
       "9        3.06        \n",
       "10       0.93        \n",
       "11       1.72        \n",
       "12       1.11        \n",
       "13       1.44        \n",
       "14       2.04        "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import statistics\n",
    "\n",
    "df = pd.read_csv('data/dataset_adjusted.csv')\n",
    "\n",
    "con_headers = [\"Feature\", \"Desc.\", \"Count\", \"% of Missing\", \"Card.\", \"Min.\", \"Q1\", \"Median\", \"Q3\", \"Max.\", \"Mean\", \"Std. Dev.\", \"Notes\"]\n",
    "\n",
    "con_features = ['hobby_diff_phys', 'hobby_diff_out', 'hobby_diff_in', 'attr_diff', 'sinc_diff', 'intel_diff', 'amb_diff', 'fun_diff', 'income_diff', 'age_diff', 'confidence', 'exphappy', 'out_freq', 'date_freq', 'imprace'] \n",
    "con_df = pd.DataFrame(columns=con_headers)\n",
    "features_df = pd.read_csv('data/features.csv', index_col='feature')\n",
    "\n",
    "for col in con_features:\n",
    "    con_df.loc[len(con_df.index)] = {\n",
    "        \"Feature\": col,\n",
    "        \"Desc.\": features_df.at[col, \"desc\"],\n",
    "        \"Count\": len(df) - df[col].isnull().sum(),\n",
    "        \"% of Missing\": round(df[col].isnull().sum() / len(df), 2),\n",
    "        \"Card.\": len(df[col].unique()),\n",
    "        \"Min.\": df[col].min(),\n",
    "        \"Q1\": round(pd.qcut(df[col], [.25], retbins=True)[1][0], 2),\n",
    "        \"Median\": round(df[col].median(),2),\n",
    "        \"Q3\": round(pd.qcut(df[col], [.75], retbins=True)[1][0], 2),\n",
    "        \"Max.\": df[col].max(),\n",
    "        \"Mean\": round(df[col].mean(),2),\n",
    "        \"Std. Dev.\": round(statistics.stdev([i for i in df[col] if not pd.isna(i)], df[col].mean()),2),\n",
    "        \"Notes\": \"\"\n",
    "    }\n",
    "\n",
    "con_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DQR: Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature</th>\n",
       "      <th>Desc.</th>\n",
       "      <th>Count</th>\n",
       "      <th>% of Missing</th>\n",
       "      <th>Card.</th>\n",
       "      <th>Mode</th>\n",
       "      <th>Mode Freq.</th>\n",
       "      <th>Mode %</th>\n",
       "      <th>2nd Mode</th>\n",
       "      <th>2nd Mode Freq.</th>\n",
       "      <th>2nd Mode %</th>\n",
       "      <th>Notes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>samerace</td>\n",
       "      <td>are the two participants the same race</td>\n",
       "      <td>8346</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>5039</td>\n",
       "      <td>60.38</td>\n",
       "      <td>1</td>\n",
       "      <td>3307</td>\n",
       "      <td>39.62</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>same_goal</td>\n",
       "      <td>whether both people have the same goal in part...</td>\n",
       "      <td>8346</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>5805</td>\n",
       "      <td>69.55</td>\n",
       "      <td>1</td>\n",
       "      <td>2541</td>\n",
       "      <td>30.45</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>same_career</td>\n",
       "      <td>whether both intended career paths fall into t...</td>\n",
       "      <td>8346</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6852</td>\n",
       "      <td>82.10</td>\n",
       "      <td>1</td>\n",
       "      <td>1494</td>\n",
       "      <td>17.90</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>match</td>\n",
       "      <td>target: did they end up matching</td>\n",
       "      <td>8346</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6972</td>\n",
       "      <td>83.54</td>\n",
       "      <td>1</td>\n",
       "      <td>1374</td>\n",
       "      <td>16.46</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Feature                                              Desc.  Count  \\\n",
       "0     samerace             are the two participants the same race   8346   \n",
       "1    same_goal  whether both people have the same goal in part...   8346   \n",
       "2  same_career  whether both intended career paths fall into t...   8346   \n",
       "3        match                   target: did they end up matching   8346   \n",
       "\n",
       "   % of Missing  Card.  Mode  Mode Freq.  Mode %  2nd Mode  2nd Mode Freq.  \\\n",
       "0           0.0      2     0        5039   60.38         1            3307   \n",
       "1           0.0      2     0        5805   69.55         1            2541   \n",
       "2           0.0      2     0        6852   82.10         1            1494   \n",
       "3           0.0      2     0        6972   83.54         1            1374   \n",
       "\n",
       "   2nd Mode % Notes  \n",
       "0       39.62        \n",
       "1       30.45        \n",
       "2       17.90        \n",
       "3       16.46        "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_features = ['samerace', 'same_goal', 'same_career', 'match']\n",
    "\n",
    "cat_headers = [\"Feature\", \"Desc.\", \"Count\", \"% of Missing\", \"Card.\", \"Mode\", \"Mode Freq.\", \"Mode %\", \"2nd Mode\", \"2nd Mode Freq.\", \"2nd Mode %\", \"Notes\"]\n",
    "\n",
    "cat_df = pd.DataFrame(columns=cat_headers)\n",
    "\n",
    "for col in cat_features:\n",
    "    multi_mode = True if len(df[col].mode()) > 1 else False\n",
    "    mode = df[col].mode()[0]\n",
    "    mode_2 = list(df[col].value_counts().keys())[1]\n",
    "    cat_df.loc[len(cat_df.index)] = {\n",
    "        \"Feature\": col,\n",
    "        \"Desc.\": features_df.at[col, \"desc\"],\n",
    "        \"Count\": len(df) - df[col].isnull().sum(),\n",
    "        \"% of Missing\": round(100 * df[col].isnull().sum() / len(df), 2),\n",
    "        \"Card.\": len(df[col].unique()),\n",
    "        \"Mode\": mode,\n",
    "        \"Mode Freq.\": df[col].value_counts()[mode],\n",
    "        \"Mode %\": round(df[col].value_counts()[mode] * 100 / (len(df) - df[col].isnull().sum()),2),\n",
    "        \"2nd Mode\": mode_2,\n",
    "        \"2nd Mode Freq.\": df[col].value_counts()[mode_2],\n",
    "        \"2nd Mode %\": round(df[col].value_counts()[mode_2] * 100 / (len(df) - df[col].isnull().sum()),2),\n",
    "        \"Notes\": \"\"\n",
    "    }\n",
    "\n",
    "cat_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing Continuous Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "for col in con_features:\n",
    "    bins=10\n",
    "    if col in ['date_freq', 'out_freq']: bins=7\n",
    "    plt.title(col)\n",
    "    sns.histplot(df[col], bins=bins)\n",
    "    plt.savefig(f'figs/figs/hist_{col}')\n",
    "    plt.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in cat_features:\n",
    "    vc = df[col].value_counts()\n",
    "    plt.title(col)\n",
    "    sns.barplot(x=list(vc.index), y=vc.iloc[:], color='#3274a1')\n",
    "    plt.savefig(f'figs/figs/bar_{col}')\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Missing Values and Outliers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Missing Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "0\n",
      "0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>samerace</th>\n",
       "      <th>hobby_diff_phys</th>\n",
       "      <th>hobby_diff_out</th>\n",
       "      <th>hobby_diff_in</th>\n",
       "      <th>same_goal</th>\n",
       "      <th>attr_diff</th>\n",
       "      <th>sinc_diff</th>\n",
       "      <th>intel_diff</th>\n",
       "      <th>fun_diff</th>\n",
       "      <th>amb_diff</th>\n",
       "      <th>income_diff</th>\n",
       "      <th>age_diff</th>\n",
       "      <th>same_career</th>\n",
       "      <th>confidence</th>\n",
       "      <th>imprace</th>\n",
       "      <th>date_freq</th>\n",
       "      <th>out_freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>4.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>1.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1</td>\n",
       "      <td>24.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>2.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1</td>\n",
       "      <td>30.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.425000</td>\n",
       "      <td>1.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.275000</td>\n",
       "      <td>2.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8341</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.0</td>\n",
       "      <td>15.5</td>\n",
       "      <td>34.0</td>\n",
       "      <td>28.5</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>3.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8342</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1</td>\n",
       "      <td>74.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>3.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8343</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>33.5</td>\n",
       "      <td>19.5</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8344</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>26.5</td>\n",
       "      <td>29.5</td>\n",
       "      <td>18.5</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8345</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>22.5</td>\n",
       "      <td>12.5</td>\n",
       "      <td>25.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>1.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8346 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      match  exphappy  samerace  hobby_diff_phys  hobby_diff_out  \\\n",
       "0         0       3.0         0              7.0            21.0   \n",
       "1         0       3.0         0              7.0            19.0   \n",
       "2         1       3.0         1              9.0            18.0   \n",
       "3         1       3.0         0              6.0            14.0   \n",
       "4         1       3.0         0              2.0            28.0   \n",
       "...     ...       ...       ...              ...             ...   \n",
       "8341      0       3.0         0             21.0            21.0   \n",
       "8342      0       3.0         0             16.0             7.0   \n",
       "8343      0       3.0         0             11.0            26.0   \n",
       "8344      0       3.0         0             11.0            16.0   \n",
       "8345      0       3.0         0              9.0            15.0   \n",
       "\n",
       "      hobby_diff_in  same_goal  attr_diff  sinc_diff  intel_diff  fun_diff  \\\n",
       "0              24.0          0       36.0       23.0        24.0      20.0   \n",
       "1              15.0          0       60.0       19.0        18.0      38.0   \n",
       "2              18.0          1       24.0       23.0        23.0      17.0   \n",
       "3              16.0          1       30.0       14.0        18.0      38.0   \n",
       "4              20.0          0       32.0       15.0        23.0      10.0   \n",
       "...             ...        ...        ...        ...         ...       ...   \n",
       "8341           13.0          1       40.0       15.5        34.0      28.5   \n",
       "8342           18.0          1       74.0       23.0        12.0      14.0   \n",
       "8343           25.0          0       64.0       16.0        33.5      19.5   \n",
       "8344           22.0          0       37.0       26.5        29.5      18.5   \n",
       "8345           33.0          0       50.0       22.5        12.5      25.0   \n",
       "\n",
       "      amb_diff  income_diff  age_diff  same_career  confidence  imprace  \\\n",
       "0         17.0          6.0       6.0            0    0.125000      4.5   \n",
       "1         13.0          1.0       1.0            0    0.550000      1.5   \n",
       "2         19.0          1.0       1.0            0    0.150000      2.5   \n",
       "3          8.0          2.0       2.0            0    0.425000      1.5   \n",
       "4          9.0          3.0       3.0            0    0.275000      2.5   \n",
       "...        ...          ...       ...          ...         ...      ...   \n",
       "8341      15.0          1.0       1.0            0    0.386816      3.5   \n",
       "8342      14.0          3.0       3.0            0    0.386816      3.5   \n",
       "8343      14.0          2.0       2.0            0    0.386816      5.0   \n",
       "8344      12.0          5.0       5.0            0    0.386816      3.0   \n",
       "8345       7.0          5.0       5.0            0    0.386816      1.5   \n",
       "\n",
       "      date_freq  out_freq  \n",
       "0           7.0       1.0  \n",
       "1           7.0       1.0  \n",
       "2           7.0       1.0  \n",
       "3           7.0       1.0  \n",
       "4           7.0       1.0  \n",
       "...         ...       ...  \n",
       "8341        6.0       3.0  \n",
       "8342        6.0       3.0  \n",
       "8343        6.0       3.0  \n",
       "8344        6.0       3.0  \n",
       "8345        6.0       3.0  \n",
       "\n",
       "[8346 rows x 19 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.impute import KNNImputer\n",
    "import numpy as np\n",
    "\n",
    "# Verify there are no missing values among categorical features\n",
    "for col in cat_features:\n",
    "    indeces = df[df[col].isna()].index\n",
    "    print(len(indeces))\n",
    "\n",
    "# Mean imputation for continuous features\n",
    "for col in con_features:\n",
    "    if col != 'income_diff':\n",
    "        mean = df[col].mean()\n",
    "        df[col].fillna(value=mean, inplace=True)\n",
    "\n",
    "\n",
    "# KNN imputation for income_diff\n",
    "imputer = KNNImputer()\n",
    "index = list(df.columns).index('income_diff')\n",
    "imputed_col = imputer.fit_transform(df)[:, 13]\n",
    "df['income_diff'] = imputed_col\n",
    "\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Outlier Handling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hobby_diff_phys (108 features updated with bounds 0.0 to 24.0)\n",
      "hobby_diff_out (183 features updated with bounds 1.0 to 33.0)\n",
      "hobby_diff_in (87 features updated with bounds 1.5 to 29.5)\n",
      "attr_diff (444 features updated with bounds -6.899999999999999 to 62.54)\n",
      "sinc_diff (258 features updated with bounds 1.5 to 37.5)\n",
      "intel_diff (376 features updated with bounds 4.75 to 42.75)\n",
      "amb_diff (52 features updated with bounds -3.4437500000000005 to 26.06625)\n",
      "fun_diff (290 features updated with bounds 3.1750000000000043 to 36.495)\n",
      "income_diff (362 features updated with bounds 0.0 to 9.0)\n",
      "age_diff (158 features updated with bounds -5.0 to 11.0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>samerace</th>\n",
       "      <th>hobby_diff_phys</th>\n",
       "      <th>hobby_diff_out</th>\n",
       "      <th>hobby_diff_in</th>\n",
       "      <th>same_goal</th>\n",
       "      <th>attr_diff</th>\n",
       "      <th>sinc_diff</th>\n",
       "      <th>intel_diff</th>\n",
       "      <th>fun_diff</th>\n",
       "      <th>amb_diff</th>\n",
       "      <th>income_diff</th>\n",
       "      <th>age_diff</th>\n",
       "      <th>same_career</th>\n",
       "      <th>confidence</th>\n",
       "      <th>imprace</th>\n",
       "      <th>date_freq</th>\n",
       "      <th>out_freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0</td>\n",
       "      <td>36.00</td>\n",
       "      <td>23.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>20.000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>4.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0</td>\n",
       "      <td>60.00</td>\n",
       "      <td>19.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>36.495</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>1.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1</td>\n",
       "      <td>24.00</td>\n",
       "      <td>23.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>17.000</td>\n",
       "      <td>19.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>2.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1</td>\n",
       "      <td>30.00</td>\n",
       "      <td>14.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>36.495</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.425000</td>\n",
       "      <td>1.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0</td>\n",
       "      <td>32.00</td>\n",
       "      <td>15.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10.000</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.275000</td>\n",
       "      <td>2.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8341</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.00</td>\n",
       "      <td>15.5</td>\n",
       "      <td>34.0</td>\n",
       "      <td>28.500</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>3.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8342</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1</td>\n",
       "      <td>62.54</td>\n",
       "      <td>23.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>14.000</td>\n",
       "      <td>14.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>3.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8343</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0</td>\n",
       "      <td>62.54</td>\n",
       "      <td>16.0</td>\n",
       "      <td>33.5</td>\n",
       "      <td>19.500</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8344</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>37.00</td>\n",
       "      <td>26.5</td>\n",
       "      <td>29.5</td>\n",
       "      <td>18.500</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8345</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>29.5</td>\n",
       "      <td>0</td>\n",
       "      <td>50.00</td>\n",
       "      <td>22.5</td>\n",
       "      <td>12.5</td>\n",
       "      <td>25.000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.386816</td>\n",
       "      <td>1.5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8346 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      match  exphappy  samerace  hobby_diff_phys  hobby_diff_out  \\\n",
       "0         0       3.0         0              7.0            21.0   \n",
       "1         0       3.0         0              7.0            19.0   \n",
       "2         1       3.0         1              9.0            18.0   \n",
       "3         1       3.0         0              6.0            14.0   \n",
       "4         1       3.0         0              2.0            28.0   \n",
       "...     ...       ...       ...              ...             ...   \n",
       "8341      0       3.0         0             21.0            21.0   \n",
       "8342      0       3.0         0             16.0             7.0   \n",
       "8343      0       3.0         0             11.0            26.0   \n",
       "8344      0       3.0         0             11.0            16.0   \n",
       "8345      0       3.0         0              9.0            15.0   \n",
       "\n",
       "      hobby_diff_in  same_goal  attr_diff  sinc_diff  intel_diff  fun_diff  \\\n",
       "0              24.0          0      36.00       23.0        24.0    20.000   \n",
       "1              15.0          0      60.00       19.0        18.0    36.495   \n",
       "2              18.0          1      24.00       23.0        23.0    17.000   \n",
       "3              16.0          1      30.00       14.0        18.0    36.495   \n",
       "4              20.0          0      32.00       15.0        23.0    10.000   \n",
       "...             ...        ...        ...        ...         ...       ...   \n",
       "8341           13.0          1      40.00       15.5        34.0    28.500   \n",
       "8342           18.0          1      62.54       23.0        12.0    14.000   \n",
       "8343           25.0          0      62.54       16.0        33.5    19.500   \n",
       "8344           22.0          0      37.00       26.5        29.5    18.500   \n",
       "8345           29.5          0      50.00       22.5        12.5    25.000   \n",
       "\n",
       "      amb_diff  income_diff  age_diff  same_career  confidence  imprace  \\\n",
       "0         17.0          6.0       6.0            0    0.125000      4.5   \n",
       "1         13.0          1.0       1.0            0    0.550000      1.5   \n",
       "2         19.0          1.0       1.0            0    0.150000      2.5   \n",
       "3          8.0          2.0       2.0            0    0.425000      1.5   \n",
       "4          9.0          3.0       3.0            0    0.275000      2.5   \n",
       "...        ...          ...       ...          ...         ...      ...   \n",
       "8341      15.0          1.0       1.0            0    0.386816      3.5   \n",
       "8342      14.0          3.0       3.0            0    0.386816      3.5   \n",
       "8343      14.0          2.0       2.0            0    0.386816      5.0   \n",
       "8344      12.0          5.0       5.0            0    0.386816      3.0   \n",
       "8345       7.0          5.0       5.0            0    0.386816      1.5   \n",
       "\n",
       "      date_freq  out_freq  \n",
       "0           7.0       1.0  \n",
       "1           7.0       1.0  \n",
       "2           7.0       1.0  \n",
       "3           7.0       1.0  \n",
       "4           7.0       1.0  \n",
       "...         ...       ...  \n",
       "8341        6.0       3.0  \n",
       "8342        6.0       3.0  \n",
       "8343        6.0       3.0  \n",
       "8344        6.0       3.0  \n",
       "8345        6.0       3.0  \n",
       "\n",
       "[8346 rows x 19 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clamped_df = df.copy()\n",
    "\n",
    "for col in con_features:\n",
    "    if col not in ['confidence', 'out_freq', 'date_freq', 'imprace', 'exphappy']:\n",
    "        q1 = pd.qcut(df[col], [.25], retbins=True)[1][0] if col != 'income_diff' else pd.qcut(df[col], [.05], retbins=True)[1][0]\n",
    "        q3 = pd.qcut(df[col], [.75], retbins=True)[1][0] if col != 'income_diff' else pd.qcut(df[col], [.95], retbins=True)[1][0]\n",
    "        iqr = q3 - q1\n",
    "        upper_bound = q3 + (iqr * 1.5) if col != 'income_diff' else q3\n",
    "        lower_bound = q1 - (iqr * 1.5)  if col != 'income_diff' else q1\n",
    "        outlier_count = len(df[df[col] > upper_bound])\n",
    "        outlier_count += len(df[df[col] < lower_bound])\n",
    "        clamped_df[col] = df[col].clip(lower=lower_bound, upper=upper_bound)\n",
    "        print(col, f\"({outlier_count} features updated with bounds {lower_bound} to {upper_bound})\")\n",
    "\n",
    "df = clamped_df\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>samerace</th>\n",
       "      <th>hobby_diff_phys</th>\n",
       "      <th>hobby_diff_out</th>\n",
       "      <th>hobby_diff_in</th>\n",
       "      <th>same_goal</th>\n",
       "      <th>attr_diff</th>\n",
       "      <th>sinc_diff</th>\n",
       "      <th>intel_diff</th>\n",
       "      <th>fun_diff</th>\n",
       "      <th>amb_diff</th>\n",
       "      <th>income_diff</th>\n",
       "      <th>age_diff</th>\n",
       "      <th>same_career</th>\n",
       "      <th>confidence</th>\n",
       "      <th>imprace</th>\n",
       "      <th>date_freq</th>\n",
       "      <th>out_freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.571036</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.506579</td>\n",
       "      <td>0.504952</td>\n",
       "      <td>0.652184</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0</td>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.472727</td>\n",
       "      <td>0</td>\n",
       "      <td>0.958946</td>\n",
       "      <td>0.486111</td>\n",
       "      <td>0.348684</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.498729</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0.027500</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>1</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>1</td>\n",
       "      <td>0.377081</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.480263</td>\n",
       "      <td>0.414916</td>\n",
       "      <td>0.728912</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007500</td>\n",
       "      <td>0.210526</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.366667</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>1</td>\n",
       "      <td>0.474059</td>\n",
       "      <td>0.347222</td>\n",
       "      <td>0.348684</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.306910</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0</td>\n",
       "      <td>0.021250</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.654545</td>\n",
       "      <td>0</td>\n",
       "      <td>0.506384</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.480263</td>\n",
       "      <td>0.204832</td>\n",
       "      <td>0.345274</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0</td>\n",
       "      <td>0.013750</td>\n",
       "      <td>0.210526</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8341</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.635688</td>\n",
       "      <td>0.388889</td>\n",
       "      <td>0.769737</td>\n",
       "      <td>0.760054</td>\n",
       "      <td>0.575457</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8342</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.190789</td>\n",
       "      <td>0.324880</td>\n",
       "      <td>0.537093</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8343</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.836364</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.402778</td>\n",
       "      <td>0.756579</td>\n",
       "      <td>0.489946</td>\n",
       "      <td>0.537093</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.473684</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8344</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.433333</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0</td>\n",
       "      <td>0.587199</td>\n",
       "      <td>0.694444</td>\n",
       "      <td>0.651316</td>\n",
       "      <td>0.459934</td>\n",
       "      <td>0.460365</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8345</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.797317</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.203947</td>\n",
       "      <td>0.655012</td>\n",
       "      <td>0.268546</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8346 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      match  exphappy  samerace  hobby_diff_phys  hobby_diff_out  \\\n",
       "0         0  0.222222         0         0.291667        0.600000   \n",
       "1         0  0.222222         0         0.291667        0.533333   \n",
       "2         1  0.222222         1         0.375000        0.500000   \n",
       "3         1  0.222222         0         0.250000        0.366667   \n",
       "4         1  0.222222         0         0.083333        0.833333   \n",
       "...     ...       ...       ...              ...             ...   \n",
       "8341      0  0.222222         0         0.875000        0.600000   \n",
       "8342      0  0.222222         0         0.666667        0.133333   \n",
       "8343      0  0.222222         0         0.458333        0.766667   \n",
       "8344      0  0.222222         0         0.458333        0.433333   \n",
       "8345      0  0.222222         0         0.375000        0.400000   \n",
       "\n",
       "      hobby_diff_in  same_goal  attr_diff  sinc_diff  intel_diff  fun_diff  \\\n",
       "0          0.800000          0   0.571036   0.597222    0.506579  0.504952   \n",
       "1          0.472727          0   0.958946   0.486111    0.348684  1.000000   \n",
       "2          0.581818          1   0.377081   0.597222    0.480263  0.414916   \n",
       "3          0.509091          1   0.474059   0.347222    0.348684  1.000000   \n",
       "4          0.654545          0   0.506384   0.375000    0.480263  0.204832   \n",
       "...             ...        ...        ...        ...         ...       ...   \n",
       "8341       0.400000          1   0.635688   0.388889    0.769737  0.760054   \n",
       "8342       0.581818          1   1.000000   0.597222    0.190789  0.324880   \n",
       "8343       0.836364          0   1.000000   0.402778    0.756579  0.489946   \n",
       "8344       0.727273          0   0.587199   0.694444    0.651316  0.459934   \n",
       "8345       1.000000          0   0.797317   0.583333    0.203947  0.655012   \n",
       "\n",
       "      amb_diff  income_diff  age_diff  same_career  confidence   imprace  \\\n",
       "0     0.652184     0.666667  0.545455            0    0.006250  0.421053   \n",
       "1     0.498729     0.111111  0.090909            0    0.027500  0.105263   \n",
       "2     0.728912     0.111111  0.090909            0    0.007500  0.210526   \n",
       "3     0.306910     0.222222  0.181818            0    0.021250  0.105263   \n",
       "4     0.345274     0.333333  0.272727            0    0.013750  0.210526   \n",
       "...        ...          ...       ...          ...         ...       ...   \n",
       "8341  0.575457     0.111111  0.090909            0    0.019341  0.315789   \n",
       "8342  0.537093     0.333333  0.272727            0    0.019341  0.315789   \n",
       "8343  0.537093     0.222222  0.181818            0    0.019341  0.473684   \n",
       "8344  0.460365     0.555556  0.454545            0    0.019341  0.263158   \n",
       "8345  0.268546     0.555556  0.454545            0    0.019341  0.105263   \n",
       "\n",
       "      date_freq  out_freq  \n",
       "0      1.000000  0.000000  \n",
       "1      1.000000  0.000000  \n",
       "2      1.000000  0.000000  \n",
       "3      1.000000  0.000000  \n",
       "4      1.000000  0.000000  \n",
       "...         ...       ...  \n",
       "8341   0.833333  0.333333  \n",
       "8342   0.833333  0.333333  \n",
       "8343   0.833333  0.333333  \n",
       "8344   0.833333  0.333333  \n",
       "8345   0.833333  0.333333  \n",
       "\n",
       "[8346 rows x 19 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_df = df.copy()\n",
    "\n",
    "# range normalization ## ai' = (ai - min(a)) / (max(a) - min(a))) * (high - low) + low\n",
    "for col in con_features:\n",
    "    high = 1\n",
    "    low = 0\n",
    "    min = norm_df[col].min()\n",
    "    max = norm_df[col].max()\n",
    "    norm_df[col] = norm_df[col].apply(lambda a: (((a - min) / (max - min)) * (high - low)) + low)\n",
    "\n",
    "df = norm_df\n",
    "\n",
    "# df.to_csv('data/speed_dating_transformed.csv')\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.4 Visualizing Transformed Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing continuous features post-transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "for col in con_features:\n",
    "    bins=10\n",
    "    if col in ['date_freq', 'out_freq']: bins=7\n",
    "    # plt.title(col)\n",
    "    sns.histplot(df[col], bins=bins)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f'figs/figs_transformed/hist_{col}')\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing continuous distributions to the target feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in con_features:\n",
    "    df.pivot(columns='match', values=col).plot.hist()\n",
    "    plt.xlabel(col)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f'figs/figs_compare/hist_{col}')\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A. Impurity-Based Univariate Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8346 entries, 0 to 8345\n",
      "Data columns (total 19 columns):\n",
      " #   Column           Non-Null Count  Dtype  \n",
      "---  ------           --------------  -----  \n",
      " 0   match            8346 non-null   int64  \n",
      " 1   exphappy         8346 non-null   float64\n",
      " 2   samerace         8346 non-null   int64  \n",
      " 3   hobby_diff_phys  8346 non-null   float64\n",
      " 4   hobby_diff_out   8346 non-null   float64\n",
      " 5   hobby_diff_in    8346 non-null   float64\n",
      " 6   same_goal        8346 non-null   int64  \n",
      " 7   attr_diff        8346 non-null   float64\n",
      " 8   sinc_diff        8346 non-null   float64\n",
      " 9   intel_diff       8346 non-null   float64\n",
      " 10  fun_diff         8346 non-null   float64\n",
      " 11  amb_diff         8346 non-null   float64\n",
      " 12  income_diff      8346 non-null   float64\n",
      " 13  age_diff         8346 non-null   float64\n",
      " 14  same_career      8346 non-null   int64  \n",
      " 15  confidence       8346 non-null   float64\n",
      " 16  imprace          8346 non-null   float64\n",
      " 17  date_freq        8346 non-null   float64\n",
      " 18  out_freq         8346 non-null   float64\n",
      "dtypes: float64(15), int64(4)\n",
      "memory usage: 1.2 MB\n"
     ]
    }
   ],
   "source": [
    "# df = pd.read_csv('data/speed_dating_transformed.csv', encoding='latin-1', index_col=0)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculating Impurity\n",
    "\n",
    "With Entropy & Gini Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entropy(feature, dataset):\n",
    "    val_count={}\n",
    "    prob_list = []\n",
    "    entr_list = []\n",
    "    tot_entropy = 0\n",
    "    unique_list = dataset[feature].unique()\n",
    "#     print(\"Unique values list is \"+str(unique_list))\n",
    "\n",
    "# finding unique values and their sum\n",
    "    for uniq in unique_list:\n",
    "        val_count[uniq] = (dataset[feature] == uniq).sum()\n",
    "#     print(\"Count of each unique value: \"+str(val_count))\n",
    "\n",
    "# finding probabilities\n",
    "    for val in val_count.values():\n",
    "        prob_list.append(val/(len(dataset[feature])))\n",
    "#     print(\"Probabilities list is \"+str(prob_list))\n",
    "\n",
    "# finding entropy for each type value in feature\n",
    "    for prob in prob_list:\n",
    "        entr_uniq = (-1)*prob*(np.log2(prob))\n",
    "        entr_list.append(entr_uniq)\n",
    "        \n",
    "# calculating total entropy of a particular feature\n",
    "    for each_entr in entr_list:\n",
    "        tot_entropy += each_entr\n",
    "    return tot_entropy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gini(feature, dataset):\n",
    "    val_count={}\n",
    "    prob_list = []\n",
    "    prob_sqr = 0\n",
    "    prob_sqr_list = []\n",
    "    tot_sqr_prob = 0\n",
    "    gini_index = 0\n",
    "    unique_list = dataset[feature].unique()\n",
    "#     print(\"Unique values list is \"+str(unique_list))\n",
    "\n",
    "# finding unique values and their sum\n",
    "    for uniq in unique_list:\n",
    "        val_count[uniq] = (dataset[feature] == uniq).sum()\n",
    "#     print(\"Count of each unique value: \"+str(val_count))\n",
    "\n",
    "# finding probabilities\n",
    "    for val in val_count.values():\n",
    "        prob_list.append(val/(len(dataset[feature])))\n",
    "#     print(\"Probabilities list is \"+str(prob_list))\n",
    "\n",
    "# finding prob squares for each type value in feature\n",
    "    for prob in prob_list:\n",
    "        prob_sqr = prob**2\n",
    "        prob_sqr_list.append(prob_sqr)\n",
    "#     print(\"Squared Probabilities list is \"+str(prob_sqr_list))\n",
    "\n",
    "# finding prob squares sum and the gini index\n",
    "    for each_prob_sqr in prob_sqr_list:\n",
    "        tot_sqr_prob += each_prob_sqr\n",
    "    gini_index = 1 - tot_sqr_prob\n",
    "    return gini_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_measure_impurity(feature, measure, dataset):\n",
    "\n",
    "    if measure == 'entropy':\n",
    "        impurity = entropy(feature, dataset)\n",
    "    elif measure == 'gini':\n",
    "        impurity = gini(feature, dataset)\n",
    "    \n",
    "    return impurity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculating Gain Metrics\n",
    "\n",
    "Information Gain & Gain Ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def IG(feature, target, dataset, measure):\n",
    "           \n",
    "    target_entropy = find_measure_impurity(target, measure, dataset)\n",
    "#     print('Target entropy is '+str(target_entropy))\n",
    "    entropy_list = []\n",
    "    weight_list = []\n",
    "\n",
    "# calculating weights for partitions and their impurities\n",
    "    for level in dataset[feature].unique():\n",
    "        df_feature_level = dataset[dataset[feature] == level]\n",
    "        entropy_level = find_measure_impurity(target, measure, df_feature_level)\n",
    "        entropy_list.append(entropy_level)  \n",
    "        weight_level = len(df_feature_level) / len(dataset)\n",
    "        weight_list.append(weight_level)\n",
    "\n",
    "#     print('impurity of partitions:', entropy_list)\n",
    "#     print('weights of partitions:', weight_list)\n",
    "\n",
    "    feature_rem_impurity = np.sum(np.array(entropy_list) * np.array(weight_list))\n",
    "#     print('Remaining impurity of '+feature+' is '+ str(feature_rem_impurity))\n",
    "\n",
    "# calculating information gain \n",
    "    info_gain = target_entropy - feature_rem_impurity\n",
    "\n",
    "    return info_gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GR(feature, target, dataset, measure):\n",
    "    entr = entropy(feature, dataset)\n",
    "    info_gain = IG(feature, target, dataset, measure)\n",
    "    gain_ratio = info_gain/entr\n",
    "#     print(\"Information gain: \"+str(info_gain)+\" ; Entropy: \"+str(entr)+\" ; Gain ratio: \"+str(gain_ratio))\n",
    "    return gain_ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_gain_metric(feature, target, dataset, measure, gain):\n",
    "\n",
    "    if gain == 'IG':\n",
    "        gain_value = IG(feature, target, dataset, measure)\n",
    "    elif gain == 'GR':\n",
    "        gain_value = GR(feature, target, dataset, measure)\n",
    "    \n",
    "    return gain_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculating IUFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def IUFS(target, dataset, k, measure='entropy', gain='IG'):\n",
    "    feature_gains = {}\n",
    "    feature_list = []\n",
    "    \n",
    "# finding feature gains using measures given\n",
    "    for col in dataset.drop(columns=target).columns:\n",
    "        feature_gains[col] = find_gain_metric(col, target, dataset, measure, gain)\n",
    "#     print(feature_gains)\n",
    "    \n",
    "# sorting the feature gains to select k best features \n",
    "    sorted_feature_gains = sorted(feature_gains.items(), key=lambda x:x[1], reverse=True)\n",
    "    sorted_dict = dict(sorted_feature_gains)\n",
    "#     print(sorted_dict)\n",
    "    for val in range(k):\n",
    "        feature_list = (list(sorted_dict)[0:k])\n",
    "    return feature_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features = ['samerace', 'same_goal', 'same_career', 'match']\n",
    "\n",
    "# creating a dataframe using categorical features\n",
    "cat_df = pd.DataFrame()\n",
    "index_of_col = 0\n",
    "for col in cat_features:\n",
    "    cat_df.insert(index_of_col, col, df[col])\n",
    "    index_of_col += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected features list:  ['same_career', 'samerace']\n",
      "Features to be dropped:  ['same_goal']\n"
     ]
    }
   ],
   "source": [
    "desc_features = ['exphappy', 'samerace', 'hobby_diff_phys', 'hobby_diff_out',\n",
    "       'hobby_diff_in', 'same_goal', 'attr_diff', 'sinc_diff', 'intel_diff',\n",
    "       'fun_diff', 'amb_diff', 'income_diff', 'age_diff', 'same_career',\n",
    "       'confidence', 'imprace', 'date_freq', 'out_freq']\n",
    "\n",
    "# selecting features using IUFS function \n",
    "select_feats = IUFS('match', cat_df, 2, measure='gini', gain='GR')\n",
    "print(\"Selected features list: \", select_feats)\n",
    "\n",
    "# categorical descriptive features list\n",
    "cat_descr_fts = ['samerace', 'same_goal', 'same_career']\n",
    "\n",
    "# features to be dropped \n",
    "drop_feats = list(set(cat_descr_fts) - set(select_feats))\n",
    "print(\"Features to be dropped: \", drop_feats)\n",
    "\n",
    "# copied df\n",
    "ufs_df = df.copy()\n",
    "# new df created after dropping features\n",
    "ufs_df = ufs_df.drop(columns=drop_feats)\n",
    "\n",
    "ufs_df_desc_fts = list(set(desc_features) - set(drop_feats))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comparing pre-IUFS results to post-IUFS results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# decision tree \n",
    "def model_DecisionTree(desc_features, target, dataframe):\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(dataframe[desc_features], dataframe[target],  \n",
    "                                                        test_size=0.25, random_state=0)\n",
    "\n",
    "#     print(desc_features)\n",
    "\n",
    "    dt_clf = DecisionTreeClassifier(max_depth = 2)\n",
    "    dt_clf.fit(X_train, Y_train)\n",
    "\n",
    "    y_pred = dt_clf.predict(X_test)\n",
    "    # print(y_pred)\n",
    "\n",
    "    accuracy = accuracy_score(Y_test, y_pred).round(3)\n",
    "    print(\"Accuracy score using DecisionTreeClassifier: {}\".format(accuracy))\n",
    "\n",
    "    cm = confusion_matrix(Y_test, y_pred)\n",
    "    cm_df = pd.DataFrame(cm)\n",
    "    return cm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# KNN classifier \n",
    "def model_kNN_clf(desc_features, target, dataframe,k):\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(dataframe[desc_features], dataframe[target],  \n",
    "                                                            test_size=0.25, random_state=0)\n",
    "\n",
    "    nn_clf = KNeighborsClassifier(n_neighbors=k)\n",
    "    nn_clf.fit(X_train, Y_train)\n",
    "\n",
    "    y_pred = nn_clf.predict(X_test)\n",
    "    print('kNN accuracy score: {:.5f}'.format(accuracy_score(y_pred, Y_test)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pre-IUFS:\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1736</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>348</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0  1\n",
       "0  1736  0\n",
       "1   348  3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kNN accuracy score: 0.81792\n",
      "\n",
      "Post-IUFS:\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1736</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>348</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0  1\n",
       "0  1736  0\n",
       "1   348  3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kNN accuracy score: 0.82175\n"
     ]
    }
   ],
   "source": [
    "print('Pre-IUFS:')\n",
    "\n",
    "# decision tree classifier on original dataframe\n",
    "display(model_DecisionTree(desc_features, 'match', df))\n",
    "\n",
    "# KNN classifier on original dataframe\n",
    "model_kNN_clf(desc_features, 'match', df, 5)\n",
    "\n",
    "print('\\nPost-IUFS:')\n",
    "\n",
    "# decision tree classifier on new dataframe after univariate FS using IUFS\n",
    "display(model_DecisionTree(ufs_df_desc_fts, 'match', ufs_df))\n",
    "\n",
    "# kNN classifier on new dataframe after univariate FS using IUFS\n",
    "model_kNN_clf(ufs_df_desc_fts, 'match', ufs_df,5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that the accuracy score has improved in the KNN model when we performed univariate FS using IUFS, so we drop the feature from the initial df."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = ufs_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### B. Recursive Feature Elimination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you encounter problems, please make sure your scikit-learn package is up to date before running this code. `pip install scikit-learn==1.1.3`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import RFE\n",
    "\n",
    "# Copying univariate feature selected dataframe to new df_cp\n",
    "df_cp = ufs_df.copy()\n",
    "\n",
    "# dropping match column \n",
    "df_cp = df_cp.drop(df_cp.columns[[0]],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature selection using RFE and evaluating results\n",
    "def FS_RFE(desc_cols, target_col, no_of_fts):\n",
    "    # splitting the df into desc and target\n",
    "    X = desc_cols\n",
    "    Y = target_col\n",
    "\n",
    "    # using logistic regression \n",
    "    model = LogisticRegression()\n",
    "\n",
    "    # performing recursive feature elimination to select features \n",
    "    rfe = RFE(model, n_features_to_select=no_of_fts)\n",
    "\n",
    "    fit = rfe.fit(X, Y)\n",
    "\n",
    "    print(f\"Selecting {fit.n_features_} out of {len(X.columns)} features...\")\n",
    "    # selected features list\n",
    "    slct_fts = fit.get_feature_names_out(None)\n",
    "    # print(\"Selected these features: %s\"% slct_fts)\n",
    "    # print(\"Selected Features as bool: %s\"% fit.support_) \n",
    "    # print(\"Feature ranking: %s\"% fit.ranking_)\n",
    "    \n",
    "    log_fs_desc_fts = slct_fts\n",
    "\n",
    "    # accuracy and confusion matrix using decision tree classifier\n",
    "    model_DecisionTree(log_fs_desc_fts, 'match', df)\n",
    "    \n",
    "    # accuracy and confusion matrix using KNN classifier\n",
    "    model_kNN_clf(log_fs_desc_fts, 'match', df, 2)\n",
    "    print()\n",
    "    \n",
    "    return slct_fts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting 10 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.81984\n",
      "\n",
      "Selecting 11 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.81936\n",
      "\n",
      "Selecting 12 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.81888\n",
      "\n",
      "Selecting 13 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.83086\n",
      "\n",
      "Selecting 14 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.82846\n",
      "\n",
      "Selecting 15 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.82846\n",
      "\n",
      "Selecting 16 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.83421\n",
      "\n",
      "Selecting 17 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.83325\n",
      "\n"
     ]
    }
   ],
   "source": [
    "desc_features = list(df.columns)\n",
    "desc_features.remove('match')\n",
    "\n",
    "for i in range(10,18):\n",
    "    FS_RFE(df[desc_features], df['match'], i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that when 15 features are selected, we have the highest accuracy with the K Neighbors model, so we select 16 features and discard the remaining features and perform model selection and evaluation. We have evaluated `amb_diff` to be the least important feature based on RFE selection, and `same_goal` to be least important based on IUFS, so they are both removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting 16 out of 17 features...\n",
      "Accuracy score using DecisionTreeClassifier: 0.833\n",
      "kNN accuracy score: 0.83421\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exphappy</th>\n",
       "      <th>samerace</th>\n",
       "      <th>hobby_diff_phys</th>\n",
       "      <th>hobby_diff_out</th>\n",
       "      <th>hobby_diff_in</th>\n",
       "      <th>attr_diff</th>\n",
       "      <th>sinc_diff</th>\n",
       "      <th>intel_diff</th>\n",
       "      <th>fun_diff</th>\n",
       "      <th>income_diff</th>\n",
       "      <th>age_diff</th>\n",
       "      <th>same_career</th>\n",
       "      <th>confidence</th>\n",
       "      <th>imprace</th>\n",
       "      <th>date_freq</th>\n",
       "      <th>out_freq</th>\n",
       "      <th>match</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.571036</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.506579</td>\n",
       "      <td>0.504952</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0</td>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.472727</td>\n",
       "      <td>0.958946</td>\n",
       "      <td>0.486111</td>\n",
       "      <td>0.348684</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0.027500</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>1</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>0.377081</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.480263</td>\n",
       "      <td>0.414916</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007500</td>\n",
       "      <td>0.210526</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.366667</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>0.474059</td>\n",
       "      <td>0.347222</td>\n",
       "      <td>0.348684</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0</td>\n",
       "      <td>0.021250</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.654545</td>\n",
       "      <td>0.506384</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.480263</td>\n",
       "      <td>0.204832</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0</td>\n",
       "      <td>0.013750</td>\n",
       "      <td>0.210526</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8341</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.635688</td>\n",
       "      <td>0.388889</td>\n",
       "      <td>0.769737</td>\n",
       "      <td>0.760054</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8342</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.190789</td>\n",
       "      <td>0.324880</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8343</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.836364</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.402778</td>\n",
       "      <td>0.756579</td>\n",
       "      <td>0.489946</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.473684</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8344</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.433333</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.587199</td>\n",
       "      <td>0.694444</td>\n",
       "      <td>0.651316</td>\n",
       "      <td>0.459934</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8345</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.797317</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.203947</td>\n",
       "      <td>0.655012</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019341</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8346 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      exphappy  samerace  hobby_diff_phys  hobby_diff_out  hobby_diff_in  \\\n",
       "0     0.222222         0         0.291667        0.600000       0.800000   \n",
       "1     0.222222         0         0.291667        0.533333       0.472727   \n",
       "2     0.222222         1         0.375000        0.500000       0.581818   \n",
       "3     0.222222         0         0.250000        0.366667       0.509091   \n",
       "4     0.222222         0         0.083333        0.833333       0.654545   \n",
       "...        ...       ...              ...             ...            ...   \n",
       "8341  0.222222         0         0.875000        0.600000       0.400000   \n",
       "8342  0.222222         0         0.666667        0.133333       0.581818   \n",
       "8343  0.222222         0         0.458333        0.766667       0.836364   \n",
       "8344  0.222222         0         0.458333        0.433333       0.727273   \n",
       "8345  0.222222         0         0.375000        0.400000       1.000000   \n",
       "\n",
       "      attr_diff  sinc_diff  intel_diff  fun_diff  income_diff  age_diff  \\\n",
       "0      0.571036   0.597222    0.506579  0.504952     0.666667  0.545455   \n",
       "1      0.958946   0.486111    0.348684  1.000000     0.111111  0.090909   \n",
       "2      0.377081   0.597222    0.480263  0.414916     0.111111  0.090909   \n",
       "3      0.474059   0.347222    0.348684  1.000000     0.222222  0.181818   \n",
       "4      0.506384   0.375000    0.480263  0.204832     0.333333  0.272727   \n",
       "...         ...        ...         ...       ...          ...       ...   \n",
       "8341   0.635688   0.388889    0.769737  0.760054     0.111111  0.090909   \n",
       "8342   1.000000   0.597222    0.190789  0.324880     0.333333  0.272727   \n",
       "8343   1.000000   0.402778    0.756579  0.489946     0.222222  0.181818   \n",
       "8344   0.587199   0.694444    0.651316  0.459934     0.555556  0.454545   \n",
       "8345   0.797317   0.583333    0.203947  0.655012     0.555556  0.454545   \n",
       "\n",
       "      same_career  confidence   imprace  date_freq  out_freq  match  \n",
       "0               0    0.006250  0.421053   1.000000  0.000000      0  \n",
       "1               0    0.027500  0.105263   1.000000  0.000000      0  \n",
       "2               0    0.007500  0.210526   1.000000  0.000000      1  \n",
       "3               0    0.021250  0.105263   1.000000  0.000000      1  \n",
       "4               0    0.013750  0.210526   1.000000  0.000000      1  \n",
       "...           ...         ...       ...        ...       ...    ...  \n",
       "8341            0    0.019341  0.315789   0.833333  0.333333      0  \n",
       "8342            0    0.019341  0.315789   0.833333  0.333333      0  \n",
       "8343            0    0.019341  0.473684   0.833333  0.333333      0  \n",
       "8344            0    0.019341  0.263158   0.833333  0.333333      0  \n",
       "8345            0    0.019341  0.105263   0.833333  0.333333      0  \n",
       "\n",
       "[8346 rows x 17 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "slct_fts = FS_RFE(df[desc_features], df['match'], 16)\n",
    "print\n",
    "\n",
    "feats = list(slct_fts)\n",
    "feats.append('match')\n",
    "\n",
    "new_df = df[feats]\n",
    "new_df.to_csv('data/cupid_scientists_pp.csv', index=False)\n",
    "\n",
    "new_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}