ensemble_bagging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeClassifier as DTC\n",
    "from sklearn.neighbors import KNeighborsClassifier as KNN\n",
    "from sklearn.linear_model import LogisticRegression as LR\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.preprocessing import StandardScaler as SS\n",
    "from joblib import dump\n",
    "from config_store import targetDataSet, randomSeed\n",
    "from utility_functions import dataSetSplit, doGridSearch, score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Website</th>\n",
       "      <th>Packets</th>\n",
       "      <th>Bytes</th>\n",
       "      <th>Packets A → B</th>\n",
       "      <th>Bytes A → B</th>\n",
       "      <th>Packets B → A</th>\n",
       "      <th>Bytes B → A</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Bits/s A → B</th>\n",
       "      <th>Bits/s B → A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>712</td>\n",
       "      <td>1</td>\n",
       "      <td>712</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16.763719</td>\n",
       "      <td>339</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>718</td>\n",
       "      <td>1</td>\n",
       "      <td>718</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10.784664</td>\n",
       "      <td>532</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>889</td>\n",
       "      <td>1</td>\n",
       "      <td>889</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.927569</td>\n",
       "      <td>1443</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>712</td>\n",
       "      <td>1</td>\n",
       "      <td>712</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13.479684</td>\n",
       "      <td>422</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>718</td>\n",
       "      <td>1</td>\n",
       "      <td>718</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12.706807</td>\n",
       "      <td>452</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Website  Packets  Bytes  Packets A → B  Bytes A → B  Packets B → A  \\\n",
       "0        0        1    712              1          712              0   \n",
       "1        0        1    718              1          718              0   \n",
       "2        0        1    889              1          889              0   \n",
       "3        0        1    712              1          712              0   \n",
       "4        0        1    718              1          718              0   \n",
       "\n",
       "   Bytes B → A   Duration  Bits/s A → B  Bits/s B → A  \n",
       "0            0  16.763719           339             0  \n",
       "1            0  10.784664           532             0  \n",
       "2            0   4.927569          1443             0  \n",
       "3            0  13.479684           422             0  \n",
       "4            0  12.706807           452             0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import and preview data set\n",
    "data = pd.read_csv(targetDataSet)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate data set using test/train split\n",
    "XTrain, XTest, yTrain, yTest = dataSetSplit(data.drop(columns = [\"Website\"]).values, data[\"Website\"].values)\n",
    "\n",
    "# Scale data\n",
    "ss = SS()\n",
    "XTrainScaled = ss.fit_transform(XTrain)\n",
    "XTestScaled = ss.transform(XTest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define top 3 models to use in bagging\n",
    "dtc = DTC(random_state = randomSeed, min_samples_split = 6)\n",
    "knn = KNN(n_neighbors = 6)\n",
    "lr = LR()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging - DTC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform hyperparameter optimization\n",
    "# Define hyperparameters to check\n",
    "hyperparameters = dict(max_features = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1], max_samples = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1], n_estimators = np.array(range(10, 110, 10)))\n",
    "\n",
    "# Set up and perform grid search\n",
    "grid = doGridSearch(BaggingClassifier(estimator = dtc, n_jobs = -1, random_state = randomSeed), hyperparameters, XTrain, yTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_max_samples</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.517256</td>\n",
       "      <td>898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.554268</td>\n",
       "      <td>895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>30</td>\n",
       "      <td>0.574146</td>\n",
       "      <td>892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>40</td>\n",
       "      <td>0.648110</td>\n",
       "      <td>858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.687866</td>\n",
       "      <td>793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>0.183354</td>\n",
       "      <td>911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70</td>\n",
       "      <td>0.178354</td>\n",
       "      <td>941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>80</td>\n",
       "      <td>0.183354</td>\n",
       "      <td>911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>90</td>\n",
       "      <td>0.170854</td>\n",
       "      <td>971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>0.175732</td>\n",
       "      <td>961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_features param_max_samples param_n_estimators  mean_test_score  \\\n",
       "0                  0.1               0.1                 10         0.517256   \n",
       "1                  0.1               0.1                 20         0.554268   \n",
       "2                  0.1               0.1                 30         0.574146   \n",
       "3                  0.1               0.1                 40         0.648110   \n",
       "4                  0.1               0.1                 50         0.687866   \n",
       "..                 ...               ...                ...              ...   \n",
       "995                  1                 1                 60         0.183354   \n",
       "996                  1                 1                 70         0.178354   \n",
       "997                  1                 1                 80         0.183354   \n",
       "998                  1                 1                 90         0.170854   \n",
       "999                  1                 1                100         0.175732   \n",
       "\n",
       "     rank_test_score  \n",
       "0                898  \n",
       "1                895  \n",
       "2                892  \n",
       "3                858  \n",
       "4                793  \n",
       "..               ...  \n",
       "995              911  \n",
       "996              941  \n",
       "997              911  \n",
       "998              971  \n",
       "999              961  \n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check grid search results\n",
    "gridDataframe = pd.DataFrame(grid.cv_results_)[[\"param_max_features\", \"param_max_samples\", \"param_n_estimators\", \"mean_test_score\", \"rank_test_score\"]]\n",
    "gridDataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_max_samples</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>70</td>\n",
       "      <td>0.861098</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.8</td>\n",
       "      <td>100</td>\n",
       "      <td>0.861098</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_features param_max_samples param_n_estimators  mean_test_score  \\\n",
       "466                0.5               0.7                 70         0.861098   \n",
       "479                0.5               0.8                100         0.861098   \n",
       "\n",
       "     rank_test_score  \n",
       "466                1  \n",
       "479                1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show best hyperparameter set\n",
    "bestHyperSet = gridDataframe[gridDataframe[\"rank_test_score\"] <= 1]\n",
    "bestHyperSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BaggingClassifier(estimator=DecisionTreeClassifier(min_samples_split=6,\n",
       "                                                   random_state=201),\n",
       "                  max_features=0.5, max_samples=0.7, n_estimators=70, n_jobs=-1,\n",
       "                  random_state=201)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BaggingClassifier</label><div class=\"sk-toggleable__content\"><pre>BaggingClassifier(estimator=DecisionTreeClassifier(min_samples_split=6,\n",
       "                                                   random_state=201),\n",
       "                  max_features=0.5, max_samples=0.7, n_estimators=70, n_jobs=-1,\n",
       "                  random_state=201)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(min_samples_split=6, random_state=201)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(min_samples_split=6, random_state=201)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "BaggingClassifier(estimator=DecisionTreeClassifier(min_samples_split=6,\n",
       "                                                   random_state=201),\n",
       "                  max_features=0.5, max_samples=0.7, n_estimators=70, n_jobs=-1,\n",
       "                  random_state=201)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Final model\n",
    "bagDTC = BaggingClassifier(estimator = dtc, n_jobs = -1, random_state = randomSeed, max_features = bestHyperSet[\"param_max_features\"].iloc(0)[0], max_samples = bestHyperSet[\"param_max_samples\"].iloc(0)[0], n_estimators = bestHyperSet[\"param_n_estimators\"].iloc(0)[0])\n",
    "bagDTC.fit(XTrain, yTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training score: 0.9306930693069307\n",
      "Testing score: 0.8333333333333334\n"
     ]
    }
   ],
   "source": [
    "# Score model\n",
    "score(bagDTC, XTrain, yTrain, XTest, yTest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/bagging_dtc.joblib']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save model\n",
    "dump(bagDTC, \"models/bagging_dtc.joblib\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging - KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\okisn\\anaconda3\\Lib\\site-packages\\sklearn\\model_selection\\_search.py:976: UserWarning: One or more of the test scores are non-finite: [0.48335366 0.51737805 0.56414634 0.62353659 0.64079268 0.61103659\n",
      " 0.59109756 0.62823171 0.63341463 0.63829268 0.56170732 0.57920732\n",
      " 0.61676829 0.67530488 0.71       0.6802439  0.67786585 0.7\n",
      " 0.70006098 0.70993902 0.60621951 0.66054878 0.68359756 0.74237805\n",
      " 0.7570122  0.70493902 0.7097561  0.7172561  0.7175     0.71993902\n",
      " 0.62115854 0.68786585 0.67359756 0.7225     0.7425     0.72487805\n",
      " 0.72481707 0.73719512 0.7325     0.73237805 0.60878049 0.66792683\n",
      " 0.67335366 0.75237805 0.75981707 0.74463415 0.7345122  0.7470122\n",
      " 0.75213415 0.75463415 0.65323171 0.69030488 0.68060976 0.7525\n",
      " 0.76457317 0.7570122  0.75682927 0.7645122  0.75207317 0.7472561\n",
      " 0.63567073 0.68768293 0.69304878 0.74256098 0.76207317 0.7520122\n",
      " 0.75926829 0.75457317 0.7497561  0.76689024 0.65323171 0.70286585\n",
      " 0.66829268 0.73993902 0.75963415 0.75932927 0.76445122 0.74969512\n",
      " 0.7595122  0.75945122 0.65030488 0.67536585 0.65804878 0.73481707\n",
      " 0.75445122 0.75189024 0.75189024 0.74457317 0.75689024 0.75945122\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan 0.48335366 0.51737805\n",
      " 0.56414634 0.62353659 0.64079268 0.61103659 0.59109756 0.62823171\n",
      " 0.63341463 0.63829268 0.56170732 0.57920732 0.61676829 0.67530488\n",
      " 0.71       0.6802439  0.67786585 0.7        0.70006098 0.70993902\n",
      " 0.60621951 0.66054878 0.68359756 0.74237805 0.7570122  0.70493902\n",
      " 0.7097561  0.7172561  0.7175     0.71993902 0.62115854 0.68786585\n",
      " 0.67359756 0.7225     0.7425     0.72487805 0.72481707 0.73719512\n",
      " 0.7325     0.73237805 0.60878049 0.66792683 0.67335366 0.75237805\n",
      " 0.75981707 0.74463415 0.7345122  0.7470122  0.75213415 0.75463415\n",
      " 0.65323171 0.69030488 0.68060976 0.7525     0.76457317 0.7570122\n",
      " 0.75682927 0.7645122  0.75207317 0.7472561  0.63567073 0.68768293\n",
      " 0.69304878 0.74256098 0.76207317 0.7520122  0.75926829 0.75457317\n",
      " 0.7497561  0.76689024 0.65323171 0.70286585 0.66829268 0.73993902\n",
      " 0.75963415 0.75932927 0.76445122 0.74969512 0.7595122  0.75945122\n",
      " 0.65030488 0.67536585 0.65804878 0.73481707 0.75445122 0.75189024\n",
      " 0.75189024 0.74457317 0.75689024 0.75945122        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan 0.58884146 0.62341463 0.61865854 0.65810976\n",
      " 0.65073171 0.65067073 0.64560976 0.64341463 0.66073171 0.65573171\n",
      " 0.69       0.6852439  0.70231707 0.7097561  0.71237805 0.70731707\n",
      " 0.70231707 0.71231707 0.71487805 0.71231707 0.73457317 0.71493902\n",
      " 0.73963415 0.74207317 0.7520122  0.74945122 0.74439024 0.75207317\n",
      " 0.74231707 0.7397561  0.7470122  0.74207317 0.74707317 0.75445122\n",
      " 0.7595122  0.7620122  0.75695122 0.76469512 0.76731707 0.76707317\n",
      " 0.76926829 0.75445122 0.76439024 0.76439024 0.77426829 0.77670732\n",
      " 0.77414634 0.77682927 0.77695122 0.78432927 0.79396341 0.76932927\n",
      " 0.76195122 0.76439024 0.78176829 0.77926829 0.76926829 0.78676829\n",
      " 0.77682927 0.78176829 0.78164634 0.77176829 0.76707317 0.7745122\n",
      " 0.77439024 0.78445122 0.77176829 0.77939024 0.78182927 0.78926829\n",
      " 0.78164634 0.78170732 0.77939024 0.78695122 0.77939024 0.77939024\n",
      " 0.77432927 0.77189024 0.77432927 0.78164634 0.77926829 0.77195122\n",
      " 0.77432927 0.77457317 0.76445122 0.77689024 0.76932927 0.76707317\n",
      " 0.76207317 0.76689024        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      " 0.57652439 0.62615854 0.64310976 0.63829268 0.63591463 0.62847561\n",
      " 0.64317073 0.65317073 0.65054878 0.65817073 0.66579268 0.68993902\n",
      " 0.6877439  0.68036585 0.7025     0.69518293 0.70006098 0.70262195\n",
      " 0.70518293 0.70518293 0.7102439  0.72469512 0.73963415 0.7570122\n",
      " 0.7595122  0.7570122  0.76457317 0.7570122  0.76695122 0.76945122\n",
      " 0.72518293 0.76189024 0.76432927 0.77176829 0.77920732 0.78426829\n",
      " 0.78652439 0.77432927 0.77439024 0.7770122  0.73981707 0.77676829\n",
      " 0.78176829 0.77670732 0.79170732 0.78664634 0.79414634 0.79432927\n",
      " 0.79920732 0.80420732 0.7572561  0.77932927 0.78670732 0.78426829\n",
      " 0.80408537 0.80152439 0.80402439 0.79926829 0.80170732 0.80414634\n",
      " 0.77939024 0.79908537 0.79676829 0.79914634 0.81640244 0.80890244\n",
      " 0.81140244 0.79902439 0.79664634 0.80658537 0.77939024 0.78676829\n",
      " 0.79670732 0.79676829 0.81634146 0.80896341 0.81384146 0.78664634\n",
      " 0.79646341 0.79652439 0.78926829 0.79664634 0.80658537 0.81152439\n",
      " 0.82378049 0.81134146 0.80396341 0.80152439 0.79896341 0.80390244\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan 0.49487805 0.59658537\n",
      " 0.62121951 0.61609756 0.61884146 0.64103659 0.63853659 0.61884146\n",
      " 0.62871951 0.61878049 0.58420732 0.68304878 0.6904878  0.67573171\n",
      " 0.69310976 0.70012195 0.71262195 0.69560976 0.69304878 0.69060976\n",
      " 0.64323171 0.71006098 0.74469512 0.74       0.77445122 0.75939024\n",
      " 0.77432927 0.76695122 0.74469512 0.74469512 0.66573171 0.75170732\n",
      " 0.76195122 0.75207317 0.77439024 0.77920732 0.78664634 0.76445122\n",
      " 0.76939024 0.7570122  0.68768293 0.7397561  0.76445122 0.78189024\n",
      " 0.78432927 0.79658537 0.79670732 0.78926829 0.77445122 0.7695122\n",
      " 0.68042683 0.74695122 0.74707317 0.76713415 0.77695122 0.80396341\n",
      " 0.81152439 0.79426829 0.77439024 0.77932927 0.70018293 0.75945122\n",
      " 0.77445122 0.78195122 0.79676829 0.81884146 0.82146341 0.79408537\n",
      " 0.77939024 0.78682927 0.705      0.77414634 0.77695122 0.78439024\n",
      " 0.79170732 0.80634146 0.81640244 0.80158537 0.79914634 0.79170732\n",
      " 0.70237805 0.77164634 0.78664634 0.77682927 0.79158537 0.80140244\n",
      " 0.81128049 0.80408537 0.79408537 0.79414634        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan 0.46243902 0.55195122 0.52670732 0.53908537\n",
      " 0.55896341 0.57646341 0.57634146 0.56646341 0.56896341 0.57634146\n",
      " 0.55195122 0.66823171 0.61859756 0.64353659 0.66329268 0.67079268\n",
      " 0.67817073 0.66823171 0.67317073 0.66067073 0.59152439 0.6854878\n",
      " 0.67804878 0.70292683 0.70780488 0.74231707 0.73231707 0.7004878\n",
      " 0.71018293 0.7077439  0.64103659 0.72231707 0.70518293 0.72487805\n",
      " 0.7347561  0.76213415 0.75707317 0.72981707 0.72737805 0.7397561\n",
      " 0.66560976 0.7225     0.70993902 0.71737805 0.71993902 0.75963415\n",
      " 0.73963415 0.73231707 0.73237805 0.73231707 0.66292683 0.7197561\n",
      " 0.7        0.71981707 0.71487805 0.74463415 0.74213415 0.72731707\n",
      " 0.7322561  0.7297561  0.66304878 0.72731707 0.69506098 0.7122561\n",
      " 0.71487805 0.75219512 0.75957317 0.73469512 0.7272561  0.7297561\n",
      " 0.6529878  0.72469512 0.70487805 0.72207317 0.71737805 0.7645122\n",
      " 0.7620122  0.74213415 0.7347561  0.73963415 0.65542683 0.72719512\n",
      " 0.71707317 0.7295122  0.72963415 0.76939024 0.76432927 0.75445122\n",
      " 0.7445122  0.7570122         nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      " 0.46012195 0.47743902 0.4845122  0.51189024 0.52926829 0.51439024\n",
      " 0.51445122 0.52182927 0.52676829 0.52920732 0.53469512 0.58408537\n",
      " 0.60359756 0.62591463 0.61597561 0.61353659 0.62591463 0.61121951\n",
      " 0.62103659 0.62597561 0.58176829 0.62371951 0.64317073 0.64823171\n",
      " 0.65573171 0.64817073 0.66067073 0.64323171 0.64817073 0.66554878\n",
      " 0.62371951 0.66310976 0.6827439  0.68530488 0.6704878  0.67762195\n",
      " 0.6827439  0.66786585 0.67280488 0.6952439  0.62859756 0.66786585\n",
      " 0.68512195 0.6827439  0.67780488 0.68512195 0.69512195 0.6802439\n",
      " 0.68762195 0.69762195 0.63347561 0.6677439  0.68262195 0.68012195\n",
      " 0.6802439  0.69487805 0.70481707 0.69743902 0.69993902 0.71481707\n",
      " 0.62347561 0.66786585 0.695      0.68743902 0.67768293 0.68256098\n",
      " 0.69487805 0.68756098 0.69737805 0.70987805 0.62835366 0.6902439\n",
      " 0.69743902 0.69231707 0.6825     0.68743902 0.69993902 0.6925\n",
      " 0.69493902 0.71731707 0.63323171 0.70481707 0.70219512 0.7072561\n",
      " 0.69987805 0.70981707 0.71969512 0.70969512 0.7122561  0.72713415\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan 0.46743902 0.46\n",
      " 0.4820122  0.49439024 0.49939024 0.51939024 0.52670732 0.51426829\n",
      " 0.52420732 0.52926829 0.53719512 0.57652439 0.57402439 0.59865854\n",
      " 0.60115854 0.58146341 0.59621951 0.59371951 0.60615854 0.60597561\n",
      " 0.57920732 0.60890244 0.61103659 0.60859756 0.61365854 0.61817073\n",
      " 0.61323171 0.61085366 0.62079268 0.63073171 0.62121951 0.65823171\n",
      " 0.63841463 0.63097561 0.62121951 0.6529878  0.6702439  0.63079268\n",
      " 0.64329268 0.66786585 0.63353659 0.65304878 0.62829268 0.63579268\n",
      " 0.63829268 0.6652439  0.67268293 0.64804878 0.65542683 0.68268293\n",
      " 0.62853659 0.6604878  0.6429878  0.64054878 0.64554878 0.6727439\n",
      " 0.68256098 0.66042683 0.6777439  0.68262195 0.63335366 0.65060976\n",
      " 0.66292683 0.65780488 0.66036585 0.66286585 0.6777439  0.67280488\n",
      " 0.68512195 0.70237805 0.64317073 0.68030488 0.66536585 0.67512195\n",
      " 0.66792683 0.6925     0.695      0.68762195 0.6925     0.69987805\n",
      " 0.64554878 0.69256098 0.6752439  0.68756098 0.68512195 0.69737805\n",
      " 0.69987805 0.69737805 0.69981707 0.70731707        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan 0.46268293 0.45006098 0.4647561  0.47957317\n",
      " 0.48713415 0.49207317 0.50457317 0.49707317 0.4945122  0.49219512\n",
      " 0.51743902 0.53945122 0.55926829 0.56402439 0.55432927 0.57140244\n",
      " 0.56640244 0.57371951 0.57121951 0.55158537 0.55445122 0.57664634\n",
      " 0.58890244 0.58396341 0.57414634 0.58628049 0.59347561 0.58871951\n",
      " 0.58121951 0.58865854 0.57414634 0.61365854 0.60871951 0.58390244\n",
      " 0.58140244 0.60121951 0.60115854 0.59371951 0.59871951 0.59621951\n",
      " 0.59890244 0.62347561 0.62585366 0.60353659 0.61591463 0.62573171\n",
      " 0.62829268 0.62579268 0.62579268 0.62335366 0.59140244 0.62103659\n",
      " 0.61835366 0.61347561 0.62073171 0.63317073 0.63804878 0.63060976\n",
      " 0.63554878 0.63054878 0.60878049 0.63067073 0.63317073 0.61841463\n",
      " 0.62823171 0.63310976 0.6479878  0.64304878 0.6504878  0.64060976\n",
      " 0.61109756 0.63835366 0.64310976 0.63329268 0.63317073 0.64560976\n",
      " 0.65792683 0.6504878  0.66292683 0.6504878  0.61109756 0.6504878\n",
      " 0.64323171 0.64310976 0.64804878 0.65792683 0.66542683 0.66042683\n",
      " 0.66786585 0.66536585        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      " 0.48335366 0.51737805 0.56414634 0.62353659 0.64079268 0.61103659\n",
      " 0.59109756 0.62823171 0.63341463 0.63829268 0.56170732 0.57920732\n",
      " 0.61676829 0.67530488 0.71       0.6802439  0.67786585 0.7\n",
      " 0.70006098 0.70993902 0.60621951 0.66054878 0.68359756 0.74237805\n",
      " 0.7570122  0.70493902 0.7097561  0.7172561  0.7175     0.71993902\n",
      " 0.62115854 0.68786585 0.67359756 0.7225     0.7425     0.72487805\n",
      " 0.72481707 0.73719512 0.7325     0.73237805 0.60878049 0.66792683\n",
      " 0.67335366 0.75237805 0.75981707 0.74463415 0.7345122  0.7470122\n",
      " 0.75213415 0.75463415 0.65323171 0.69030488 0.68060976 0.7525\n",
      " 0.76457317 0.7570122  0.75682927 0.7645122  0.75207317 0.7472561\n",
      " 0.63567073 0.68768293 0.69304878 0.74256098 0.76207317 0.7520122\n",
      " 0.75926829 0.75457317 0.7497561  0.76689024 0.65323171 0.70286585\n",
      " 0.66829268 0.73993902 0.75963415 0.75932927 0.76445122 0.74969512\n",
      " 0.7595122  0.75945122 0.65030488 0.67536585 0.65804878 0.73481707\n",
      " 0.75445122 0.75189024 0.75189024 0.74457317 0.75689024 0.75945122\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan]\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Perform hyperparameter optimization\n",
    "# Set up and perform grid search\n",
    "grid = doGridSearch(BaggingClassifier(estimator = knn, n_jobs = -1, random_state = randomSeed), hyperparameters, XTrainScaled, yTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_max_samples</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.483354</td>\n",
       "      <td>888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.517378</td>\n",
       "      <td>871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>30</td>\n",
       "      <td>0.564146</td>\n",
       "      <td>843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>40</td>\n",
       "      <td>0.623537</td>\n",
       "      <td>736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.640793</td>\n",
       "      <td>677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70</td>\n",
       "      <td>NaN</td>\n",
       "      <td>901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>80</td>\n",
       "      <td>NaN</td>\n",
       "      <td>901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>901</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_features param_max_samples param_n_estimators  mean_test_score  \\\n",
       "0                  0.1               0.1                 10         0.483354   \n",
       "1                  0.1               0.1                 20         0.517378   \n",
       "2                  0.1               0.1                 30         0.564146   \n",
       "3                  0.1               0.1                 40         0.623537   \n",
       "4                  0.1               0.1                 50         0.640793   \n",
       "..                 ...               ...                ...              ...   \n",
       "995                  1                 1                 60              NaN   \n",
       "996                  1                 1                 70              NaN   \n",
       "997                  1                 1                 80              NaN   \n",
       "998                  1                 1                 90              NaN   \n",
       "999                  1                 1                100              NaN   \n",
       "\n",
       "     rank_test_score  \n",
       "0                888  \n",
       "1                871  \n",
       "2                843  \n",
       "3                736  \n",
       "4                677  \n",
       "..               ...  \n",
       "995              901  \n",
       "996              901  \n",
       "997              901  \n",
       "998              901  \n",
       "999              901  \n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check grid search results\n",
    "gridDataframe = pd.DataFrame(grid.cv_results_)[[\"param_max_features\", \"param_max_samples\", \"param_n_estimators\", \"mean_test_score\", \"rank_test_score\"]]\n",
    "gridDataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_max_samples</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>0.4</td>\n",
       "      <td>0.9</td>\n",
       "      <td>50</td>\n",
       "      <td>0.82378</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_features param_max_samples param_n_estimators  mean_test_score  \\\n",
       "384                0.4               0.9                 50          0.82378   \n",
       "\n",
       "     rank_test_score  \n",
       "384                1  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show best hyperparameter set\n",
    "bestHyperSet = gridDataframe[gridDataframe[\"rank_test_score\"] <= 1]\n",
    "bestHyperSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BaggingClassifier(estimator=KNeighborsClassifier(n_neighbors=6),\n",
       "                  max_features=0.4, max_samples=0.9, n_estimators=50, n_jobs=-1,\n",
       "                  random_state=201)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BaggingClassifier</label><div class=\"sk-toggleable__content\"><pre>BaggingClassifier(estimator=KNeighborsClassifier(n_neighbors=6),\n",
       "                  max_features=0.4, max_samples=0.9, n_estimators=50, n_jobs=-1,\n",
       "                  random_state=201)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier(n_neighbors=6)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier(n_neighbors=6)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "BaggingClassifier(estimator=KNeighborsClassifier(n_neighbors=6),\n",
       "                  max_features=0.4, max_samples=0.9, n_estimators=50, n_jobs=-1,\n",
       "                  random_state=201)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Final model\n",
    "bagKNN = BaggingClassifier(estimator = knn, n_jobs = -1, random_state = randomSeed, max_features = bestHyperSet[\"param_max_features\"].iloc(0)[0], max_samples = bestHyperSet[\"param_max_samples\"].iloc(0)[0], n_estimators = bestHyperSet[\"param_n_estimators\"].iloc(0)[0])\n",
    "bagKNN.fit(XTrainScaled, yTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training score: 0.8836633663366337\n",
      "Testing score: 0.7843137254901961\n"
     ]
    }
   ],
   "source": [
    "# Score model\n",
    "score(bagKNN, XTrainScaled, yTrain, XTestScaled, yTest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/bagging_knn.joblib']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save model\n",
    "dump(bagKNN, \"models/bagging_knn.joblib\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging - Log Reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform hyperparameter optimization\n",
    "# Set up and perform grid search\n",
    "grid = doGridSearch(BaggingClassifier(estimator = lr, n_jobs = -1, random_state = randomSeed), hyperparameters, XTrainScaled, yTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_max_samples</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.344268</td>\n",
       "      <td>625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.289695</td>\n",
       "      <td>826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>30</td>\n",
       "      <td>0.269695</td>\n",
       "      <td>886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>40</td>\n",
       "      <td>0.296646</td>\n",
       "      <td>787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.318963</td>\n",
       "      <td>675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>0.183354</td>\n",
       "      <td>927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>70</td>\n",
       "      <td>0.178354</td>\n",
       "      <td>947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>80</td>\n",
       "      <td>0.165976</td>\n",
       "      <td>981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>90</td>\n",
       "      <td>0.170854</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>0.175732</td>\n",
       "      <td>957</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_features param_max_samples param_n_estimators  mean_test_score  \\\n",
       "0                  0.1               0.1                 10         0.344268   \n",
       "1                  0.1               0.1                 20         0.289695   \n",
       "2                  0.1               0.1                 30         0.269695   \n",
       "3                  0.1               0.1                 40         0.296646   \n",
       "4                  0.1               0.1                 50         0.318963   \n",
       "..                 ...               ...                ...              ...   \n",
       "995                  1                 1                 60         0.183354   \n",
       "996                  1                 1                 70         0.178354   \n",
       "997                  1                 1                 80         0.165976   \n",
       "998                  1                 1                 90         0.170854   \n",
       "999                  1                 1                100         0.175732   \n",
       "\n",
       "     rank_test_score  \n",
       "0                625  \n",
       "1                826  \n",
       "2                886  \n",
       "3                787  \n",
       "4                675  \n",
       "..               ...  \n",
       "995              927  \n",
       "996              947  \n",
       "997              981  \n",
       "998              968  \n",
       "999              957  \n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check grid search results\n",
    "gridDataframe = pd.DataFrame(grid.cv_results_)[[\"param_max_features\", \"param_max_samples\", \"param_n_estimators\", \"mean_test_score\", \"rank_test_score\"]]\n",
    "gridDataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_max_samples</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.4</td>\n",
       "      <td>90</td>\n",
       "      <td>0.490122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    param_max_features param_max_samples param_n_estimators  mean_test_score  \\\n",
       "438                0.5               0.4                 90         0.490122   \n",
       "\n",
       "     rank_test_score  \n",
       "438                1  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show best hyperparameter set\n",
    "bestHyperSet = gridDataframe[gridDataframe[\"rank_test_score\"] <= 1]\n",
    "bestHyperSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BaggingClassifier(estimator=LogisticRegression(), max_features=0.5,\n",
       "                  max_samples=0.4, n_estimators=90, n_jobs=-1,\n",
       "                  random_state=201)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BaggingClassifier</label><div class=\"sk-toggleable__content\"><pre>BaggingClassifier(estimator=LogisticRegression(), max_features=0.5,\n",
       "                  max_samples=0.4, n_estimators=90, n_jobs=-1,\n",
       "                  random_state=201)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "BaggingClassifier(estimator=LogisticRegression(), max_features=0.5,\n",
       "                  max_samples=0.4, n_estimators=90, n_jobs=-1,\n",
       "                  random_state=201)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Final model\n",
    "bagLR = BaggingClassifier(estimator = lr, n_jobs = -1, random_state = randomSeed, max_features = bestHyperSet[\"param_max_features\"].iloc(0)[0], max_samples = bestHyperSet[\"param_max_samples\"].iloc(0)[0], n_estimators = bestHyperSet[\"param_n_estimators\"].iloc(0)[0])\n",
    "bagLR.fit(XTrainScaled, yTrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training score: 0.48514851485148514\n",
      "Testing score: 0.49019607843137253\n"
     ]
    }
   ],
   "source": [
    "# Score model\n",
    "score(bagLR, XTrainScaled, yTrain, XTestScaled, yTest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/bagging_log_reg.joblib']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save model\n",
    "dump(bagLR, \"models/bagging_log_reg.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}