full-time_other_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92964322",
   "metadata": {},
   "outputs": [],
   "source": [
    "#general package\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from scipy.stats import pearsonr\n",
    "from scipy.stats import chi2_contingency\n",
    "from scipy.stats import ttest_ind\n",
    "from scipy import stats\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "import random\n",
    "import tensorflow as tf \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "92b9eb22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Result</th>\n",
       "      <th>match_number</th>\n",
       "      <th>team_code</th>\n",
       "      <th>opponent_code</th>\n",
       "      <th>gender</th>\n",
       "      <th>season</th>\n",
       "      <th>team_score</th>\n",
       "      <th>team_wickets</th>\n",
       "      <th>winner_code</th>\n",
       "      <th>toss_winner_code</th>\n",
       "      <th>home_advantage</th>\n",
       "      <th>venue_city_code</th>\n",
       "      <th>choose_to_bat</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>avg_win_rate</th>\n",
       "      <th>avg_wickets_out</th>\n",
       "      <th>last_match_result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2021</td>\n",
       "      <td>132</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>139.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2021</td>\n",
       "      <td>113</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>128.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2021</td>\n",
       "      <td>149</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>131.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2021</td>\n",
       "      <td>110</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>130.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2021</td>\n",
       "      <td>151</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2022</td>\n",
       "      <td>139</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140.642857</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>6.571429</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2022</td>\n",
       "      <td>135</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>152.714286</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2022</td>\n",
       "      <td>143</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>145.500000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>5.785714</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2022</td>\n",
       "      <td>150</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>140.533333</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>6.533333</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>2022</td>\n",
       "      <td>120</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>148.800000</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>115 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Result  match_number  team_code  opponent_code  gender  season  \\\n",
       "0         1             5          4              1       0    2021   \n",
       "1         1             6          0              2       0    2021   \n",
       "2         1             7          3              6       0    2021   \n",
       "3         0             8          7              5       0    2021   \n",
       "4         1            10          6              1       0    2021   \n",
       "..      ...           ...        ...            ...     ...     ...   \n",
       "110       0            30          1              0       1    2022   \n",
       "111       1            31          3              5       1    2022   \n",
       "112       0            32          4              2       1    2022   \n",
       "113       0            40          1              2       1    2022   \n",
       "114       0            50          2              6       1    2022   \n",
       "\n",
       "     team_score  team_wickets  winner_code  toss_winner_code  home_advantage  \\\n",
       "0           132             7            4                 1               0   \n",
       "1           113             9            0                 0               0   \n",
       "2           149             7            3                 3               0   \n",
       "3           110             7            5                 5               1   \n",
       "4           151             4            6                 6               0   \n",
       "..          ...           ...          ...               ...             ...   \n",
       "110         139             6            0                 0               1   \n",
       "111         135             8            3                 5               1   \n",
       "112         143             9            2                 4               0   \n",
       "113         150             7            2                 1               0   \n",
       "114         120             9            6                 2               0   \n",
       "\n",
       "     venue_city_code  choose_to_bat   avg_score  avg_win_rate  \\\n",
       "0                  1              0  139.000000      1.000000   \n",
       "1                  2              1  128.000000      0.000000   \n",
       "2                  6              1  131.000000      1.000000   \n",
       "3                  5              0  130.000000      0.000000   \n",
       "4                  1              1  116.000000      0.000000   \n",
       "..               ...            ...         ...           ...   \n",
       "110                0              0  140.642857      0.428571   \n",
       "111                5              0  152.714286      0.428571   \n",
       "112                2              1  145.500000      0.571429   \n",
       "113                2              1  140.533333      0.400000   \n",
       "114                6              1  148.800000      0.533333   \n",
       "\n",
       "     avg_wickets_out  last_match_result  \n",
       "0           5.000000                  1  \n",
       "1           6.000000                  0  \n",
       "2           4.000000                  1  \n",
       "3           8.000000                  0  \n",
       "4           7.000000                  0  \n",
       "..               ...                ...  \n",
       "110         6.571429                  0  \n",
       "111         6.000000                  1  \n",
       "112         5.785714                  1  \n",
       "113         6.533333                  0  \n",
       "114         6.000000                  1  \n",
       "\n",
       "[115 rows x 17 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bat = pd.read_csv('/Users/quinne/Desktop/all_code/dissertation_dataset/halftime_encode_python.csv')\n",
    "df_bat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0543641e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 115 entries, 0 to 114\n",
      "Data columns (total 17 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   Result             115 non-null    int64  \n",
      " 1   match_number       115 non-null    int64  \n",
      " 2   team_code          115 non-null    int64  \n",
      " 3   opponent_code      115 non-null    int64  \n",
      " 4   gender             115 non-null    int64  \n",
      " 5   season             115 non-null    int64  \n",
      " 6   team_score         115 non-null    int64  \n",
      " 7   team_wickets       115 non-null    int64  \n",
      " 8   winner_code        115 non-null    int64  \n",
      " 9   toss_winner_code   115 non-null    int64  \n",
      " 10  home_advantage     115 non-null    int64  \n",
      " 11  venue_city_code    115 non-null    int64  \n",
      " 12  choose_to_bat      115 non-null    int64  \n",
      " 13  avg_score          115 non-null    float64\n",
      " 14  avg_win_rate       115 non-null    float64\n",
      " 15  avg_wickets_out    115 non-null    float64\n",
      " 16  last_match_result  115 non-null    int64  \n",
      "dtypes: float64(3), int64(14)\n",
      "memory usage: 15.4 KB\n"
     ]
    }
   ],
   "source": [
    "df_bat.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26ca0bcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(115, 17)\n"
     ]
    }
   ],
   "source": [
    "print(df_bat.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "35796fce",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_keep = ['Result', \"match_number\", \"team_code\", \"opponent_code\", \n",
    "                   \"gender\", 'season', 'home_advantage', 'venue_city_code',\n",
    "                   'avg_score', 'avg_win_rate', 'avg_wickets_out', 'last_match_result']\n",
    "df = df_bat[columns_to_keep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "deabcd54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(115, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "25ed2951",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(\"Result\", axis=1)\n",
    "y = df['Result']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dbfba781",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "636bace7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set random seeds\n",
    "np.random.seed(0)\n",
    "random.seed(0)\n",
    "tf.random.set_seed(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8d945b5",
   "metadata": {},
   "source": [
    "### KNN (K-Nearest Neighbors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b0bede55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for KNN: {'metric': 'euclidean', 'n_neighbors': 10, 'weights': 'uniform'}\n"
     ]
    }
   ],
   "source": [
    "# Standardizing the data\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "\n",
    "# Finding the optimal parameters for KNN\n",
    "knn = KNeighborsClassifier()\n",
    "param_grid = {\n",
    "    'n_neighbors': np.arange(1, 51),\n",
    "    'weights': ['uniform', 'distance'],\n",
    "    'metric': ['euclidean', 'manhattan']\n",
    "}\n",
    "\n",
    "knn_gscv = GridSearchCV(knn, param_grid, cv=10, scoring='accuracy')\n",
    "knn_gscv.fit(X_scaled, y)\n",
    "\n",
    "print(\"Best parameters for KNN:\", knn_gscv.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e7ec4ca7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted Probabilities for KNN:\n",
      "[0.5 0.5 0.7 0.2 0.6 0.3 0.2 0.6 0.4 0.1 0.3 0.6 0.2 0.9 0.6 0.1 0.4 0.5\n",
      " 0.8 0.4 0.3 0.2 0.4 0.7 0.1 0.4 0.3 0.5 0.6 0.5 0.3 0.3 0.2 0.5 0.4 0.5\n",
      " 0.4 0.3 0.5 0.5 0.5 0.4 0.7 0.5 0.5 0.1 0.3 0.4 0.5 0.2 0.3 0.5 0.4 0.3\n",
      " 0.3 0.4 0.4 0.5 0.2 0.5 0.5 0.7 0.3 0.6 0.7 0.5 0.3 0.5 0.4 0.5 0.6 0.4\n",
      " 0.3 0.4 0.5 0.4 0.6 0.5 0.3 0.5 0.4 0.4 0.6 0.5 0.6 0.5 0.5 0.7 0.5 0.8\n",
      " 0.4 0.5 0.6 0.7 0.5 0.5 0.6 0.4 0.4 0.4 0.6 0.5 0.6 0.5 0.6 0.2 0.4 0.7\n",
      " 0.4 0.7 0.3 0.4 0.4 0.5 0.4]\n"
     ]
    }
   ],
   "source": [
    "#Best parameters for KNN: {'metric': 'euclidean', 'n_neighbors': 10, 'weights': 'uniform'}\n",
    "\n",
    "knn = KNeighborsClassifier(metric='euclidean', n_neighbors=10, weights='uniform')\n",
    "y_prob_knn = np.zeros_like(y, dtype=float)\n",
    "\n",
    "for train_idx, test_idx in cv.split(X, y):\n",
    "    scaler = StandardScaler()\n",
    "    \n",
    "    X_train = scaler.fit_transform(X.iloc[train_idx])\n",
    "    X_test = scaler.transform(X.iloc[test_idx])\n",
    "    \n",
    "    y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n",
    "    \n",
    "    knn.fit(X_train, y_train)\n",
    "    y_prob_knn[test_idx] = knn.predict_proba(X_test)[:, 1]\n",
    "\n",
    "y_pred_knn = (y_prob_knn > 0.5).astype(int)\n",
    "\n",
    "print(\"Predicted Probabilities for KNN:\")\n",
    "print(y_prob_knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "de31f1dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,\n",
       "       0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1,\n",
       "       0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1,\n",
       "       0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f402a4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAHHCAYAAACiDxGKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzG0lEQVR4nO3deXRU9f3/8dcNJBMgCxKEgCSAiCAFXNiMaEAWCQgKpHUByyLVYiNfIC7AtyIgYsAFUEGgaFmUxaJAK1ZRUIIoCAYDYiUKYoNCAug3CQlmCMn8/uhxfh3DMjfcyST3Ph+ee07mc+/c+54e7Wven7uM4fF4PAIAALYREuwCAACAtQh3AABshnAHAMBmCHcAAGyGcAcAwGYIdwAAbIZwBwDAZgh3AABshnAHAMBmCHcAAGyGcAeCYOnSpTIMQ5999pnPeH5+vjp37qzw8HC9++67mjp1qgzDUMOGDXXq1Kly+2nWrJn69+/vM2YYhgzD0HPPPef3cQHYC+EOVBEFBQW65ZZbtHfvXq1bt05JSUnedceOHdOCBQtM7e+ZZ5456xcCAPZHuANVwMmTJ9WnTx9lZmbqzTffVN++fX3WX3PNNXrmmWf0888/+7W/a665Rrm5uVq4cGEgygVQxRHuQJAVFhYqKSlJu3fv1ptvvqlbb7213DaPP/64cnNz/e7eu3btqh49eujpp5/2+wsBAPsg3IEgKioqUt++fbVr1y6tWbOm3PnzX9x0002mw3rq1KmmvhAAsA/CHQii4cOH69NPP9WaNWt02223nXfbKVOmmJpqv+mmm3TzzTebms4HYA+EOxBEubm5Cg8PV1xc3AW3TUxM1M0332y6e8/JyeHcO+AwhDsQRIsWLVJYWJiSkpKUlZV1we3NhnVFvhAAqP4IdyCI2rRpo3/+85/6+eef1bt3bx0+fPi82ycmJqp79+6mwnrKlCnKycnRokWLrCgZQDVAuANB1rlzZ61fv17Hjh1T7969dfz48fNu/0v37m9Yd+vWTd27d9esWbPo3gGHINyBKqBnz55atWqVDhw4oKSkJBUUFJxz2/8O6+LiYr/2/8sXgr/85S9WlQygCiPcgSpi0KBBWrx4sXbv3q3bbrvtvMH9y1R7bm6uX/vu1q2bunXrpszMTIuqBVCVEe5AFTJy5Eg9++yzSk9P1+9+9zudOXPmrNt1795d3bp1M7XvqVOnWlAhgOrA8Hg8nmAXAQAArEPnDgCAzRDuAADYDOEOAIDNEO4AANgM4Q4AgM0Q7gAA2AzhDgCAzdQMdgGB0M9ICXYJQEDdZdQLdglAQA0rmx7wY1iVFf/0zLdkP1aicwcAwGZs2bkDAHAhdu5uCXcAgCMZMoJdQsAQ7gAAR7Jz527nzwYAgCPRuQMAHIlpeQAAbMbOU9d2/mwAADgSnTsAwJHsOylPuAMAHCrExvHOtDwAADZD5w4AcCT79u2EOwDAoZiWBwAA1QadOwDAkezbtxPuAACHsvPUNeEOAHAkOz9+1s5fXAAAcCQ6dwCAI9m5uyXcAQCOxK1wAACg2qBzBwA4kn37dsIdAOBQIYZ9451peQAAbIbOHQDgSPbt2wl3AIBD2Xnq2s6fDQCAKm3mzJkyDEPjxo3zjnXv3l2GYfgso0ePNrVfOncAgCMF+/Gzu3bt0qJFi9S+ffty6+677z498cQT3te1a9c2tW86dwCAI4VYtFREYWGhhg4dqsWLF+uSSy4pt7527dqKjY31LlFRUaY/GwAAjmNY9I/b7VZBQYHP4na7z3vslJQU3XrrrerVq9dZ169YsUL169dX27ZtNWnSJJ06dcrUZyPcAQC4CGlpaYqOjvZZ0tLSzrn96tWrtXv37nNuM2TIEL322mv68MMPNWnSJL366qu65557TNXEOXcAgCNZ1d1OmjRJqampPmMul+us2x4+fFhjx47V+++/r/Dw8LNuc//993v/bteunRo1aqSePXvq4MGDatGihV81Ee4AAEey6nI6l8t1zjD/tYyMDB07dkzXXXedd6y0tFRbt27VvHnz5Ha7VaNGDZ/3dOnSRZJ04MABwh0AgKqmZ8+e+uKLL3zGRo4cqdatW2vChAnlgl2SMjMzJUmNGjXy+ziEOwDAkYLxk6+RkZFq27atz1idOnUUExOjtm3b6uDBg1q5cqX69eunmJgY7d27V+PHj1diYuJZb5k7F8IdAOBIVfHxs2FhYdq0aZPmzp2roqIixcXFKTk5WY899pip/RDuAAAE0ZYtW7x/x8XFKT09/aL3SbgDABwpGNPylYVwBwA4kp0f9GLnzwYAgCPRuQMAHMm+k/KEOwDAoTjnDgCAzdg32jnnDgCA7dC5AwAciWl5AABsxr7RzrQ8AAC2Q+cOAHAkO3e3hDsAwJGYlgcAANUGnTsAwJG4Wh4AAJux89S1nT8bAACOROcOAHAk+07KE+4AAIey89Q14Q4AcCQ7X1Bn5y8uAAA4Ep07AMCR7Nu3E+4AAIey89S1nT8bAACOROcOAHAkO3e3hDsAwJHsfM7dzl9cAABwJDp3AIAj2fk+d8IdAOBI9o12puUBALAdOncAgCPZubsl3AEAjkS4AwBgM4aNz7rb+YsLAACOROcOAHAkO3e3hDsAwJHsHO52/mwAADgSnTsAwJHsezkd4Q4AcCg7T13b+bMBAOBIdO4AAEfiPncAAGwmxKLlYsycOVOGYWjcuHHeseLiYqWkpCgmJkYRERFKTk5Wbm6uqf0S7gAABMGuXbu0aNEitW/f3md8/Pjxeuutt7RmzRqlp6fryJEjGjx4sKl9E+4AAEcKZudeWFiooUOHavHixbrkkku84/n5+XrllVc0e/Zs9ejRQx06dNCSJUv0ySefaMeOHaY+GwAAjmNYtLjdbhUUFPgsbrf7vMdOSUnRrbfeql69evmMZ2RkqKSkxGe8devWio+P1/bt2/3+bIQ7AMCRQmRYsqSlpSk6OtpnSUtLO+dxV69erd27d591m5ycHIWFhalu3bo+4w0bNlROTo7fn42r5QEAuAiTJk1Samqqz5jL5TrrtocPH9bYsWP1/vvvKzw8PGA1Ee4AAEeyaura5XKdM8x/LSMjQ8eOHdN1113nHSstLdXWrVs1b948bdy4UadPn1ZeXp5P956bm6vY2Fi/ayLcAQCOFIy73Hv27KkvvvjCZ2zkyJFq3bq1JkyYoLi4OIWGhmrz5s1KTk6WJGVlZSk7O1sJCQl+H4dwBwCgkkRGRqpt27Y+Y3Xq1FFMTIx3fNSoUUpNTVW9evUUFRWlMWPGKCEhQddff73fx+GCOlTI7yb01j8983X/nGTv2CUNo/Tw8uF67Wia1hbO1gsZE9R18DXBKxKooLYTbtKwsunqOKevdyzEVVOd5/XXnccn6e6Cx9RtzV0Kb1AniFXiYlWFh9iczZw5c9S/f38lJycrMTFRsbGxWrt2ral90LnDtJYd49X3jzfq2z3f+4w/tHyY6tStpSduW6iCE4XqPqSTJv5tlMZ2nKVvM78/x96AqiWm42VqeX8n/bTH98rkTnP6qkm/K5V+x2qdznery4u3qvubd+vdm14OUqW4WFWlu92yZYvP6/DwcM2fP1/z58+v8D6rymdDNRFex6VHV4zQC/etVOH/nfJZd9UNl+utF9P19a5/K+fQj1o9410V5Z1Syw7xQaoWMKdmnTDd9NpvteP+9Tr9fz97x0OjXLri3uu066F3lfPhIf20+4g+vnedGnRtqvpdmgSxYuDsghruJ06c0NNPP61BgwYpISFBCQkJGjRokJ555hkdP348mKXhHP40/w7tfPtLZW7OKrfuq0++VeKd1yniktoyDEOJd3ZQWHio9m75JgiVAuZ1mddf3//zax3d/K3PeEyHxqoRVlNHNx30jhVknVDhv/N0aUJcZZcJixgW/VMVBW1afteuXerTp49q166tXr166corr5T0n8v9X3jhBc2cOVMbN25Ux44dg1UifiXxzg664ro4je309FnXp93xiia+fq/+9tMzOlNSKvep05o+6C86epAvaqj6mt3ZTvWua6y3Oy8st65WbKRK3WdUkl/sM16cW6hasZGVVSIsZuep66CF+5gxY/S73/1OCxculGH4fvPxeDwaPXq0xowZc8HH7bnd7nKP+StVqWqohuU1O1n9JnX1x+d/qz/3flEl7jNn3eb30/srom5tTer5ggpOFCph4NWa9LdRevSmOfpu35FKrhjwX+0mUeo0t5/ev2Wpys7x7zdQnQQt3Pfs2aOlS5eWC3ZJMgxD48eP17XXXnvB/aSlpWnatGk+Y1eoo1qqs2W1QmrZIV6XNIzSi7snesdq1KyhtolXaMCD3XRfqyd025juGv2bJ5X9r6OSpEN7f9Bvbmqh/imJmvfA6mCVDlxQTIfLVKthhPpnPOAdC6lZQw0Tm6p1ShdtSlquGq6aCo0O9+newxtG6Oeck8EoGRaomhPq1ghauMfGxmrnzp1q3br1Wdfv3LlTDRs2vOB+zvbYv99FP2pJjfj/Mjdn6YG2T/qMjV/ye32/P1drZr2n8NphkiRPWZnPNmWlZTJC7PyfEOzg6OaD+ke7F33GbvjrIOXvP6Evn/5IRYfzVXr6jBr1vFzZa/8lSYq6sr4imtbV8e2Hg1EyLBBi4/9vClq4P/zww7r//vuVkZGhnj17eoM8NzdXmzdv1uLFi/Xss89ecD9ne+wfU/LW+7nQrX9/edRnrLjIrYIfC/XvL4+qRs0Q/fDNMY1ZNEQvP7xWBT8WKWHg1bq2d2tN7V/+HCZQlZwpPK28L4/5jhWVyP3TKe/4gb/uVsfn+sr9088qKXCr8wu36tgn2TrxKbd5Vld2bjyCFu4pKSmqX7++5syZo5deekmlpaWSpBo1aqhDhw5aunSp7rjjjmCVB5NKz5RpSr+XNHLm7Zry1mjVinDpyIHjmj38VX32zpfBLg+4aLvGvyNPmUfd37hLIa6aOrLxgD5NeSvYZQFnZXg8Hk+wiygpKdGJEyckSfXr11doaOhF7a+fkWJFWUCVdZdRL9glAAE1rGx6wI+xwfWEJfvp737ckv1YqUo8oS40NFSNGjUKdhkAAAcxbHwvnI0/GgAAzlQlOncAACpbyFluxbYLwh0A4Eh2vlqeaXkAAGyGzh0A4Eg8xAYAAJux8Sl3puUBALAbOncAgCMxLQ8AgM2c7VdJ7YJwBwA4kp07d865AwBgM3TuAABHsvNDbAh3AIAj2TjbmZYHAMBu6NwBAI7EtDwAADZj51vhmJYHAMBm6NwBAI5k5/vcCXcAgCPZ+Zw70/IAANgMnTsAwJFs3LgT7gAAZ7LztDzhDgBwpBBuhQMAANUFnTsAwJGYlgcAwGZCbDx3beOPBgCAM9G5AwAcyc7PlifcAQCOZOfHzzItDwCAzRDuAABHMgzDksWMBQsWqH379oqKilJUVJQSEhL0zjvveNd379693P5Hjx5t+rMxLQ8AcKRgTMs3adJEM2fOVMuWLeXxeLRs2TLdfvvt+vzzz/Wb3/xGknTffffpiSee8L6ndu3apo9DuAMAUEkGDBjg83rGjBlasGCBduzY4Q332rVrKzY29qKOw7Q8AMCRjBBrlooqLS3V6tWrVVRUpISEBO/4ihUrVL9+fbVt21aTJk3SqVOnTO+bzh0A4EhWPVve7XbL7Xb7jLlcLrlcrrNu/8UXXyghIUHFxcWKiIjQunXr1KZNG0nSkCFD1LRpUzVu3Fh79+7VhAkTlJWVpbVr15qqiXAHADiSVY+fTUtL07Rp03zGpkyZoqlTp551+1atWikzM1P5+fl64403NHz4cKWnp6tNmza6//77vdu1a9dOjRo1Us+ePXXw4EG1aNHC75oMj8fjudBGe/fu9XuH7du393vbQOlnpAS7BCCg7jLqBbsEIKCGlU0P+DG+vvYlS/bTdMcoU537r/Xq1UstWrTQokWLyq0rKipSRESE3n33XfXp08fvmvzq3K+55hoZhqFzfQ/4ZZ1hGCotLfX74AAABItV0/JmgvxsysrKyn05+EVmZqYkqVGjRqb26Ve4Hzp0yNROAQCo6oLxq3CTJk1S3759FR8fr5MnT2rlypXasmWLNm7cqIMHD2rlypXq16+fYmJitHfvXo0fP16JiYmmZ8X9CvemTZtW6EMAAID/79ixYxo2bJiOHj2q6OhotW/fXhs3blTv3r11+PBhbdq0SXPnzlVRUZHi4uKUnJysxx57zPRxKnRB3auvvqqFCxfq0KFD2r59u5o2baq5c+eqefPmuv322yuySwAAKtXF3MZWUa+88so518XFxSk9Pd2S45j+aAsWLFBqaqr69eunvLw87zn2unXrau7cuZYUBQBAoBmGNUtVZDrcX3zxRS1evFh//vOfVaNGDe94x44d9cUXX1haHAAAMM/0tPyhQ4d07bXXlht3uVwqKiqypCgAAAItGBfUVRbTnXvz5s29l+b/t3fffVdXXXWVFTUBABBwwX78bCCZ7txTU1OVkpKi4uJieTwe7dy5U6tWrVJaWppefvnlQNQIAIDlqur5ciuYDvc//OEPqlWrlh577DGdOnVKQ4YMUePGjfX888/rrrvuCkSNAADAhArdCjd06FANHTpUp06dUmFhoRo0aGB1XQAABJaNz7lX+Idjjh07pqysLEn/efzspZdeallRAAAEmp2n5U1fCnDy5En9/ve/V+PGjdWtWzd169ZNjRs31j333KP8/PxA1AgAAEwwHe5/+MMf9Omnn+rtt99WXl6e8vLytGHDBn322Wf64x//GIgaAQCwHFfL/5cNGzZo48aNuvHGG71jffr00eLFi5WUlGRpcQAABAr3uf+XmJgYRUdHlxuPjo7WJZdcYklRAACg4kyH+2OPPabU1FTl5OR4x3JycvTII49o8uTJlhYHAECg2PnZ8n5Ny1977bUy/usTfPPNN4qPj1d8fLwkKTs7Wy6XS8ePH+e8OwCgeqii58ut4Fe4Dxw4MMBlAAAAq/gV7lOmTAl0HQAAVCo7X1BX4YfYAABQnVXV8+VWMB3upaWlmjNnjv72t78pOztbp0+f9ln/008/WVYcAACBUlXvUbeC6Y82bdo0zZ49W3feeafy8/OVmpqqwYMHKyQkRFOnTg1AiQAAwAzT4b5ixQotXrxYDz30kGrWrKm7775bL7/8sh5//HHt2LEjEDUCAGA5wzAsWaoi0+Gek5Ojdu3aSZIiIiK8z5Pv37+/3n77bWurAwAgQOz8+FnTZTVp0kRHjx6VJLVo0ULvvfeeJGnXrl1yuVzWVgcAAEwzHe6DBg3S5s2bJUljxozR5MmT1bJlSw0bNkz33nuv5QUCABAQIRYtVZDpq+Vnzpzp/fvOO+9UfHy8tm/frpYtW2rAgAGWFgcAQKBU0dPllrjo+9wTEhKUkJBgRS0AAMACfoX7P/7xD793eNttt1W4GAAAKovjn1Dn77PlDcNQaWnpxdQDAEClqKpXulvBr3AvKysLdB0AAMAiPFseAOBIXFAHAIDdOP2cOwAAdmPnc+42/mgAADgTnTsAwJEcf869oKDA7x1GRUVVuBgAACqL4+9zr1u3rt8/a8d97gAABJdf4f7hhx96//7uu+80ceJEjRgxwvvY2e3bt2vZsmVKS0sLTJUAAFjM8dPy3bp18/79xBNPaPbs2br77ru9Y7fddpvatWunv/zlLxo+fLj1VQIAYDGulv8v27dvV8eOHcuNd+zYUTt37rSkKAAAUHGmwz0uLk6LFy8uN/7yyy8rLi7OkqIAAAi4EMOapQoyfSvcnDlzlJycrHfeeUddunSRJO3cuVPffPON3nzzTcsLBAAgEOx8zt10596vXz99/fXXGjBggH766Sf99NNPGjBggL7++mv169cvEDUCAAATKnQ5QVxcnJ566imtXbtWa9eu1YwZM5iSBwBUK0aINYsZCxYsUPv27RUVFaWoqCglJCTonXfe8a4vLi5WSkqKYmJiFBERoeTkZOXm5pr+bBUK948++kj33HOPbrjhBv3www+SpFdffVXbtm2ryO4AAKh0RohhyWJGkyZNNHPmTGVkZOizzz5Tjx49dPvtt+vLL7+UJI0fP15vvfWW1qxZo/T0dB05ckSDBw82/dlMh/ubb76pPn36qFatWtq9e7fcbrckKT8/X0899ZTpAgAACAbDsGYxY8CAAerXr59atmypK6+8UjNmzFBERIR27Nih/Px8vfLKK5o9e7Z69OihDh06aMmSJfrkk0+0Y8cOU8cxHe5PPvmkFi5cqMWLFys0NNQ73rVrV+3evdvs7gAAqNbcbrcKCgp8ll8a3/MpLS3V6tWrVVRUpISEBGVkZKikpES9evXybtO6dWvFx8dr+/btpmoyfbV8VlaWEhMTy41HR0crLy/P7O4C4u2cPwe7BCCgjGh+wwG4aBY9xCYtLU3Tpk3zGZsyZYqmTp161u2/+OILJSQkqLi4WBEREVq3bp3atGmjzMxMhYWFqW7duj7bN2zYUDk5OaZqMh3usbGxOnDggJo1a+Yzvm3bNl1++eVmdwcAQHBYdI/6pEmTlJqa6jPmcrnOuX2rVq2UmZmp/Px8vfHGGxo+fLjS09MtqeUXpsP9vvvu09ixY/XXv/5VhmHoyJEj2r59ux5++GFNnjzZ0uIAAKjqXC7XecP818LCwnTFFVdIkjp06KBdu3bp+eef15133qnTp08rLy/Pp3vPzc1VbGysqZpMh/vEiRNVVlamnj176tSpU0pMTJTL5dLDDz+sMWPGmN0dAADBUUWeYlNWVia3260OHTooNDRUmzdvVnJysqT/nArPzs72/lCbv0yHu2EY+vOf/6xHHnlEBw4cUGFhodq0aaOIiAizuwIAIHiC8MMxkyZNUt++fRUfH6+TJ09q5cqV2rJlizZu3Kjo6GiNGjVKqampqlevnqKiojRmzBglJCTo+uuvN3Uc0x/t3nvv1cmTJxUWFqY2bdqoc+fOioiIUFFRke69916zuwMAwDGOHTumYcOGqVWrVurZs6d27dqljRs3qnfv3pL+84j3/v37Kzk5WYmJiYqNjdXatWtNH8fweDweM2+oUaOGjh49qgYNGviMnzhxQrGxsTpz5ozpIqzmyT0S7BKAgOJqedheeOBng4vHr7BkP+FzhlqyHyv5PS1fUFAgj8cjj8ejkydPKjw83LuutLRU//znP8sFPgAAVVUVOeUeEH6He926dWUYhgzD0JVXXlluvWEY5e7zAwAAlc/vcP/www/l8XjUo0cPvfnmm6pXr553XVhYmJo2barGjRsHpEgAACxXRX+L3Qp+h3u3bt0kSYcOHVJ8fLwMO89nAADsz8bhbvpq+Q8++EBvvPFGufE1a9Zo2bJllhQFAEDAhVi0VEGmy0pLS1P9+vXLjTdo0IBfhQMAoAow/RCb7OxsNW/evNx406ZNlZ2dbUlRAAAEnI1PL5vu3Bs0aKC9e/eWG9+zZ49iYmIsKQoAgIALMaxZqiDT4X733Xfrf/7nf/Thhx+qtLRUpaWl+uCDDzR27FjdddddgagRAACYYHpafvr06fruu+/Us2dP1az5n7eXlZVp2LBhnHMHAFQfVfRiOCuYDvewsDC9/vrrmj59uvbs2aNatWqpXbt2atq0aSDqAwAgMKrolLoVTIf7L6688sqzPqkOAAAEl1/hnpqaqunTp6tOnTpKTU0977azZ8+2pDAAAALKvo27f+H++eefq6SkxPv3ufDUOgBAteH0afkPP/zwrH8DAICqp8Ln3AEAqNac3rkPHjzY7x2uXbu2wsUAAFBZ7Hwm2a9wj46O9v7t8Xi0bt06RUdHq2PHjpKkjIwM5eXlmfoSAABAUDm9c1+yZIn37wkTJuiOO+7QwoULVaNGDUlSaWmp/vSnPykqKiowVQIAAL8ZHo/HY+YNl156qbZt26ZWrVr5jGdlZemGG27Qjz/+aGmBFeHJPRLsEoCAMqL5Ig2bC48I+CFOz7LmNHLYhKo3a2364XtnzpzR/v37y43v379fZWVllhQFAEDA2fj33E1fLT9y5EiNGjVKBw8eVOfOnSVJn376qWbOnKmRI0daXiAAADDHdLg/++yzio2N1XPPPaejR49Kkho1aqRHHnlEDz30kOUFAgAQEDa+XN70Off/VlBQIElV7kI6zrnD7jjnDturjHPus9dZsp+w1EGW7MdKFTpbcObMGW3atEmrVq3yPnL2yJEjKiwstLQ4AABgnulp+X//+99KSkpSdna23G63evfurcjISM2aNUtut1sLFy4MRJ0AAFjLxve5m+7cx44dq44dO+r//u//VKtWLe/4oEGDtHnzZkuLAwAgYEIMa5YqyHTn/tFHH+mTTz5RWFiYz3izZs30ww8/WFYYAACoGNPhXlZWptLS0nLj33//vSIjIy0pCgCAgKuaTbclTE/L33LLLZo7d673tWEYKiws1JQpU9SvXz8rawMAIHCYlv//nn32WSUlJalNmzYqLi7WkCFD9M0336h+/fpatWpVIGoEAMB6VTSYrWA63OPi4rRnzx69/vrr2rNnjwoLCzVq1CgNHTrU5wI7AAAQHKbCvaSkRK1bt9aGDRs0dOhQDR06NFB1AQAQUDZ+QJ25cA8NDVVxcXGgagEAoPLYeFre9AV1KSkpmjVrls6cOROIegAAwEUyfc59165d2rx5s9577z21a9dOderU8Vm/dq01v48LAEBA2bhzNx3udevWVXJyciBqAQCg8lTR32K3gulwX7JkSSDqAAAAFvH7e0tZWZlmzZqlrl27qlOnTpo4caJ+/vnnQNYGAEDgGIY1SxXkd7jPmDFD//u//6uIiAhddtllev7555WSkhLI2gAACJwQi5YqyO+yli9frpdeekkbN27U+vXr9dZbb2nFihUqKysLZH0AANhGWlqaOnXqpMjISDVo0EADBw5UVlaWzzbdu3eXYRg+y+jRo00dx+9wz87O9nl2fK9evWQYho4cOWLqgAAAVAlBmJZPT09XSkqKduzYoffff18lJSW65ZZbVFRU5LPdfffdp6NHj3qXp59+2tRx/L6g7syZMwoPD/cZCw0NVUlJiakDAgBQJQThdPm7777r83rp0qVq0KCBMjIylJiY6B2vXbu2YmNjK3wcv8Pd4/FoxIgRcrlc3rHi4mKNHj3a51537nMHAFQLFoW72+2W2+32GXO5XD55eS75+fmSpHr16vmMr1ixQq+99ppiY2M1YMAATZ48WbVr1/a7Jr/Dffjw4eXG7rnnHr8PBACAHaWlpWnatGk+Y1OmTNHUqVPP+76ysjKNGzdOXbt2Vdu2bb3jQ4YMUdOmTdW4cWPt3btXEyZMUFZWlqnm2fB4PB5Tn6Ia8ORyHQDszYiOCnYJQGCFRwT8ECUr373wRn4oS765Qp37Aw88oHfeeUfbtm1TkyZNzrndBx98oJ49e+rAgQNq0aKFXzWZfogNAAC2YNG0vL9T8P/twQcf1IYNG7R169bzBrskdenSRZIIdwAAqiKPx6MxY8Zo3bp12rJli5o3b37B92RmZkqSGjVq5PdxCHcAgDMF4Wr5lJQUrVy5Un//+98VGRmpnJwcSVJ0dLRq1aqlgwcPauXKlerXr59iYmK0d+9ejR8/XomJiWrfvr3fx+GcO1ANcc4dtlcZ59xf32jJfkLv7OP3tsY57otfsmSJRowYocOHD+uee+7Rvn37VFRUpLi4OA0aNEiPPfaYoqL8/++ezh0AgEpyoX46Li5O6enpF30cwh0A4EhV9DdfLEG4AwCcycbhXkV/zwYAAFQUnTsAwJlC7Nu6E+4AAGeyb7YT7gAAh7JxuHPOHQAAm6FzBwA4k43vhSPcAQDOZN9sZ1oeAAC7oXMHADiTjdtbwh0A4Ew2Pudu4+8tAAA4E507AMCZ7Nu4E+4AAIeycbgzLQ8AgM3QuQMAnMnGF9QR7gAAZ7Lx3DXhDgBwJvs27nb+3gIAgDPRuQMAnIlz7gAA2Ix9s51peQAA7IbOHQDgSDaelSfcAQAOFWLfdGdaHgAAm6FzBwA4k30bd8IdAOBQNj7pzrQ8AAA2Q+cOv+zK3KNXVr+uL7O+1vEff9S8GdPV66YbvesnPjVT69/d6POeGzt30svPPl3ZpQIVsitjt15Zulz7vvpKx4+f0Pw5z6pXj5slSSUlJZo7b4G2btumw9//oIjICN3QpYseGjtGDRtcGuTKUWH2bdwJd/jn5+JitW7RQsn9+mrMY4+fdZubunTWUxMneF+HhYVWVnnARTv1889q1epKJQ+8TQ+mPuKzrri4WP/av18P3P8HtW51pQoKTmrGrGf0wNjxWrvqtSBVjItGuMPpEq/vosTru5x3m7DQUF0aU6+SKgKs1e3Grup2Y9ezrouMjNSSRS/5jE2eNEG/GzpMR44eVeNGjSqjRFjNxrfCEe6wzM7MTN1w2yBFRUbq+uuu1dg/3KtLoqODXRYQEIWFhTIMQ1GRkcEuBSinSl9Qd/jwYd17773n3cbtdqugoMBncbvdlVQhfnFTl86a9b+TtGTOc3p49P3alblH9z8yUaWlpcEuDbCc2+3Ws3Nf0K19+ygiIiLY5aCiDIuWKqhKh/tPP/2kZcuWnXebtLQ0RUdH+yxpL8yrpArxi1t79lCPG7uqVYvL1eumG7Vw1lP6Yv9+7czMDHZpgKVKSko09pGJ8ng8mvbnScEuBxfDMKxZqqCgTsv/4x//OO/6b7/99oL7mDRpklJTU33GwvJ+vKi6cPHiGjfWJdHR+vf3PyihQ4dglwNYoqSkROMemagjR49q2eKFdO2osoIa7gMHDpRhGPJ4POfcxrjAtyKXyyWXy+Uz5vm50JL6UHE5x44rr6BADWJigl0KYIlfgv3f2Ye1/OVFuqRu3WCXhItWNbtuKwQ13Bs1aqSXXnpJt99++1nXZ2ZmqgNdX5VQdOpnZf/wg/f190eP6qtvDig6KlLRkVGav3SZbumWqPr16unwkR/0zIJFir/sMt3YuVMQqwb8V3TqlLKzD3tff//DEX21P0vR0VG6tH59/c/DE/Svr/Zr0YtzVVpWquMnTkiSoqOjFRbKbZ/VUhWdUrdCUMO9Q4cOysjIOGe4X6irR+XZl5Wl4WPHe1/PnPef24IGJvXR1IfGK+vgQa1/d6NOFhbq0vox6tqpo8aOuldhYWHBKhkwZd+X/9KwP/zR+zrt2dmSpEG39deDo/+oD7akS5Juv+Nun/ctf3mRunTqWHmFAn4wPEFMz48++khFRUVKSko66/qioiJ99tln6tatm6n9enKPWFEeUGUZ0VHBLgEIrPDAX89Q+uk2S/ZTo8uNF96okgX1avmbbrrpnMEuSXXq1DEd7AAA+CUIV8unpaWpU6dOioyMVIMGDTRw4EBlZWX5bFNcXKyUlBTFxMQoIiJCycnJys3NNXWcKn0rHAAAdpKenq6UlBTt2LFD77//vkpKSnTLLbeoqKjIu8348eP11ltvac2aNUpPT9eRI0c0ePBgU8cJ6rR8oDAtD7tjWh62VxnT8rs+sWQ/NTrdUOH3Hj9+XA0aNFB6eroSExOVn5+vSy+9VCtXrtRvf/tbSdL+/ft11VVXafv27br++uv92i+dOwDAmarAQ2zy8/MlSfXq/ed3OTIyMlRSUqJevXp5t2ndurXi4+O1fft2v/fLs+UBAM5k0a1wbre73GPPz/YMll8rKyvTuHHj1LVrV7Vt21aSlJOTo7CwMNX91XMUGjZsqJycHL9ronMHAOAinPUx6GlpF3xfSkqK9u3bp9WrV1teE507AMChrOncz/YY9At17Q8++KA2bNigrVu3qkmTJt7x2NhYnT59Wnl5eT7de25urmJjY/2uic4dAOBMFp1zd7lcioqK8lnOFe4ej0cPPvig1q1bpw8++EDNmzf3Wd+hQweFhoZq8+bN3rGsrCxlZ2crISHB749G5w4AQCVJSUnRypUr9fe//12RkZHe8+jR0dGqVauWoqOjNWrUKKWmpqpevXqKiorSmDFjlJCQ4PeV8hK3wgHVErfCwfYq41a4z3dZsp8a1/r/Gxrn+jG0JUuWaMSIEZL+8xCbhx56SKtWrZLb7VafPn300ksvmZqWJ9yBaohwh+1VRrhnfmbJfmpcU/V+W4Bz7gAA2Azn3AEAzsRPvgIAYDM2Dnem5QEAsBnCHQAAm2FaHgDgTDaelifcAQCOdK57zu2AaXkAAGyGzh0A4Ew27twJdwCAM9k43JmWBwDAZujcAQDOZOPOnXAHADiTjcOdaXkAAGyGzh0A4FD27dwJdwCAMzEtDwAAqgs6dwCAM9m4cyfcAQDORLgDAGAz9s12zrkDAGA3dO4AAGdiWh4AAJuxcbgzLQ8AgM3QuQMAHMq+nTvhDgBwJqblAQBAdUHnDgBwJht37oQ7AMCZbBzuTMsDAGAzdO4AAGeicwcAANUFnTsAwJno3AEAQHVB5w4AcCYbd+6EOwDAmeyb7UzLAwBgN3TuAACHsm/rTrgDAJzJxufcmZYHAMBm6NwBAM5k38adzh0A4EyGRf+YsXXrVg0YMECNGzeWYRhav369z/oRI0bIMAyfJSkpyfRnI9wBAKgkRUVFuvrqqzV//vxzbpOUlKSjR496l1WrVpk+DtPyAABnCsIFdX379lXfvn3Pu43L5VJsbOxFHYfOHQDgTIZhyeJ2u1VQUOCzuN3uCpe1ZcsWNWjQQK1atdIDDzygH3/80fQ+CHcAgDMZ1ixpaWmKjo72WdLS0ipUUlJSkpYvX67Nmzdr1qxZSk9PV9++fVVaWmruo3k8Hk+FKqjCPLlHgl0CEFBGdFSwSwACKzwi4IfwHLMmK05Hx5Tr1F0ul1wu13nfZxiG1q1bp4EDB55zm2+//VYtWrTQpk2b1LNnT79ronMHADiUNa27y+VSVFSUz3KhYPfX5Zdfrvr16+vAgQOm3scFdQAAZ6oGT6j7/vvv9eOPP6pRo0am3ke4AwBQSQoLC3268EOHDikzM1P16tVTvXr1NG3aNCUnJys2NlYHDx7Uo48+qiuuuEJ9+vQxdRzOuQPVEOfcYXuVcc79RI4l+zHq+3/b2pYtW3TzzTeXGx8+fLgWLFiggQMH6vPPP1deXp4aN26sW265RdOnT1fDhg3N1US4A9UP4Q7bq5Rwz7VkP0Z9c8FbGbigDgAAm+GcOwDAmarBBXUVRbgDAJzJvtnOtDwAAHZD5w4AcCam5QEAsBvCHQAAe7FvtnPOHQAAu6FzBwA4E+fcAQCwGRuHO9PyAADYDOEOAIDNMC0PAHAkg2l5AABQXdC5AwCcycadO+EOAHAo+4Y70/IAANgMnTsAwJns27gT7gAAh+KcOwAANmPjcOecOwAANkO4AwBgM0zLAwCciWl5AABQXRgej8cT7CJQvbndbqWlpWnSpElyuVzBLgewFP9+ozoi3HHRCgoKFB0drfz8fEVFRQW7HMBS/PuN6ohpeQAAbIZwBwDAZgh3AABshnDHRXO5XJoyZQoXG8GW+Pcb1REX1AEAYDN07gAA2AzhDgCAzRDuAADYDOEOAIDNEO64KPPnz1ezZs0UHh6uLl26aOfOncEuCbDM1q1bNWDAADVu3FiGYWj9+vXBLgnwC+GOCnv99deVmpqqKVOmaPfu3br66qvVp08fHTt2LNilAZYoKirS1Vdfrfnz5we7FMAUboVDhXXp0kWdOnXSvHnzJEllZWWKi4vTmDFjNHHixCBXB1jLMAytW7dOAwcODHYpwAXRuaNCTp8+rYyMDPXq1cs7FhISol69emn79u1BrAwAQLijQk6cOKHS0lI1bNjQZ7xhw4bKyckJUlUAAIlwBwDAdgh3VEj9+vVVo0YN5ebm+ozn5uYqNjY2SFUBACTCHRUUFhamDh06aPPmzd6xsrIybd68WQkJCUGsDABQM9gFoPpKTU3V8OHD1bFjR3Xu3Flz585VUVGRRo4cGezSAEsUFhbqwIED3teHDh1SZmam6tWrp/j4+CBWBpwft8LhosybN0/PPPOMcnJydM011+iFF15Qly5dgl0WYIktW7bo5ptvLjc+fPhwLV26tPILAvxEuAMAYDOccwcAwGYIdwAAbIZwBwDAZgh3AABshnAHAMBmCHcAAGyGcAcAwGYId6CaatasmebOnev39kuXLlXdunUv+riGYWj9+vUXvR8AgUO4AyYYhnHeZerUqcEuEQB4tjxgxtGjR71/v/7663r88ceVlZXlHYuIiPD+7fF4VFpaqpo1+c8MQOWicwdMiI2N9S7R0dEyDMP7ev/+/YqMjNQ777yjDh06yOVyadu2bRoxYoQGDhzos59x48ape/fu3tdlZWVKS0tT8+bNVatWLV199dV64403TNU2e/ZstWvXTnXq1FFcXJz+9Kc/qbCwsNx269evV8uWLRUeHq4+ffro8OHDPuv//ve/67rrrlN4eLguv/xyTZs2TWfOnDFVC4DgItwBi02cOFEzZ87UV199pfbt2/v1nrS0NC1fvlwLFy7Ul19+qfHjx+uee+5Renq638cNCQnRCy+8oC+//FLLli3TBx98oEcffdRnm1OnTmnGjBlavny5Pv74Y+Xl5emuu+7yrv/oo480bNgwjR07Vv/617+0aNEiLV26VDNmzPC7DgDBx3whYLEnnnhCvXv39nt7t9utp556Sps2bVJCQoIk6fLLL9e2bdu0aNEidevWza/9jBs3zvt3s2bN9OSTT2r06NF66aWXvOMlJSWaN2+e95f7li1bpquuuko7d+5U586dNW3aNE2cOFHDhw/31jF9+nQ9+uijmjJlit+fCUBwEe6AxTp27Ghq+wMHDujUqVPlvhCcPn1a1157rd/72bRpk9LS0rR//34VFBTozJkzKi4u1qlTp1S7dm1JUs2aNdWpUyfve1q3bq26devqq6++UufOnbVnzx59/PHHPp16aWlpuf0AqNoId8BiderU8XkdEhKiX/+ycklJiffvX86Lv/3227rssst8tnO5XH4d87vvvlP//v31wAMPaMaMGapXr562bdumUaNG6fTp036HcmFhoaZNm6bBgweXWxceHu7XPgAEH+EOBNill16qffv2+YxlZmYqNDRUktSmTRu5XC5lZ2f7PQX/axkZGSorK9Nzzz2nkJD/XErzt7/9rdx2Z86c0WeffabOnTtLkrKyspSXl6errrpKknTdddcpKytLV1xxRYXqAFA1EO5AgPXo0UPPPPOMli9froSEBL322mvat2+fd8o9MjJSDz/8sMaPH6+ysjLdeOONys/P18cff6yoqCjv+e/zueKKK1RSUqIXX3xRAwYM0Mcff6yFCxeW2y40NFRjxozRCy+8oJo1a+rBBx/U9ddf7w37xx9/XP3791d8fLx++9vfKiQkRHv27NG+ffv05JNPWvs/DICA4Wp5IMD69OmjyZMn69FHH1WnTp108uRJDRs2zGeb6dOna/LkyUpLS9NVV12lpKQkvf3222revLlfx7j66qs1e/ZszZo1S23bttWKFSuUlpZWbrvatWtrwoQJGjJkiLp27aqIiAi9/vrrPrVu2LBB7733njp16qTrr79ec+bMUdOmTS/ufwQAlcrw/PpkIAAAqNbo3AEAsBnCHQAAmyHcAQCwGcIdAACbIdwBALAZwh0AAJsh3AEAsBnCHQAAmyHcAQCwGcIdAACbIdwBALAZwh0AAJv5f8KmpJDiOr2bAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:\t0.521739\n",
      "F1-score:\t0.469780\n",
      "Precision:\t0.494949\n",
      "Recall:\t\t0.496337\n",
      "\n",
      "Classification performance:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      0.76      0.64        63\n",
      "           1       0.44      0.23      0.30        52\n",
      "\n",
      "    accuracy                           0.52       115\n",
      "   macro avg       0.49      0.50      0.47       115\n",
      "weighted avg       0.50      0.52      0.49       115\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluation for KNN\n",
    "mat_knn = confusion_matrix(y, y_pred_knn)\n",
    "sns.heatmap(mat_knn.T, square=True, annot=True, fmt=\"d\", cmap=\"RdPu\")\n",
    "plt.xlabel(\"True label\")\n",
    "plt.ylabel(\"Predicted label\")\n",
    "plt.title(\"KNN\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Accuracy:\\t%f\" % accuracy_score(y, y_pred_knn))\n",
    "print(\"F1-score:\\t%f\" % f1_score(y, y_pred_knn, average='macro'))\n",
    "print(\"Precision:\\t%f\" % precision_score(y, y_pred_knn, average='macro'))\n",
    "print(\"Recall:\\t\\t%f\" % recall_score(y, y_pred_knn, average='macro'))\n",
    "print(\"\\nClassification performance:\\n%s\" % classification_report(y, y_pred_knn))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94292a39",
   "metadata": {},
   "source": [
    "### SVM (Support Vector Machine) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "63f28344",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for SVM: {'C': 1000, 'gamma': 'scale', 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "#Finding the optimal parameters for SVM\n",
    "svm = SVC(probability=True)\n",
    "param_grid = {\n",
    "    'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
    "    'kernel': ['linear', 'rbf'],\n",
    "    'gamma': ['scale', 'auto']}\n",
    "\n",
    "svm_gscv = GridSearchCV(svm, param_grid, cv=10, scoring='accuracy')\n",
    "svm_gscv.fit(X_scaled, y)\n",
    "\n",
    "print(\"Best parameters for SVM:\", svm_gscv.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bb99d243",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predicted Probabilities for SVM:\n",
      "[0.44670089 0.48080693 0.48439917 0.43319145 0.38236457 0.42632282\n",
      " 0.45264522 0.44309299 0.40493585 0.45544217 0.44703636 0.40069474\n",
      " 0.44441899 0.5606926  0.45136959 0.40945237 0.34715819 0.54900783\n",
      " 0.49108164 0.46440752 0.41184173 0.43897782 0.42510332 0.53478455\n",
      " 0.4400003  0.41830396 0.44172505 0.44312807 0.40822861 0.46387802\n",
      " 0.43822479 0.43802843 0.31902581 0.474781   0.38699548 0.48304727\n",
      " 0.36253236 0.44648027 0.44360682 0.44237877 0.47104668 0.45240733\n",
      " 0.46155075 0.45679917 0.42537058 0.39603884 0.44479248 0.45238036\n",
      " 0.44116608 0.44364709 0.43588796 0.47424993 0.40102081 0.43490208\n",
      " 0.43049624 0.47933146 0.48108833 0.44379442 0.44859367 0.47712825\n",
      " 0.48824125 0.45647297 0.4867557  0.43304209 0.50991853 0.46646741\n",
      " 0.41722016 0.5        0.47616438 0.47131069 0.43717883 0.43758613\n",
      " 0.40528128 0.51338462 0.53942656 0.441766   0.45867496 0.47199944\n",
      " 0.46006419 0.45808106 0.44935959 0.41886749 0.43637057 0.4733532\n",
      " 0.49254061 0.42356916 0.48065118 0.46987864 0.4833095  0.45301503\n",
      " 0.44905174 0.43775079 0.4859683  0.46418489 0.45170379 0.44197836\n",
      " 0.45285932 0.44976625 0.44856896 0.43503388 0.44895156 0.44218167\n",
      " 0.44085527 0.46471684 0.43600176 0.44126333 0.44099247 0.46373793\n",
      " 0.44542877 0.42725206 0.45096092 0.45416036 0.45049986 0.42100527\n",
      " 0.44359308]\n"
     ]
    }
   ],
   "source": [
    "#Best parameters for SVM: {'C': 1000, 'gamma': 'scale', 'kernel': 'linear'}\n",
    "\n",
    "# svm = SVC(kernel='linear', probability=True)\n",
    "svm = SVC(C=1000, gamma='scale', kernel='linear', probability=True)\n",
    "y_prob_svm = np.zeros_like(y, dtype=float)\n",
    "\n",
    "for train_idx, test_idx in cv.split(X, y):\n",
    "    scaler = StandardScaler()\n",
    "    \n",
    "    X_train = scaler.fit_transform(X.iloc[train_idx])\n",
    "    X_test = scaler.transform(X.iloc[test_idx])\n",
    "    \n",
    "    y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n",
    "    \n",
    "    svm.fit(X_train, y_train)\n",
    "    y_prob_svm[test_idx] = svm.predict_proba(X_test)[:, 1]\n",
    "\n",
    "y_pred_svm = (y_prob_svm > 0.5).astype(int)\n",
    "print(\"\\nPredicted Probabilities for SVM:\")\n",
    "print(y_prob_svm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5b30ca49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "901869cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAHHCAYAAACiDxGKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyHUlEQVR4nO3deVxVdf7H8fdF4UKikBuICmJZ6GSm4kI6OpmGWZZLi6mTmU2/ihgVHZOZ3EqjVc3cyhqXGTW10tIWKyq3cMOfljUSloWpYKWAQFwQ7u+PHt3f3LS6F8/lwjmvZ4/zGPiewzmf09Djzed7lmtzOp1OAQAA0wjwdwEAAMBYhDsAACZDuAMAYDKEOwAAJkO4AwBgMoQ7AAAmQ7gDAGAyhDsAACZDuAMAYDKEOwAAJkO4A37w6aef6pZbblFMTIyCg4PVvHlz9evXT88995z27dsnm82mhx9++Fd/Pjs7WzabTSkpKZKk6dOny2azKSAgQEePHj1n+8LCQoWEhMhms+nBBx/02XkBqBkId6Caffzxx4qPj9eBAwf0l7/8RfPnz9c999yjgIAAPfvss+rUqZPi4uK0evXqX93HqlWrJEkjR450G7fb7ef9uddee83YkwBQo9X1dwGA1cyaNUthYWHas2ePwsPD3dadPHlSkjRixAhNmTJFO3fuVPfu3c/Zx+rVqxUXF6dOnTq5jQ8YMECrV6/WpEmT3MZXrVqlG264Qa+++qqxJwOgRqJzB6rZl19+qT/84Q/nBLskNW3aVNJP4S79f4f+3zIzM5WVleXa5r8NHz5c+/fv16FDh1xjubm5+uCDDzR8+HCDzgBATUe4A9UsJiZGmZmZOnjw4K9uExsbq6uvvlpr165VRUWF27qfA/98Yd2rVy+1aNHC7Y+CNWvWKDQ0VDfccINBZwCgpiPcgWo2ceJElZSU6KqrrtLVV1+thx56SO+++67Ky8vdthsxYoTy8vKUnp7uGqusrNSaNWuUkJCg1q1bn7Nvm82mYcOGuV13X7lypYYMGSK73e67kwJQoxDuQDXr16+fMjIydNNNN+nAgQN68sknlZiYqObNm+uNN95wbXf77bcrMDDQrQvfsmWLjh07dt4p+Z8NHz5chw8f1p49e1z/y5Q8YC2EO+AHXbp00WuvvabTp09r9+7dSk1N1ZkzZ3TLLbfo888/lyQ1atRIiYmJWr9+vUpLSyX9NCVft25d3Xbbbb+6744dOyouLk6rVq3SypUrFRkZqT59+lTLeQGoGQh3wI+CgoLUpUsXPfbYY1q0aJHKy8u1bt061/qRI0eqsLBQmzZtUllZmV599VVdd911atKkyW/ud/jw4VqzZo1WrVql22+/XQEB/KcOWAn/xQM1RHx8vCTpxIkTrrGbbrpJ9evX16pVq/T222/r9OnTvzkl/7Phw4frxIkT+uKLL5iSByyI59yBavbhhx/qT3/6k2w2m9v4W2+9JUm6/PLLXWMhISEaPHiw1qxZo5KSEtWrV08333zz7x7jkksu0dy5c/Xjjz+qa9euxp4AgBqPcAeqWXJyskpKSjR48GDFxcWprKxMH3/8sdasWaNWrVpp9OjRbtuPHDlSK1as0ObNmzVixAjVq1fPo+OMHTvWF+UDqAUId6CaPf3001q3bp3eeustvfDCCyorK1N0dLQeeOABPfzww+e83KZPnz5q1qyZTpw44dGUPADYnE6n099FAAAA43BDHQAAJkO4AwBgMoQ7AAAmQ7gDAFCNjh07ppEjR6pRo0YKCQlR+/bttXfvXtd6p9OpqVOnqlmzZgoJCVHfvn2VnZ3t1TEIdwAAqsnp06fVo0cPBQYG6u2339bnn3+uZ555RhdffLFrmyeffFLz5s3T4sWLtWvXLtWrV0+JiYmu11B7grvlAQCoJpMnT9aOHTu0bdu28653Op2KiorShAkTNHHiRElSQUGBIiIitGzZMg0bNsyj49C5AwBwARwOhwoLC90Wh8Nx3m3feOMNxcfH69Zbb1XTpk3VsWNHLVmyxLX+yJEjys3NVd++fV1jYWFh6tatmzIyMjyuyZQvsRlgS/J3CYBPDbRd/PsbAbXY/ZUzfX4Mo7Ki67QmmjFjhtvYtGnTNH369HO2/eqrr7Ro0SKlpKTo73//u/bs2aO//vWvCgoK0qhRo5SbmytJioiIcPu5iIgI1zpPmDLcAQCoLqmpqUpJSXEbs9vt5922srJS8fHxeuyxxyT99BHNBw8e1OLFizVq1CjDamJaHgBgSQEGLXa7XQ0aNHBbfi3cmzVrpnbt2rmNtW3bVjk5OZKkyMhISVJeXp7bNnl5ea51np4bAACWYzPoH2/06NFDWVlZbmNffPGFYmJiJEmxsbGKjIxUenq6a31hYaF27dqlhIQEj4/DtDwAwJL80d2OHz9eV199tR577DHddttt2r17t1544QW98MILkiSbzaZx48Zp5syZatOmjWJjYzVlyhRFRUVp0KBBHh+HcAcAoJp06dJF69evV2pqqh555BHFxsZq7ty5bp/4OGnSJBUXF+vee+9Vfn6+evbsqXfeeUfBwcEeH8eUz7lztzzMjrvlYXbVcbf8IFuyIfvZ4HzOkP0Yic4dAGBJZr7pzMznBgCAJdG5AwAsybv73GsXwh0AYEkBJo53puUBADAZOncAgCWZt28n3AEAFsW0PAAAqDXo3AEAlmTevp1wBwBYlJmnrgl3AIAlefuJbrWJmf9wAQDAkujcAQCWZObulnAHAFgSj8IBAIBag84dAGBJ5u3bCXcAgEUF2Mwb70zLAwBgMnTuAABLMm/fTrgDACzKzFPXZj43AAAsic4dAGBJZn79LOEOALAkM09dE+4AAEsyc+du5j9cAACwJDp3AIAlmbm7JdwBAJZk3kl5c//hAgCAJdG5AwAsycwf+Uq4AwAsybzRzrQ8AACmQ+cOALAkpuUBADAZM09dm/ncAACwJDp3AIAlmXdSnnAHAFgU19wBADAZ80Y719wBADAdOncAgCUxLQ8AgMmYN9qZlgcAwHTo3AEAlmTm7pZwBwBYEtPyAACg1qBzBwBYEnfLAwBgMmaeujbzuQEAYEl07gAASzLvpDzhDgCwKDNPXRPuAABLMvMNdWb+wwUAAEuicwcAWJJ5+3bCHQBgUWaeujbzuQEAYEmEOwDAkgIMWrwxffp02Ww2tyUuLs61vrS0VElJSWrUqJFCQ0M1dOhQ5eXlVencAACwHJtBi7f+8Ic/6MSJE65l+/btrnXjx4/Xxo0btW7dOm3ZskXHjx/XkCFDvD4G19wBAKhGdevWVWRk5DnjBQUFeumll7Rq1Sr16dNHkrR06VK1bdtWO3fuVPfu3T0+Bp07AMCSAmQzZHE4HCosLHRbHA7Hrx43OztbUVFRat26tUaMGKGcnBxJUmZmpsrLy9W3b1/XtnFxcYqOjlZGRoaX5wYAgAUZNS2flpamsLAwtyUtLe28x+zWrZuWLVumd955R4sWLdKRI0f0xz/+UWfOnFFubq6CgoIUHh7u9jMRERHKzc316tyYlgcA4AKkpqYqJSXFbcxut5932+uvv9719ZVXXqlu3bopJiZGa9euVUhIiGE1Ee4AAEsyaurabrf/apj/nvDwcF122WU6fPiw+vXrp7KyMuXn57t173l5eee9Rv9bmJYHAFiSPx6F+6WioiJ9+eWXatasmTp37qzAwEClp6e71mdlZSknJ0cJCQle7ZfOHQBgSTY/vIB24sSJGjhwoGJiYnT8+HFNmzZNderU0R133KGwsDCNGTNGKSkpatiwoRo0aKDk5GQlJCR4dae8RLgDAFBtvv32W91xxx364Ycf1KRJE/Xs2VM7d+5UkyZNJElz5sxRQECAhg4dKofDocTERC1cuNDr4xDuAABL8sd16Zdffvk31wcHB2vBggVasGDBBR2HcAcAWJKZbzoz87kBAGBJdO4AAEvi89wBADAZM09dm/ncAACwJDp3AIAl+eM59+pCuAMALMnMU9dmPjcAACyJzh0AYElm7m4JdwCAJZn3ijvhDgCwqAATx7uZZyUAALAkOncAgCWZubsl3AEAlmTeSXlz/+ECAIAl0bnDY42iwjT6iUGKv76d7BcF6cTh7zRn9L+VnZnj2mbkjBvU/y89VC88RJ/v+EoL7n9Zxw9/58eqAe91fKiXuqddp0+e/Vg7xr8lSWrQuqESnuqvZj1jVMdeRznvZGv7Xzfpx5PFfq4WVWXm7tbM5wYDhYaH6OkdE1RRXqGp1y/Ufe1masmE13TmdIlrm1sm9dNNf/2T5t/3ssZ3e0qlxWV6dPODCrTzNyRqjybxzdXu3i76/sAJ11jdiwJ14+a7JKdTb1z7T63vuUR1guro+jf+LNnMPLlrbgEGLTVRTa0LNcwtD12n746e1py7/60v9nyjvK9/0P++d0i5X33v2mbQuGv08sx3tPONT/T1p8f1zJ3L1SgqTAmDOvixcsBzdesFqe+/b9VH926Q43SpazyyR4zqtwrXB6Nf06mDeTp1ME8f3PWqmsZHqXmf1n6sGDg/v4b7999/ryeffFKDBw9WQkKCEhISNHjwYD311FP67jumcmuS7je1V/beHKWuHaNVeY/ruX2TlXjP1a71kbGN1LBZmPa/n+UaKyksVdaur9U2IdYfJQNe6zV/oL55K0vH0r90G69jryM5napwnHWNnS09K2elU816xlR3mTCIzaB/aiK/hfuePXt02WWXad68eQoLC1OvXr3Uq1cvhYWFad68eYqLi9PevXv9VR5+IbJ1Y91w/x91PPs7PZw4X28u2qb75t2qa+/sJkm6OLKBJOl0XqHbz+XnnXGtA2qyS29vr8admmlX6nvnrMvbeVTlxeVKeCJRdUMCVfeiQF39dH8F1K2ji5rV90O1MIKZp+X9djE0OTlZt956qxYvXizbL65ZOZ1O3XfffUpOTlZGRsZv7sfhcMjhcLiNVahCdVTH8JqtzBZgU/beHC3/xxuSpK/2f6uYK6I04L6eSl+xy8/VARemXosw9Zh7gzZet9StO/9Z6fcleve2l9Vr4U1qn9xdzkqnsld/qu8yj0mVlX6oGPhtfgv3AwcOaNmyZecEuyTZbDaNHz9eHTt2/N39pKWlacaMGW5jlypebdTVsFohnT5RqKOfn3AbO/qfXPUYetVP63N/6tgvjmjg+lqSwiPq66v931ZbnUBVNOkcpYsiQnVr5gOusYC6dRTVK0ZXJHXTC8HT9e17h7WqzWwFN7pIlWcrVVZQqlHHH1LhmtN+rBwXomZOqBvDb+EeGRmp3bt3Ky4u7rzrd+/erYiIiN/dT2pqqlJSUtzGbg2bZEiN+H+f7/hSzS93//+j+WVNdfKbU5Kk3CM/6NSJAnW49nJ9deCnMA+pH6zLu7XSm4u2VXu9gDeOpX+pNe3nuY1d888hOn3oe+1/cquclU7XeOkPPz0h0vya1gppWk9fv3GoWmuFcQICzBvvfgv3iRMn6t5771VmZqauvfZaV5Dn5eUpPT1dS5Ys0dNPP/27+7Hb7bLb7W5jTMkbb/2cD/TMxxN1W2qitq3dp8u7xuj6e3to3r2rXdtsmPuhhj3cX8ezTyrvyA/686M36ofjBcrYcMCPlQO/r7yoTKc+O+k+Vlwux6kS1/jld3VS/n9O6sfvShSR0FI9596gA3M/Vv4X359vl6gFbIS78ZKSktS4cWPNmTNHCxcuVEVFhSSpTp066ty5s5YtW6bbbrvNX+XhF7L35mjm4Bd0V9pNGj71euUe+UHPj3tFH63a49rmlSffU3C9ICW/MFyh4SH6bPuXmtp/gcrPcw0TqG3CL2+s7o/1k71hiM58na/Mxz7SJ3M+9ndZwHnZnE6n8/c3863y8nJ9//1Pf/02btxYgYGBF7S/AbYkI8oCaqyBtov9XQLgU/dXzvT5MTbZHzFkPzc6phqyHyPViFeHBQYGqlmzZv4uAwBgIbaa+hybAUx8agAAWFON6NwBAKhuASb+XADCHQBgSWa+W55peQAATIbOHQBgSbzEBgAAkzHxJXem5QEAMBs6dwCAJTEtDwCAyZzvU0nNgnAHAFiSmTt3rrkDAGAydO4AAEsy80tsCHcAgCWZONuZlgcAwGzo3AEAlsS0PAAAJmPmR+GYlgcAwGTo3AEAlmTm59wJdwCAJZn5mjvT8gAAmAydOwDAkkzcuBPuAABrMvO0POEOALCkAB6FAwAAtQWdOwDAkpiWBwDAZAJMPHdt4lMDAMCaCHcAgCXZbDZDlgvx+OOPy2azady4ca6x0tJSJSUlqVGjRgoNDdXQoUOVl5fn1X4JdwCAJQUE2AxZqmrPnj16/vnndeWVV7qNjx8/Xhs3btS6deu0ZcsWHT9+XEOGDPHu3KpcFQAAqJKioiKNGDFCS5Ys0cUXX+waLygo0EsvvaTZs2erT58+6ty5s5YuXaqPP/5YO3fu9Hj/hDsAwJKMmpZ3OBwqLCx0WxwOx28eOykpSTfccIP69u3rNp6Zmany8nK38bi4OEVHRysjI8PjcyPcAQCWZNS0fFpamsLCwtyWtLS0Xz3uyy+/rH379p13m9zcXAUFBSk8PNxtPCIiQrm5uR6fG4/CAQBwAVJTU5WSkuI2Zrfbz7vt0aNHNXbsWL333nsKDg72WU2EOwDAkmwGzV3b7fZfDfNfyszM1MmTJ9WpUyfXWEVFhbZu3ar58+dr8+bNKisrU35+vlv3npeXp8jISI9rItwBAJbkj3fLX3vttfr000/dxkaPHq24uDg99NBDatmypQIDA5Wenq6hQ4dKkrKyspSTk6OEhASPj0O4AwAsyR+vn61fv76uuOIKt7F69eqpUaNGrvExY8YoJSVFDRs2VIMGDZScnKyEhAR1797d4+N4FO6ffPKJxzv85fN6AADAc3PmzFFAQICGDh0qh8OhxMRELVy40Kt92JxOp/P3NgoICJDNZtOvbfrzOpvNpoqKCq8K8IUBtiR/lwD41EDbxb+/EVCL3V850+fHONxpkSH7uXTf/Ybsx0gede5HjhzxdR0AAFQry38qXExMjK/rAAAABqnSgwD/+te/1KNHD0VFRembb76RJM2dO1evv/66ocUBAOArtgBjlprI67IWLVqklJQUDRgwQPn5+a5r7OHh4Zo7d67R9QEA4BM2mzFLTeR1uD/33HNasmSJ/vGPf6hOnTqu8fj4+HOe3QMAANXP6+fcjxw5oo4dO54zbrfbVVxcbEhRAAD4mplvqPO6c4+NjdX+/fvPGX/nnXfUtm1bI2oCAMDnzHzN3evOPSUlRUlJSSotLZXT6dTu3bu1evVqpaWl6cUXX/RFjQAAGK6mXi83gtfhfs899ygkJEQPP/ywSkpKNHz4cEVFRenZZ5/VsGHDfFEjAADwQpXeLT9ixAiNGDFCJSUlKioqUtOmTY2uCwAA3zLxNfcqf3DMyZMnlZWVJemn1882adLEsKIAAPA1M0/Le30rwJkzZ/TnP/9ZUVFR6t27t3r37q2oqCiNHDlSBQUFvqgRAAB4wetwv+eee7Rr1y69+eabys/PV35+vjZt2qS9e/fqf/7nf3xRIwAAhuNu+f+yadMmbd68WT179nSNJSYmasmSJerfv7+hxQEA4Cs85/5fGjVqpLCwsHPGw8LCdPHFfAwlAAD+5nW4P/zww0pJSVFubq5rLDc3V3/72980ZcoUQ4sDAMBXzPxueY+m5Tt27Cjbf51Bdna2oqOjFR0dLUnKycmR3W7Xd999x3V3AEDtUEOvlxvBo3AfNGiQj8sAAABG8Sjcp02b5us6AACoVma+oa7KL7EBAKA2q6nXy43gdbhXVFRozpw5Wrt2rXJyclRWVua2/tSpU4YVBwCAr9TUZ9SN4PWpzZgxQ7Nnz9btt9+ugoICpaSkaMiQIQoICND06dN9UCIAAPCG1+G+cuVKLVmyRBMmTFDdunV1xx136MUXX9TUqVO1c+dOX9QIAIDhbDabIUtN5HW45+bmqn379pKk0NBQ1/vkb7zxRr355pvGVgcAgI+Y+fWzXpfVokULnThxQpJ0ySWX6N1335Uk7dmzR3a73djqAACA17wO98GDBys9PV2SlJycrClTpqhNmza68847dffddxteIAAAPhFg0FIDeX23/OOPP+76+vbbb1d0dLQyMjLUpk0bDRw40NDiAADwlRp6udwQF/yce0JCghISEoyoBQAAGMCjcH/jjTc83uFNN91U5WIAAKguln9DnafvlrfZbKqoqLiQegAAqBY19U53I3gU7pWVlb6uAwAAGIR3ywMALIkb6gAAMBurX3MHAMBszHzN3cSnBgCANdG5AwAsyfLX3AsLCz3eYYMGDapcDAAA1cXyz7mHh4d7/LF2POcOAIB/eRTuH374oevrr7/+WpMnT9Zdd93leu1sRkaGli9frrS0NN9UCQCAwSw/Ld+7d2/X14888ohmz56tO+64wzV20003qX379nrhhRc0atQo46sEAMBg3C3/XzIyMhQfH3/OeHx8vHbv3m1IUQAAoOq8DveWLVtqyZIl54y/+OKLatmypSFFAQDgcwE2Y5YayOtH4ebMmaOhQ4fq7bffVrdu3SRJu3fvVnZ2tl599VXDCwQAwBfMfM3d6859wIAB+uKLLzRw4ECdOnVKp06d0sCBA/XFF19owIABvqgRAAB4oUovsWnZsqUee+wxo2sBAKDacEPdL2zbtk0jR47U1VdfrWPHjkmS/vWvf2n79u2GFgcAgK/YAmyGLDWR1+H+6quvKjExUSEhIdq3b58cDockqaCggG4eAFBr2GzGLDWR1+E+c+ZMLV68WEuWLFFgYKBrvEePHtq3b5+hxQEAAO95fc09KytLvXr1Omc8LCxM+fn5RtR0wd768Ql/lwAAqOm45v7/IiMjdfjw4XPGt2/frtatWxtSFAAAPmfi59y9Dve//OUvGjt2rHbt2iWbzabjx49r5cqVmjhxou6//35f1AgAALzg9bT85MmTVVlZqWuvvVYlJSXq1auX7Ha7Jk6cqOTkZF/UCACA8Wrq3XAGsDmdTmdVfrCsrEyHDx9WUVGR2rVrp9DQUKNrq7rSIn9XAAC4EMG+z5TSsf82ZD/Bz440ZD9G8npa/u6779aZM2cUFBSkdu3aqWvXrgoNDVVxcbHuvvtuX9QIAIApLFq0SFdeeaUaNGigBg0aKCEhQW+//bZrfWlpqZKSktSoUSOFhoZq6NChysvL8/o4Xof78uXL9eOPP54z/uOPP2rFihVeFwAAgF/44Ya6Fi1a6PHHH1dmZqb27t2rPn366Oabb9Znn30mSRo/frw2btyodevWacuWLTp+/LiGDBni9al5fM29sLBQTqdTTqdTZ86cUXBwsGtdRUWF3nrrLTVt2tTrAgAA8Ad/XHIfOHCg2/ezZs3SokWLtHPnTrVo0UIvvfSSVq1apT59+kiSli5dqrZt22rnzp3q3r27x8fxONzDw8Nls9lks9l02WWXnbPeZrNpxowZHh8YAAArq6io0Lp161RcXKyEhARlZmaqvLxcffv2dW0TFxen6OhoZWRk+CbcP/zwQzmdTvXp00evvvqqGjZs6FoXFBSkmJgYRUVFeXxgAAD8yqBn1B0Oh+tV7D+z2+2y2+3n3f7TTz9VQkKCSktLFRoaqvXr16tdu3bav3+/goKCFB4e7rZ9RESEcnNzvarJ43Dv3bu3JOnIkSOKjo6WzcSPEAAALMCgcE9LSztn5nratGmaPn36ebe//PLLtX//fhUUFOiVV17RqFGjtGXLFkNq+ZnXz7l/8MEHCg0N1a233uo2vm7dOpWUlGjUqFGGFQcAgM8Y9PrZ1NRUpaSkuI39Wtcu/TTbfemll0qSOnfurD179ujZZ5/V7bffrrKyMuXn57t173l5eYqMjPSqJq9PLS0tTY0bNz5nvGnTpnwqHADAcux2u+vRtp+X3wr3X6qsrJTD4VDnzp0VGBio9PR017qsrCzl5OQoISHBq5q87txzcnIUGxt7znhMTIxycnK83R0AAP7hh8vLqampuv766xUdHa0zZ85o1apV+uijj7R582aFhYVpzJgxSklJUcOGDdWgQQMlJycrISHBq5vppCqEe9OmTfXJJ5+oVatWbuMHDhxQo0aNvN0dAAD+4YcPfTl58qTuvPNOnThxQmFhYbryyiu1efNm9evXT5I0Z84cBQQEaOjQoXI4HEpMTNTChQu9Po7X4X7HHXfor3/9q+rXr+/66NctW7Zo7NixGjZsmNcFAABgFS+99NJvrg8ODtaCBQu0YMGCCzqO1+H+6KOP6uuvv9a1116runV/+vHKykrdeeedXHMHANQeJv489yp/cMwXX3yhAwcOKCQkRO3bt1dMTIzRtVUdHxwDALVbNXxwjGPaWkP2Y59xmyH7MZLXnfvPLrvssvO+qQ4AAPiXR+GekpKiRx99VPXq1TvnWb5fmj17tiGFAQDgUyZ+F5tH4f6///u/Ki8vd339a3hrHQCg1vDD3fLVxaNw//DDD8/7NQAAqHmqfM0dAIBazeqduzcfFP/aa69VuRgAAKqLma8kexTuYWFhrq+dTqfWr1+vsLAwxcfHS5IyMzOVn5/v1R8BAAD4ldU796VLl7q+fuihh3Tbbbdp8eLFqlOnjqSfPnD+gQceUIMGDXxTJQAA8JjXL7Fp0qSJtm/frssvv9xtPCsrS1dffbV++OEHQwusEl5iAwC1WzW8xKbsCWMuIwc9VPNmrb1++d7Zs2d16NChc8YPHTqkyspKQ4oCAMDnAgxaaiCv75YfPXq0xowZoy+//FJdu3aVJO3atUuPP/64Ro8ebXiBAADAO16H+9NPP63IyEg988wzOnHihCSpWbNm+tvf/qYJEyYYXiAAAD5h4tvlq/zBMZJUWFgoSTXvRjquuQNA7VYd19xnrzdkP0Epgw3Zj5GqdLXg7Nmzev/997V69WrXK2ePHz+uoiJCFQAAf/N6Wv6bb75R//79lZOTI4fDoX79+ql+/fp64okn5HA4tHjxYl/UCQCAsUz8nLvXnfvYsWMVHx+v06dPKyQkxDU+ePBgpaenG1ocAAA+E2AzZqmBvO7ct23bpo8//lhBQUFu461atdKxY8cMKwwAAFSN1+FeWVmpioqKc8a//fZb1a9f35CiAADwuZrZdBvC62n56667TnPnznV9b7PZVFRUpGnTpmnAgAFG1gYAgO+YeFre60fhjh49qv79+8vpdCo7O1vx8fHKzs5W48aNtXXrVjVt2tRXtXqOR+EAoHarjkfhFm00ZD9B9w80ZD9G8npavmXLljpw4IDWrFmjAwcOqKioSGPGjNGIESPcbrADAAD+4VXnXl5erri4OG3atElt27b1ZV0Xhs4dAGq3aujcyxcb07kH3lfLO/fAwECVlpb6qhYAAKpPDb1ebgSvb6hLSkrSE088obNnz/qiHgAAcIG8vua+Z88epaen691331X79u1Vr149t/WvvWbM5+MCAOBTJu7cvQ738PBwDR061Be1AABQfWroZ7EbwetwX7p0qS/qAAAABvH475bKyko98cQT6tGjh7p06aLJkyfrxx9/9GVtAAD4js1mzFIDeRzus2bN0t///neFhoaqefPmevbZZ5WUlOTL2gAA8J0Ag5YayOOyVqxYoYULF2rz5s3asGGDNm7cqJUrV6qystKX9QEAAC95HO45OTlu747v27evbDabjh8/7pPCAADwKRNPy3t8Q93Zs2cVHBzsNhYYGKjy8nLDiwIAwOdqZi4bwuNwdzqduuuuu2S3211jpaWluu+++9yedec5dwBArUC4S6NGjTpnbOTIkYYWAwAALpzH4c7z7QAAU+ENdQAAmIx5s72mPqEHAACqis4dAGBNJu7cCXcAgDXV0GfUjcC0PAAAJkPnDgCwJBM37oQ7AMCiTBzuTMsDAGAydO4AAGviJTYAAJiMebOdcAcAWJSJw51r7gAAmAydOwDAmkz8LBzhDgCwJvNmO9PyAACYDZ07AMCaTNzeEu4AAGsy8TV3E//dAgBAzZKWlqYuXbqofv36atq0qQYNGqSsrCy3bUpLS5WUlKRGjRopNDRUQ4cOVV5enlfHIdwBANZkM2jxwpYtW5SUlKSdO3fqvffeU3l5ua677joVFxe7thk/frw2btyodevWacuWLTp+/LiGDBni3ak5nU6nd6XVAqVF/q4AAHAhgkN9foizb71vyH7qDuhb5Z/97rvv1LRpU23ZskW9evVSQUGBmjRpolWrVumWW26RJB06dEht27ZVRkaGunfv7tF+6dwBALgADodDhYWFbovD4fDoZwsKCiRJDRs2lCRlZmaqvLxcffv+/x8McXFxio6OVkZGhsc1Ee4AAGuy2QxZ0tLSFBYW5rakpaX97uErKys1btw49ejRQ1dccYUkKTc3V0FBQQoPD3fbNiIiQrm5uR6fGnfLAwCsyaD2NjU1VSkpKW5jdrv9d38uKSlJBw8e1Pbt240p5L8Q7gAAazLoSTi73e5RmP+3Bx98UJs2bdLWrVvVokUL13hkZKTKysqUn5/v1r3n5eUpMjLS4/0zLQ8AQDVxOp168MEHtX79en3wwQeKjY11W9+5c2cFBgYqPT3dNZaVlaWcnBwlJCR4fBw6dwCANfnhJTZJSUlatWqVXn/9ddWvX991HT0sLEwhISEKCwvTmDFjlJKSooYNG6pBgwZKTk5WQkKCx3fKSzwKBwCoiarjUbj3PzBkP3X79vF4W9uv/EGxdOlS3XXXXZJ+eonNhAkTtHr1ajkcDiUmJmrhwoVeTcsT7gCAmsek4V5dmJYHAFiSiV8tT7gDACwqwLzpzt3yAACYDJ07AMCazNu4E+4AAIsy8UV3puUBADAZOndUyaq167R67Ss6dvyEJKnNJa31wP/8Rb179vBzZYAx+B23APM27jznjqr54KOtqlMnQDHR0XI6ndqwcZNeWrZC69esUptLL/F3ecAF43fcz6rhOfeKbR8Zsp86f/yTIfsxEuEOw3T94zX62/ixunXIIH+XAvgEv+PVqDrCfccWQ/ZTp0dvQ/ZjJKblccEqKir0zrvvq+THH9Wxw5X+LgcwHL/jqG1qdLgfPXpU06ZN0z//+c9f3cbhcMjhcLiN2Z3lXn/8HryXlZ2tYX8eLUdZmS66KEQL5jytSy9p7e+yAMPwO25yJr7mXqPvlj916pSWL1/+m9ukpaUpLCzMbUl76plqqtDaYlu10oa1q7X238t1x6236KEp03T4y6/8XRZgGH7HTc5mM2apgfx6zf2NN974zfVfffWVJkyYoIqKil/dhs695rjr3vsV3aKFHpn6D3+XAvgEv+PVqDquuWdsNWQ/dRJ6GbIfI/l1Wn7QoEGy2Wz6rb8vfu3j8X5mt9vPDXJuqPOLyspKlZWX+bsMwGf4HTebmtl1G8Gv0/LNmjXTa6+9psrKyvMu+/bt82d5+A3PPPuc9mTu07fHjisrO1vPPPucdu/N1MAB1/u7NMAQ/I5bgImn5f3auXfu3FmZmZm6+eabz7v+97p6+M8Pp07roYen6uR336t+aKguv6yNXlo0Xz0Suvu7NMAQ/I6jNvPrNfdt27apuLhY/fv3P+/64uJi7d27V717e/kMIdPyAFC7Vcc1913bDdlPnW49DdmPkXiJDQCg5qmOcN+9w5D91Ola815JXKMfhQMAAN6r0S+xAQDAZ2rozXBGINwBANZEuAMAYDImDneuuQMAYDJ07gAAizJv5064AwCsiWl5AABQW9C5AwCsycSdO+EOALAmE4c70/IAAJgMnTsAwJpM3LkT7gAAazJxuDMtDwCAyRDuAACYDNPyAABrMvG0POEOALAkm4nDnWl5AABMhs4dAGBNJu7cCXcAgDWZONyZlgcAwGTo3AEA1mTizp1wBwBYk4nDnWl5AABMhs4dAGBR5u3cCXcAgDUxLQ8AAGoLOncAgDWZuHMn3AEA1kS4AwBgMubNdq65AwBgNnTuAABrYloeAACTMXG4My0PAIDJ0LkDACzKvJ074Q4AsCam5QEAwIXaunWrBg4cqKioKNlsNm3YsMFtvdPp1NSpU9WsWTOFhISob9++ys7O9vo4hDsAwJpsNmMWLxQXF6tDhw5asGDBedc/+eSTmjdvnhYvXqxdu3apXr16SkxMVGlpqXen5nQ6nV79RG1QWuTvCgAAFyI41OeHqPzmK0P2ExDTuko/Z7PZtH79eg0aNEjST117VFSUJkyYoIkTJ0qSCgoKFBERoWXLlmnYsGGe11SligAAgCTJ4XCosLDQbXE4HF7v58iRI8rNzVXfvn1dY2FhYerWrZsyMjK82hfhDgCwJoOm5dPS0hQWFua2pKWleV1Obm6uJCkiIsJtPCIiwrXOU9wtDwDABUhNTVVKSorbmN1u91M1PyHcAQDWZNCjcHa73ZAwj4yMlCTl5eWpWbNmrvG8vDxdddVVXu2LaXkAAGqA2NhYRUZGKj093TVWWFioXbt2KSEhwat90bkDAKzJDy+xKSoq0uHDh13fHzlyRPv371fDhg0VHR2tcePGaebMmWrTpo1iY2M1ZcoURUVFue6o9xThDgCwJj+8oG7v3r265pprXN//fK1+1KhRWrZsmSZNmqTi4mLde++9ys/PV8+ePfXOO+8oODjYq+PwnDsAoOapjufcj31jyH4CmscYsh8j0bkDACzKvO+WJ9wBANbEB8cAAIDags4dAGBN5m3cCXcAgDXZTJzuTMsDAGAydO4AAGsy8Q11hDsAwJoIdwAATMa82c41dwAAzIbOHQBgUeZt3Ql3AIA1mfiaO9PyAACYDJ07AMCazNu4E+4AAKsyb7ozLQ8AgMnQuQMArMnEN9QR7gAAazJvtjMtDwCA2dC5AwCsiWl5AADMhnAHAMBczJvtXHMHAMBs6NwBANbENXcAAEzGxOHOtDwAACZDuAMAYDJMywMALMnGtDwAAKgt6NwBANZk4s6dcAcAWJR5w51peQAATIbOHQBgTeZt3Al3AIBFcc0dAACTMXG4c80dAACTIdwBADAZpuUBANbEtDwAAKgtbE6n0+nvIlC7ORwOpaWlKTU1VXa73d/lAIbi9xu1EeGOC1ZYWKiwsDAVFBSoQYMG/i4HMBS/36iNmJYHAMBkCHcAAEyGcAcAwGQId1wwu92uadOmcbMRTInfb9RG3FAHAIDJ0LkDAGAyhDsAACZDuAMAYDKEOwAAJkO444IsWLBArVq1UnBwsLp166bdu3f7uyTAMFu3btXAgQMVFRUlm82mDRs2+LskwCOEO6pszZo1SklJ0bRp07Rv3z516NBBiYmJOnnypL9LAwxRXFysDh06aMGCBf4uBfAKj8Khyrp166YuXbpo/vz5kqTKykq1bNlSycnJmjx5sp+rA4xls9m0fv16DRo0yN+lAL+Lzh1VUlZWpszMTPXt29c1FhAQoL59+yojI8OPlQEACHdUyffff6+KigpFRES4jUdERCg3N9dPVQEAJMIdAADTIdxRJY0bN1adOnWUl5fnNp6Xl6fIyEg/VQUAkAh3VFFQUJA6d+6s9PR011hlZaXS09OVkJDgx8oAAHX9XQBqr5SUFI0aNUrx8fHq2rWr5s6dq+LiYo0ePdrfpQGGKCoq0uHDh13fHzlyRPv371fDhg0VHR3tx8qA38ajcLgg8+fP11NPPaXc3FxdddVVmjdvnrp16+bvsgBDfPTRR7rmmmvOGR81apSWLVtW/QUBHiLcAQAwGa65AwBgMoQ7AAAmQ7gDAGAyhDsAACZDuAMAYDKEOwAAJkO4AwBgMoQ7UEu1atVKc+fO9Xj7ZcuWKTw8/IKPa7PZtGHDhgveDwDfIdwBL9hstt9cpk+f7u8SAYB3ywPeOHHihOvrNWvWaOrUqcrKynKNhYaGur52Op2qqKhQ3br8ZwagetG5A16IjIx0LWFhYbLZbK7vDx06pPr16+vtt99W586dZbfbtX37dt11110aNGiQ237GjRunP/3pT67vKysrlZaWptjYWIWEhKhDhw565ZVXvKpt9uzZat++verVq6eWLVvqgQceUFFR0TnbbdiwQW3atFFwcLASExN19OhRt/Wvv/66OnXqpODgYLVu3VozZszQ2bNnvaoFgH8R7oDBJk+erMcff1z/+c9/dOWVV3r0M2lpaVqxYoUWL16szz77TOPHj9fIkSO1ZcsWj48bEBCgefPm6bPPPtPy5cv1wQcfaNKkSW7blJSUaNasWVqxYoV27Nih/Px8DRs2zLV+27ZtuvPOOzV27Fh9/vnnev7557Vs2TLNmjXL4zoA+B/zhYDBHnnkEfXr18/j7R0Ohx577DG9//77SkhIkCS1bt1a27dv1/PPP6/evXt7tJ9x48a5vm7VqpVmzpyp++67TwsXLnSNl5eXa/78+a5P7lu+fLnatm2r3bt3q2vXrpoxY4YmT56sUaNGuep49NFHNWnSJE2bNs3jcwLgX4Q7YLD4+Hivtj98+LBKSkrO+YOgrKxMHTt29Hg/77//vtLS0nTo0CEVFhbq7NmzKi0tVUlJiS666CJJUt26ddWlSxfXz8TFxSk8PFz/+c9/1LVrVx04cEA7duxw69QrKirO2Q+Amo1wBwxWr149t+8DAgL0y09WLi8vd33983XxN998U82bN3fbzm63e3TMr7/+WjfeeKPuv/9+zZo1Sw0bNtT27ds1ZswYlZWVeRzKRUVFmjFjhoYMGXLOuuDgYI/2AcD/CHfAx5o0aaKDBw+6je3fv1+BgYGSpHbt2slutysnJ8fjKfhfyszMVGVlpZ555hkFBPx0K83atWvP2e7s2bPau3evunbtKknKyspSfn6+2rZtK0nq1KmTsrKydOmll1apDgA1A+EO+FifPn301FNPacWKFUpISNC///1vHTx40DXlXr9+fU2cOFHjx49XZWWlevbsqYKCAu3YsUMNGjRwXf/+LZdeeqnKy8v13HPPaeDAgdqxY4cWL158znaBgYFKTk7WvHnzVLduXT344IPq3r27K+ynTp2qG2+8UdHR0brlllsUEBCgAwcO6ODBg5o5c6ax/2IA+Ax3ywM+lpiYqClTpmjSpEnq0qWLzpw5ozvvvNNtm0cffVRTpkxRWlqa2rZtq/79++vNN99UbGysR8fo0KGDZs+erSeeeEJXXHGFVq5cqbS0tHO2u+iii/TQQw9p+PDh6tGjh0JDQ7VmzRq3Wjdt2qR3331XXbp0Uffu3TVnzhzFxMRc2L8EANXK5vzlxUAAAFCr0bkDAGAyhDsAACZDuAMAYDKEOwAAJkO4AwBgMoQ7AAAmQ7gDAGAyhDsAACZDuAMAYDKEOwAAJkO4AwBgMoQ7AAAm83/K3atlaQolSwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:\t0.547826\n",
      "F1-score:\t0.400561\n",
      "Precision:\t0.525229\n",
      "Recall:\t\t0.505037\n",
      "\n",
      "Classification performance:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      0.95      0.70        63\n",
      "           1       0.50      0.06      0.10        52\n",
      "\n",
      "    accuracy                           0.55       115\n",
      "   macro avg       0.53      0.51      0.40       115\n",
      "weighted avg       0.53      0.55      0.43       115\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mat_svm = confusion_matrix(y, y_pred_svm)\n",
    "sns.heatmap(mat_svm.T, square=True, annot=True, fmt=\"d\", cmap=\"RdPu\")\n",
    "plt.xlabel(\"True label\")\n",
    "plt.ylabel(\"Predicted label\")\n",
    "plt.title(\"SVM\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Accuracy:\\t%f\" % accuracy_score(y, y_pred_svm))\n",
    "print(\"F1-score:\\t%f\" % f1_score(y, y_pred_svm, average='macro'))\n",
    "print(\"Precision:\\t%f\" % precision_score(y, y_pred_svm, average='macro'))\n",
    "print(\"Recall:\\t\\t%f\" % recall_score(y, y_pred_svm, average='macro'))\n",
    "print(\"\\nClassification performance:\\n%s\" % classification_report(y, y_pred_svm))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dd07c06",
   "metadata": {},
   "source": [
    "### Deep Netural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3df8d19e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2a60c474",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(dropout_rate=0.0, activation='relu'):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(16, input_dim=11, activation=activation))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(8, activation=activation))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1ab9cd0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/1v/gbft9lh97dggng6ytnnwkd9c0000gn/T/ipykernel_49168/4101032449.py:1: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
      "  model = KerasClassifier(build_fn=create_model, verbose=0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-28 10:03:57.939461: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:57.973493: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:57.987197: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:57.990243: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:57.999138: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:58.022424: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:58.031490: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2023-09-28 10:03:58.082825: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x159c0e790> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x1618eeb80> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x145cce790> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x161012700> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x1493ee700> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x15e605280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x138eeeb80> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x163346b80> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:5 out of the last 9 calls to <function Model.make_test_function.<locals>.test_function at 0x163842b80> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x16416d550> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x1514c5280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x162fce4c0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x14ba8e4c0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x1661b2550> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x13d36d550> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x166023550> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "2023-09-28 10:07:48.601476: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found:  {'activation': 'tanh', 'batch_size': 40, 'dropout_rate': 0.1, 'epochs': 10}\n",
      "Best score 0.6484848469495773\n"
     ]
    }
   ],
   "source": [
    "model = KerasClassifier(build_fn=create_model, verbose=0)\n",
    "\n",
    "param_grid = {\n",
    "    'batch_size': [10, 20, 30, 40],\n",
    "    'epochs': [10, 50, 100],\n",
    "    'dropout_rate': [0.0, 0.1, 0.2, 0.3],\n",
    "    'activation': ['relu', 'tanh']\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=10)\n",
    "grid_result = grid.fit(X_scaled, y)\n",
    "\n",
    "print(\"Best parameters found: \", grid_result.best_params_)\n",
    "print('Best score', grid_result.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8461ce14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 39ms/step\n",
      "1/1 [==============================] - 0s 26ms/step\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_predict_function.<locals>.predict_function at 0x2934181f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "1/1 [==============================] - 0s 26ms/step\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x29490cd30> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "1/1 [==============================] - 0s 28ms/step\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "1/1 [==============================] - 0s 27ms/step\n",
      "1/1 [==============================] - 0s 25ms/step\n",
      "DNN Model\n",
      "Predicted Probabilities:\n",
      "[0.42783809 0.6778776  0.78801483 0.10594705 0.34784281 0.20752203\n",
      " 0.25142747 0.76760411 0.59291995 0.22508584 0.2746937  0.35157236\n",
      " 0.20049822 0.81679523 0.61067539 0.14339127 0.17924632 0.63736582\n",
      " 0.95632011 0.70338267 0.48539221 0.36544129 0.13429494 0.87218463\n",
      " 0.17129792 0.50909758 0.07454517 0.17780392 0.2643134  0.30014974\n",
      " 0.58685911 0.32538998 0.07763923 0.62021542 0.07877334 0.60173237\n",
      " 0.53614342 0.31055734 0.47125563 0.55829269 0.59878308 0.25489005\n",
      " 0.54690927 0.67252779 0.25641191 0.16384624 0.21675421 0.2404419\n",
      " 0.54589552 0.07503673 0.19563848 0.59082878 0.11144892 0.14416261\n",
      " 0.12639803 0.52487528 0.56397039 0.47291142 0.39261737 0.40168563\n",
      " 0.50327367 0.81047255 0.76472861 0.41407076 0.48950875 0.52413827\n",
      " 0.28247595 0.50091428 0.6678499  0.72458923 0.48787594 0.27754495\n",
      " 0.21210399 0.6495266  0.65208608 0.30763569 0.42892915 0.72821635\n",
      " 0.27475184 0.72160828 0.70110458 0.32380408 0.69485146 0.40909404\n",
      " 0.85422111 0.1773963  0.53418738 0.83496815 0.76038146 0.61416787\n",
      " 0.31390077 0.47074908 0.7226063  0.65183473 0.60253501 0.40444288\n",
      " 0.69607443 0.38611701 0.44713193 0.55713481 0.61176443 0.51494318\n",
      " 0.14428523 0.40382695 0.37065622 0.31017733 0.44446301 0.53020102\n",
      " 0.51104325 0.65285403 0.58468485 0.22438879 0.37732121 0.18950494\n",
      " 0.42046717]\n"
     ]
    }
   ],
   "source": [
    "#Best parameters found:  {'activation': 'tanh', 'batch_size': 10, 'dropout_rate': 0.3, 'epochs': 100}\n",
    "#Best score 0.6022727280855179\n",
    "\n",
    "# Define the model with the optimal parameters\n",
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(16, input_dim=11, activation='tanh'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(8, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "y_prob_dnn = np.zeros_like(y, dtype=float)\n",
    "\n",
    "for train_idx, test_idx in cv.split(X_scaled, y):\n",
    "    X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]\n",
    "    y_train, y_test = y[train_idx], y[test_idx]\n",
    "    \n",
    "    model = create_model()\n",
    "    model.fit(X_train, y_train, epochs=100, batch_size=10, verbose=0)\n",
    "    y_prob_dnn[test_idx] = model.predict(X_test).flatten()\n",
    "\n",
    "y_pred_dnn = (y_prob_dnn > 0.5).astype(int)\n",
    "\n",
    "print(\"DNN Model\")\n",
    "print(\"Predicted Probabilities:\")\n",
    "print(y_prob_dnn) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "aa89de56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0,\n",
       "       0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1,\n",
       "       0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1,\n",
       "       0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1,\n",
       "       1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1,\n",
       "       1, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_dnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "60e28b1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAHHCAYAAACiDxGKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyKklEQVR4nO3deXgUZbr+8bsDpAOEhBMgIEJYDLIo+yIBBURZjwiCiguyiAsaohDRIXMGwqIGRNlUguJCUFEUjSCKiCAonsgOgkoEBYJCQIYhIYE0kO7fHx77ZxuQ7qQ63an6fq6rriu8Xal6eiYzdz/vW1Vtc7lcLgEAANMICXQBAADAWIQ7AAAmQ7gDAGAyhDsAACZDuAMAYDKEOwAAJkO4AwBgMoQ7AAAmQ7gDAGAyhDsAACZDuAOlbOHChbLZbO4tLCxMtWvXVq9evTR37lydOnXKY/9JkybJZrOpZs2aOn36dJHj1a9fXzfddJPH2B/Hfu655y56/i1bthj7xgAEDcIdCJApU6bojTfeUGpqqhISEiRJY8aMUfPmzfXtt98W2f/YsWNKTU316RwzZsy44AcCAOZGuAMB0qdPHw0ZMkQjRoxQUlKSVq1apc8//1zHjh3TzTffrDNnznjs36pVK82YMaPI+MW0atVKR48e1fz58/1RPoAgRrgDQaR79+6aMGGCDh48qDfffNPjtYkTJ+ro0aNed++dO3dW9+7d9cwzz3j9gQCAORDuQJC55557JEmfffaZx/h1113nc1hPmjTJpw8EAMyBcAeCTJ06dRQZGamffvqpyGvJyck+TbVfd911uv76632azgdQ9hHuQBAKDw8vctW8JHXp0kXXX3+9z917dnY2a++AhRDuQBDKy8tTlSpVLviar2FdnA8EAMo2wh0IMr/88otycnIUGxt7wde7dOmibt26+RTWycnJys7O1ksvvWRkqQCCFOEOBJk33nhDktSrV6+L7vNH9+5tWHft2lXdunXT9OnT6d4BCyDcgSCydu1aTZ06VQ0aNNDdd9990f3+HNYFBQVeHfuPDwQvv/yyUeUCCFLlA10AYFUrV67Unj17dP78eR09elRr167V6tWrVa9ePS1fvlxhYWF/+/vJycm6/vrrvT5f165d1bVrV61fv76kpQMIcoQ7ECATJ06UJIWGhioqKkrNmzfX7NmzNWLEiIteTPdn3bp18zmsJ02a5NMHAgBlk83lcrkCXQQAADAOa+4AAJgM4Q4AgMkQ7gAAmAzhDgCAyRDuAACYDOEOAIDJEO4AAJiMKR9i09cWH+gSAL/65Mz0QJcA+FdYuN9PYVRWfOJ60ZDjGInOHQAAkzFl5w4AwKWYubsl3AEAlmSTLdAl+A3hDgCwJDN37mZ+bwAAWBKdOwDAkpiWBwDAZMw8dW3m9wYAgCXRuQMALMm8k/KEOwDAokJMHO9MywMAYDJ07gAASzJv3064AwAsiml5AABQZtC5AwAsybx9O+EOALAoM09dE+4AAEsy8+NnzfzBBQAAS6JzBwBYkpm7W8IdAGBJ3AoHAADKDDp3AIAlmbdvJ9wBABYVYjNvvDMtDwCAydC5AwAsybx9O+EOALAoM09dm/m9AQBgSXTuAABL4vGzAACYTIhBmy9SU1PVokULRUREKCIiQnFxcVq5cqXHPhkZGerevbsqV66siIgIdenSRWfOnPHpPHTuAABLCkTnXqdOHU2bNk2NGjWSy+VSWlqa+vfvr+3bt+uqq65SRkaGevfuraSkJD3//PMqX768du7cqZAQ3z5G2Fwul8tP7yFg+triA10C4FefnJke6BIA/woL9/spHg4ZZ8hx5jmfLdHvR0VFacaMGRo5cqQ6duyoHj16aOrUqSU6JtPyAABLCsS0/J8VFhbqnXfeUX5+vuLi4nTs2DFt3LhR0dHR6tSpk2rWrKmuXbtqw4YNxXpvAABYjs2gzeFwKDc312NzOBwXPe+uXbsUHh4uu92uUaNGKT09Xc2aNdPPP/8sSZo0aZLuv/9+ffrpp2rTpo1uuOEG7d2716f3RrgDAFACKSkpioyM9NhSUlIuun/jxo21Y8cObdy4UQ899JCGDRum77//Xk6nU5L04IMPasSIEWrdurVmzZqlxo0b67XXXvOpJi6oAwBYklFf+ZqUlKTExESPMbvdftH9Q0NDFRsbK0lq27atNm/erDlz5mj8+PGSpGbNmnns37RpU2VlZflUE+EOALAko66Vt9vtfxvml+J0OuVwOFS/fn3Vrl1bmZmZHq//+OOP6tOnj0/HJNwBACglSUlJ6tOnj2JiYnTq1CktXrxY69at06pVq2Sz2fT4448rOTlZLVu2VKtWrZSWlqY9e/Zo6dKlPp2HcAcAWJJR0/K+OHbsmIYOHaojR44oMjJSLVq00KpVq9SjRw9J0pgxY1RQUKCxY8fqxIkTatmypVavXq0rrrjCp/NwnztQBnGfO0yvFO5zHxfyhCHHedb5jCHHMRJXywMAYDJMywMALMm8XxtDuAMALCoQa+6lhXAHAFiSeaOdNXcAAEyHzh0AYElMywMAYDLmjXam5QEAMB06dwCAJZm5uyXcAQCWxLQ8AAAoM+jcAQCWxNXyAACYjJmnrs383gAAsCQ6dwCAJZl3Up5wBwBYlJmnrgl3AIAlmfmCOjN/cAEAwJLo3AEAlmTevp1wBwBYlJmnrs383gAAsCQ6dwCAJZm5uyXcAQCWZOY1dzN/cAEAwJLo3AEAlmTm+9wJdwCAJZk32pmWBwDAdOjcAQCWZObulnAHAFgS4Q4AgMnYTLzqbuYPLgAAWBKdOwDAkszc3RLuAABLMnO4m/m9AQBgSXTuAABLMu/ldIQ7AMCizDx1beb3BgCAJdG5AwAsycz3uRPuAABLMvPUtZnfGwAAlkTnDgCwJDN3t4Q7AMCSzLviTrgDACwqxMTxbuZZCQAALInOHQBgSWbubgl3AIAlmXdS3twfXAAAsCQ6d3il76jr9N8PXaea9aMkSQe/O6K3p6zUlk+/V3S9KC08MPWCv/f0ba9ow9LtpVkqUCwvvfqaPlvzhX7ef0Bhdrtat2qhcWMeUcP69T32277zW816/kV9u2u3QsqVU9PGV+rV1BcUFhYWmMJRbGbubgl3eOX4L//R6+OX6fDeY7LZbLph2DWasOxBJbSepl/2ZOvuWkke+/d+oLMGPX6jtqz8PkAVA77ZtGWb7h58m5pfdZUKCws18/kXNHJUvD7+YKkqVaoo6fdgv+/h0Xrw3hGaMP4JlStfTnsyf1RIiJljwrzM/N8a4Q6vbFqx2+Pfi/71kf77oevUpGN9ZX1/RP85muvxeqdbWuqrd7epIN9RmmUCxfZq6gse/542ZbLirr9R3/3wg9q3bSNJSpnxnO658w49MHKEe7+/dvZAMAhouB8/flyvvfaaMjIylJ2dLUmqVauWOnXqpOHDh6tGjRqBLA8XERJi07W3tVFY5VD9kLG/yOuxberqitZ1NS9+SQCqA4xxKi9PkhQZESFJ+ve/T2jnrt3q17eP7hg6QlmHflHDBvU1ZvTDatemdSBLRTGZ+YtjAjYrsXnzZl155ZWaO3euIiMj1aVLF3Xp0kWRkZGaO3eumjRpoi1btgSqPFxA/atr6/1TM7XMMUej59+hqbcs0KEfsovs13NkJ2V9f+SCwQ+UBU6nU08/86zatGqpKxvFSpIO/fqrJOmF+S/rtoG36JV5z6tZ0yYa/sBDOnAwK5DlophCDNqCUcA694SEBN12222aP3++bDbPT08ul0ujRo1SQkKCMjIy/vY4DodDDofn1G+hClVO5Qyv2ep+yTyq0a1SVDkyTNfe2lqPpd2jJ7rO9gj40LAK6nZXO7099dMAVgqUzOSnp2nvTz9p8cJX3WNOp1OSNPjWgRo04GZJUrOmTZSxcZPe/3CZHns0ISC1AhcSsA8dO3fu1NixY4sEuyTZbDaNHTtWO3bsuORxUlJSFBkZ6bH9rK1+qBjnzxXqyE+/ad+2Q1r4z+X6eeev6v/o9R77XHtra9krhWrNoo0BqhIomSlPT9e6LzcobcFLqlWzpnu8RvXqkqQrGjb02P+KBg10OLvoDBaCn82gLRgFLNxr1aqlTZs2XfT1TZs2qeaf/od1MUlJScrJyfHYGqqtkaXiIkJCbKpg95z86TkyThuX71Lu8bwAVQUUj8vl0pSnp2v12i+UtmC+6ta53OP1OpfXVnSNGtp/4IDH+IGDWbr8sstKsVIYJSTEZsgWjAI2LT9u3Dg98MAD2rp1q2644QZ3kB89elRr1qzRggUL9Oyzz17yOHa7XXa73WOMKXnjDX/6Zm1Z+b2OZZ1QpSph6nZXOzXv1kgTer3o3ueyK2ro6i6xSu6bGsBKgeKZ/PQ0rVj5qebNnqnKlSvpt+PHJUlVwsMVFhYmm82mkcOH6vnU+WrS+Eo1bdxY6cs/0s8HDmjuc9MDXD2KwxakwWyEgIV7fHy8qlevrlmzZmnevHkqLCyUJJUrV05t27bVwoULdfvttweqPPxFZHQVPbZoqKIui1B+ToH2f/urJvR6Uds/3+Pep+e9cTr+y0lt++yHAFYKFM/b7y6VJN0z8gGP8ZQpyRrY//c19uFD7tJZh0MpM2YqJydHTRpfqdfmv6iYunVLvV7g79hcLpcr0EWcO3dOx//vU3L16tVVoUKFEh2vry3eiLKAoPXJGTpFmFxYuN9PscI+xZDj3OSY6PW+qampSk1N1YH/W9656qqrNHHiRPXp08djP5fLpb59++rTTz9Venq6BgwY4FNNQfEQmwoVKugy1qwAAKXIFoCrzurUqaNp06apUaNGcrlcSktLU//+/bV9+3ZdddVV7v1mz559wQvOvRUU4Q4AgBX069fP499PPfWUUlNT9c0337jDfceOHXruuee0ZcuWYje+hDsAwJJCStAZ/9mFnrdyoYu9/6qwsFDvvfee8vPzFRcXJ0k6ffq07rrrLr344ouqVatWsWsK1ofrAADgV7YQmyHbhZ63kpKSctHz7tq1S+Hh4bLb7Ro1apTS09PVrFkzSdLYsWPVqVMn9e/fv0Tvjc4dAIASSEpKUmJiosfY33XtjRs31o4dO5STk6OlS5dq2LBhWr9+vfbt26e1a9dq+/aSf0024Q4AsCSjHkDjzRT8n4WGhio29vfvLGjbtq02b96sOXPmqGLFivrpp59UtWpVj/0HDRqk6667TuvWrfP6HIQ7AMCSDFpyLzGn0ymHw6HJkyfrvvvu83itefPmmjVrVpEL8S6FcAcAoJQkJSWpT58+iomJ0alTp7R48WKtW7dOq1atUq1atS54EV1MTIwaNGjg03kIdwCAJQXiufDHjh3T0KFDdeTIEUVGRqpFixZatWqVevToYeh5CHcAgCWV5CExxfXqq69eeqc/Ke5DZAl3AIAlBes3uhmB+9wBADAZOncAgCXxla8AAJiMibOdaXkAAMyGzh0AYElMywMAYDKBuBWutDAtDwCAydC5AwAsycz3uRPuAABLMvOaO9PyAACYDJ07AMCSTNy4E+4AAGsy87Q84Q4AsKQQboUDAABlBZ07AMCSmJYHAMBkQkw8d23itwYAgDXRuQMALMnMz5Yn3AEAlmTmx88yLQ8AgMnQuQMALIlpeQAATIZpeQAAUGbQuQMALMlm4vaWcAcAWJKZny1PuAMALMnyj5/99ttvvT5gixYtil0MAAAoOa/CvVWrVrLZbHK5XBd8/Y/XbDabCgsLDS0QAAB/sPy0/P79+/1dBwAApcry0/L16tXzdx0AAMAgxboR4I033lDnzp1Vu3ZtHTx4UJI0e/ZsLVu2zNDiAADwF1uIMVsw8rms1NRUJSYmqm/fvjp58qR7jb1q1aqaPXu20fUBAOAXNpsxWzDyOdyff/55LViwQP/zP/+jcuXKucfbtWunXbt2GVocAADwnc/3ue/fv1+tW7cuMm6325Wfn29IUQAA+JuZL6jzuXNv0KCBduzYUWT8008/VdOmTY2oCQAAvzPzmrvPnXtiYqLi4+NVUFAgl8ulTZs26e2331ZKSopeeeUVf9QIAIDhgnW93Ag+h/t9992nihUr6l//+pdOnz6tu+66S7Vr19acOXN0xx13+KNGAADgg2I9W/7uu+/W3XffrdOnTysvL0/R0dFG1wUAgH+ZeM292F8cc+zYMWVmZkr6/fGzNWrUMKwoAAD8zczT8j5fCnDq1Cndc889ql27trp27aquXbuqdu3aGjJkiHJycvxRIwAA8IHP4X7fffdp48aN+vjjj3Xy5EmdPHlSK1as0JYtW/Tggw/6o0YAAAzH1fJ/smLFCq1atUrXXnute6xXr15asGCBevfubWhxAAD4C/e5/0m1atUUGRlZZDwyMlL/9V//ZUhRAACg+HwO93/9619KTExUdna2eyw7O1uPP/64JkyYYGhxAAD4i5mfLe/VtHzr1q1l+9M72Lt3r2JiYhQTEyNJysrKkt1u12+//ca6OwCgbAjS9XIjeBXuAwYM8HMZAADAKF6Fe3Jysr/rAACgVJn5grpiP8QGAICyLFjXy43gc7gXFhZq1qxZevfdd5WVlaWzZ896vH7ixAnDigMAwF+C9R51I/j81iZPnqyZM2dq8ODBysnJUWJiogYOHKiQkBBNmjTJDyUCAABf+Bzub731lhYsWKDHHntM5cuX15133qlXXnlFEydO1DfffOOPGgEAMJzNZjNkC0Y+h3t2draaN28uSQoPD3c/T/6mm27Sxx9/bGx1AAD4iZkfP+tzWXXq1NGRI0ckSVdccYU+++wzSdLmzZtlt9uNrQ4AAPjM53C/5ZZbtGbNGklSQkKCJkyYoEaNGmno0KG69957DS8QAAC/CDFoC0I+Xy0/bdo098+DBw9WTEyMMjIy1KhRI/Xr18/Q4gAA8JcgXS43RIk/c8TFxSkxMZFgBwDgElJTU9WiRQtFREQoIiJCcXFxWrlypaTfbyVPSEhQ48aNVbFiRcXExOiRRx5xX9vmC6869+XLl3t9wJtvvtnnIgAAKG2BeEJdnTp1NG3aNDVq1Egul0tpaWnq37+/tm/fLpfLpcOHD+vZZ59Vs2bNdPDgQY0aNUqHDx/W0qVLfTqPzeVyuS61U0iIdw2+zWZTYWGhTwX4Q19bfKBLAPzqkzPTA10C4F9h4X4/xYnbXjPkOFHvlex6s6ioKM2YMUMjR44s8tp7772nIUOGKD8/X+XLe7+S7tWeTqfT+yoBALAQh8Mhh8PhMWa32y95B1lhYaHee+895efnKy4u7oL75OTkKCIiwqdgl4L2Oj8AAPzLqO9zT0lJUWRkpMeWkpJy0fPu2rVL4eHhstvtGjVqlNLT09WsWbMi+x0/flxTp07VAw884Pt782ZavqxhWh5mx7Q8TK8UpuX/c8frhhynUtpdPnXuZ8+eVVZWlnJycrR06VK98sorWr9+vUfA5+bmqkePHoqKitLy5ctVoUIFn2riW+EAAJZk1NPlvJmC/7PQ0FDFxsZKktq2bavNmzdrzpw5eumllyRJp06dUu/evVWlShWlp6f7HOwS0/IAAASU0+l0d/65ubnq2bOnQkNDtXz5coWFhRXrmHTuAABLCsRDbJKSktSnTx/FxMTo1KlTWrx4sdatW6dVq1a5g/306dN68803lZubq9zcXElSjRo1VK5cOa/P41W4/3Fwb0RERHi9LwAAgRKI+9yPHTumoUOH6siRI4qMjFSLFi20atUq9ejRQ+vWrdPGjRslyT1t/4f9+/erfv36Xp/Hq3CvWrWq119rFwz3uQMAEIxeffXVi77WrVs3GXWNu1fh/sUXX7h/PnDggMaPH6/hw4e778vLyMhQWlra3176DwBAMDHzs+W9CveuXbu6f54yZYpmzpypO++80z128803q3nz5nr55Zc1bNgw46sEAMBgwfpd7Ebw+a1lZGSoXbt2RcbbtWunTZs2GVIUAAAoPp/DvW7dulqwYEGR8VdeeUV169Y1pCgAAPwuxGbMFoR8vhVu1qxZGjRokFauXKlrrrlGkrRp0ybt3btX77//vuEFAgDgD2Zec/e5c+/bt69+/PFH9evXTydOnNCJEyfUr18//fjjj+rbt68/agQAAD4o1kNs6tatq6efftroWgAAKDVcUPcXX331lYYMGaJOnTrp119/lSS98cYb2rBhg6HFAQDgL7YQmyFbMPI53N9//3316tVLFStW1LZt29zPw83JyaGbBwCUGUZ95Wsw8jncn3zySc2fP18LFizw+Kaazp07a9u2bYYWBwAAfOfzmntmZqa6dOlSZDwyMlInT540oqYS47uuAQCXxJr7/1erVi3t27evyPiGDRvUsGFDQ4oCAMDvTHyfu8/hfv/99+vRRx/Vxo0bZbPZdPjwYb311lsaN26cHnroIX/UCAAAfODztPz48ePldDp1ww036PTp0+rSpYvsdrvGjRunhIQEf9QIAIDxgvVqOAPYXMX8frmzZ89q3759ysvLU7NmzRQeHm50bcVXkBfoCgAAJRHm/0wpePRNQ44TNmeIIccxks/T8vfee69OnTql0NBQNWvWTB06dFB4eLjy8/N17733+qNGAADgA5/DPS0tTWfOnCkyfubMGS1atMiQogAA8DsTX1Dn9Zp7bm6uXC6XXC6XTp06pbCwMPdrhYWF+uSTTxQdHe2XIgEAMJqJl9y9D/eqVavKZrPJZrPpyiuvLPK6zWbT5MmTDS0OAAD4zutw/+KLL+RyudS9e3e9//77ioqKcr8WGhqqevXqqXbt2n4pEgAAwwXplLoRvA73rl27SpL279+vmJgY2cw8nwEAMD8Th7vPF9StXbtWS5cuLTL+3nvvKS0tzZCiAADwuxCDtiDkc1kpKSmqXr16kfHo6Gi+FQ4AgCDg8xPqsrKy1KBBgyLj9erVU1ZWliFFAQDgdyZeXva5c4+Ojta3335bZHznzp2qVq2aIUUBAOB3Jr7P3edwv/POO/XII4/oiy++UGFhoQoLC7V27Vo9+uijuuOOO/xRIwAA8IHP0/JTp07VgQMHdMMNN6h8+d9/3el0aujQoay5AwDKjiC9GM4Ixf7imB9//FE7d+5UxYoV1bx5c9WrV8/o2oqPL44BgLKtFL44xpH8riHHsU++3ZDjGMnnzv0PV1555QWfVAcAAALLq3BPTEzU1KlTVblyZSUmJv7tvjNnzjSkMAAA/Co4r4UzhFfhvn37dp07d87988Xw1DoAQJkRpFe6G8GrcP/iiy8u+DMAAAg+xV5zBwCgTLN65z5w4ECvD/jBBx8UuxgAAEqLmVeSvQr3yMhI988ul0vp6emKjIxUu3btJElbt27VyZMnffoQAABAQFm9c3/99dfdP//jH//Q7bffrvnz56tcuXKSpMLCQj388MOKiIjwT5UAAMBrPj/EpkaNGtqwYYMaN27sMZ6ZmalOnTrp3//+t6EFFgsPsQGAsq0UHmJzdroxy8ih/wi+WWufH753/vx57dmzp8j4nj175HQ6DSkKAAC/M/H3uft8tfyIESM0cuRI/fTTT+rQoYMkaePGjZo2bZpGjBhheIEAAMA3Pof7s88+q1q1aum5557TkSNHJEmXXXaZHn/8cT322GOGFwgAgF+Y+HL5Yn9xjCTl5uZKUvBdSMeaOwCUbaWx5j4z3ZDjhCbeYshxjFSs1YLz58/r888/19tvv+1+5Ozhw4eVl0eoAgAQaD5Pyx88eFC9e/dWVlaWHA6HevTooSpVqmj69OlyOByaP3++P+oEAMBYJr7P3efO/dFHH1W7du30n//8RxUrVnSP33LLLVqzZo2hxQEA4DchNmO2IORz5/7VV1/pf//3fxUaGuoxXr9+ff3666+GFQYAAIrH53B3Op0qLCwsMv7LL7+oSpUqhhQFAIDfBWfTbQifp+V79uyp2bNnu/9ts9mUl5en5ORk9e3b18jaAADwHxNPy/t8K9yhQ4fUu3dvuVwu7d27V+3atdPevXtVvXp1ffnll4qOjvZXrd7jVjgAKNtK41a41I8MOU7oQ/0MOY6RfJ6Wr1u3rnbu3KklS5Zo586dysvL08iRI3X33Xd7XGAHAAACw6fO/dy5c2rSpIlWrFihpk2b+rOukqFzB4CyrRQ693PzjencK4wq4517hQoVVFBQ4K9aAAAoPUG6Xm4Eny+oi4+P1/Tp03X+/Hl/1AMAAErI5zX3zZs3a82aNfrss8/UvHlzVa5c2eP1Dz4w5vtxAQDwKxN37j6He9WqVTVo0CB/1AIAQOkJ0u9iN4LP4f7666/7ow4AAGAQrz+3OJ1OTZ8+XZ07d1b79u01fvx4nTlzxp+1AQDgPzabMVsQ8jrcn3rqKf3zn/9UeHi4Lr/8cs2ZM0fx8fH+rA0AAP8JMWjzQWpqqlq0aKGIiAhFREQoLi5OK1eudL9eUFCg+Ph4VatWTeHh4Ro0aJCOHj1arLfmlUWLFmnevHlatWqVPvzwQ3300Ud666235HQ6fT4pAABWVKdOHU2bNk1bt27Vli1b1L17d/Xv31/fffedJGns2LH66KOP9N5772n9+vU6fPiwBg4c6PN5vH6Ijd1u1759+1S3bl33WFhYmPbt26c6der4fGK/4iE2AFC2lcZDbBatvPROXqgwtE+Jfj8qKkozZszQrbfeqho1amjx4sW69dZbJUl79uxR06ZNlZGRoY4dO3p9TK879/PnzyssLMxjrEKFCjp37pzXJwMAIGjYjNkcDodyc3M9NofDccnTFxYW6p133lF+fr7i4uK0detWnTt3TjfeeKN7nyZNmigmJkYZGRk+vTWvr5Z3uVwaPny47Ha7e6ygoECjRo3yuNed+9wBAGWCQdfCpaSkaPLkyR5jycnJmjRp0gX337Vrl+Li4lRQUKDw8HClp6erWbNm2rFjh0JDQ1W1alWP/WvWrKns7GyfavI63IcNG1ZkbMiQIT6dDAAAs0lKSlJiYqLH2J8b4b9q3LixduzYoZycHC1dulTDhg3T+vXrDa3J63Dn/nYAgKkY9IQ6u93+t2H+V6GhoYqNjZUktW3bVps3b9acOXM0ePBgnT17VidPnvTo3o8ePapatWr5VJOJn88DAMDfMGjNvaScTqccDofatm2rChUqaM2aNe7XMjMzlZWVpbi4OJ+O6fMT6gAAQPEkJSWpT58+iomJ0alTp7R48WKtW7dOq1atUmRkpEaOHKnExERFRUUpIiJCCQkJiouL8+lKeYlwBwBYVQAeLnfs2DENHTpUR44cUWRkpFq0aKFVq1apR48ekqRZs2YpJCREgwYNksPhUK9evTRv3jyfz+P1fe5lCve5A0DZVhr3uS9ZZchxKgzuZchxjMSaOwAAJsO0PADAkoL0O18MQbgDAKzJxOHOtDwAACZD5w4AsCaDHmITjAh3AIA1mTfbCXcAgEWZONxZcwcAwGTo3AEA1mTie+EIdwCANZk325mWBwDAbOjcAQDWZOL2lnAHAFiTidfcTfy5BQAAa6JzBwBYk3kbd8IdAGBRJg53puUBADAZOncAgDWZ+II6wh0AYE0mnrsm3AEA1mText3Mn1sAALAmOncAgDWx5g4AgMmYN9uZlgcAwGzo3AEAlmTiWXnCHQBgUSHmTXem5QEAMBk6dwCANZm3cSfcAQAWZeJFd6blAQAwGTp3eOWlV1/TZ2u+0M/7DyjMblfrVi00bswjali/vsd+23d+q1nPv6hvd+1WSLlyatr4Sr2a+oLCwsICUzjgJf7GLci8jTvhDu9s2rJNdw++Tc2vukqFhYWa+fwLGjkqXh9/sFSVKlWU9Pv/6d338Gg9eO8ITRj/hMqVL6c9mT8qJIQJIgQ//sYtyMThbnO5XK5AF2G4grxAV2B6J078R3HX36g3X1ug9m3bSJJuHzJMnTpeozGjHw5wdUDJ8TceYGHhfj9F4dfrDTlOuc5dDTmOkfi4iWI5lff7B6jIiAhJ0r//fUI7d+1Wtago3TF0hDpd30ND7r1fW7ZtD2SZQLHxN46yLKjD/dChQ7r33nv/dh+Hw6Hc3FyPzeFwlFKF1uR0OvX0M8+qTauWurJRrCTp0K+/SpJemP+ybht4i16Z97yaNW2i4Q88pAMHswJZLuAz/sYtwmbQFoSCOtxPnDihtLS0v90nJSVFkZGRHlvKjOdKqUJrmvz0NO396SfNeibFPeZ0OiVJg28dqEEDblazpk30z8cfU4P69fT+h8sCVSpQLPyNW4TNZswWhAJ6Qd3y5cv/9vWff/75ksdISkpSYmKix5jdda5EdeHipjw9Xeu+3KA3X1ugWjVrusdrVK8uSbqiYUOP/a9o0ECHs7NLtUagJPgbhxkENNwHDBggm82mv7umz3aJT0V2u112u91zkAvqDOdyuTQ15RmtXvuF3nj1ZdWtc7nH63Uur63oGjW0/8ABj/EDB7PU5dpOpVgpUDz8jVtRcHbdRgjotPxll12mDz74QE6n84Lbtm3bAlke/mTy09O0/JNP9Ny0p1S5ciX9dvy4fjt+XAUFBZJ+/xA2cvhQvfH2O/p09ec6mHVIs1+Yp58PHNCtt/QPcPXApfE3bkEmnpYP6K1wN998s1q1aqUpU6Zc8PWdO3eqdevW7rUur9G5G65xy7YXHE+ZkqyB/W92//vlV1/XW0veU05Ojpo0vlLjxjyidm1al1aZQLHxNx5kSuNWuG82GHKcch2vNeQ4RgpouH/11VfKz89X7969L/h6fn6+tmzZoq5dfbyHkHAHgLKtNMJ9o0Hhfg3hXjoIdwAo20oj3Dd9bchxynXobMhxjBTUt8IBAADf8Wx5AIA1BenFcEYg3AEA1kS4AwBgMiYOd9bcAQAwGTp3AIBFmbdzJ9wBANbEtDwAACgr6NwBANZk4s6dcAcAWJOJw51peQAATIbOHQBgTSbu3Al3AIA1mTjcmZYHAMBkCHcAAEpJSkqK2rdvrypVqig6OloDBgxQZmamxz7Z2dm65557VKtWLVWuXFlt2rTR+++/79N5CHcAgDXZbMZsPli/fr3i4+P1zTffaPXq1Tp37px69uyp/Px89z5Dhw5VZmamli9frl27dmngwIG6/fbbtX37du/fmsvlcvlUWVlQkBfoCgAAJREW7vdTOHfvMOQ4IVe3Kvbv/vbbb4qOjtb69evVpUsXSVJ4eLhSU1N1zz33uPerVq2apk+frvvuu8+7mopdEQAAKJGcnBxJUlRUlHusU6dOWrJkiU6cOCGn06l33nlHBQUF6tatm9fH5Wp5AIA1GXS1vMPhkMPh8Biz2+2y2+1/+3tOp1NjxoxR586ddfXVV7vH3333XQ0ePFjVqlVT+fLlValSJaWnpys2NtbrmujcAQDWZNCae0pKiiIjIz22lJSUS54+Pj5eu3fv1jvvvOMxPmHCBJ08eVKff/65tmzZosTERN1+++3atWuX92+NNXcAQNApjTX377815Djnrmjsc+c+evRoLVu2TF9++aUaNGjgHv/pp58UGxur3bt366qrrnKP33jjjYqNjdX8+fO9qolpeQCANRk0Le/NFPwfXC6XEhISlJ6ernXr1nkEuySdPn1akhQS4jmxXq5cOTmdTq9rItwBANYUgCfUxcfHa/HixVq2bJmqVKmi7OxsSVJkZKQqVqyoJk2aKDY2Vg8++KCeffZZVatWTR9++KFWr16tFStWeH0epuUBAMGnNKbl9+w25DghTa6+9E7/x3aRDxSvv/66hg8fLknau3evxo8frw0bNigvL0+xsbEaN26cx61xlzwP4Q4ACDqlEu7fGXKckCZXXXqnUsa0PADAmvjiGAAAUFbQuQMArMnEnTvhDgCwJsIdAACTMW+2s+YOAIDZ0LkDAKyJaXkAAEzGxOHOtDwAACZD5w4AsCjzdu6EOwDAmpiWBwAAZQWdOwDAmkzcuRPuAABrMnG4My0PAIDJ0LkDAKyJzh0AAJQVdO4AAGuicwcAAGUFnTsAwJpM3LkT7gAAazJvtjMtDwCA2dC5AwAsyrytO+EOALAmE6+5My0PAIDJ0LkDAKzJvI074Q4AsCabidOdaXkAAEyGzh0AYE0mvqCOcAcAWBPhDgCAyZg321lzBwDAbOjcAQAWZd7WnXAHAFiTidfcmZYHAMBk6NwBANZk3sadcAcAWJV5051peQAATIbOHQBgTSa+oI5wBwBYk3mznWl5AADMhs4dAGBNTMsDAGA2hDsAAOZi3mxnzR0AALOhcwcAWBNr7gAAmIyJw51peQAATIZwBwDAZJiWBwBYko1peQAAUFbQuQMArMnEnTvhDgCwKPOGO9PyAACYDJ07AMCazNu4E+4AAIsy8Zo70/IAAGuy2YzZfJCSkqL27durSpUqio6O1oABA5SZmVlkv4yMDHXv3l2VK1dWRESEunTpojNnznh9HsIdAIBSsn79esXHx+ubb77R6tWrde7cOfXs2VP5+fnufTIyMtS7d2/17NlTmzZt0ubNmzV69GiFhHgf2TaXy+XyxxsIqIK8QFcAACiJsHD/n+PMKWOOU7FKsX/1t99+U3R0tNavX68uXbpIkjp27KgePXpo6tSpxT4unTsAwJoCMC3/Vzk5OZKkqKgoSdKxY8e0ceNGRUdHq1OnTqpZs6a6du2qDRs2+HRcwh0AgBJwOBzKzc312BwOxyV/z+l0asyYMercubOuvvpqSdLPP/8sSZo0aZLuv/9+ffrpp2rTpo1uuOEG7d271/uiXEAJFRQUuJKTk10FBQWBLgUwHH/fuJTk5GSXJI8tOTn5kr83atQoV7169VyHDh1yj3399dcuSa6kpCSPfZs3b+4aP3681zWZc80dpSo3N1eRkZHKyclRREREoMsBDMXfNy7F4XAU6dTtdrvsdvtFf2f06NFatmyZvvzySzVo0MA9vn//fjVs2FBvvPGGhgwZ4h4fPHiwypcvr7feesurmrjPHQCAErhUkP+Zy+VSQkKC0tPTtW7dOo9gl6T69eurdu3aRW6P+/HHH9WnTx+vayLcAQAoJfHx8Vq8eLGWLVumKlWqKDs7W5IUGRmpihUrymaz6fHHH1dycrJatmypVq1aKS0tTXv27NHSpUu9Pg/hDgBAKUlNTZUkdevWzWP89ddf1/DhwyVJY8aMUUFBgcaOHasTJ06oZcuWWr16ta644gqvz8OaO0rM4XAoJSVFSUlJXk9NAWUFf98oiwh3AABMhvvcAQAwGcIdAACTIdwBADAZwh0AAJMh3FEiL774ourXr6+wsDBdc8012rRpU6BLAgzz5Zdfql+/fqpdu7ZsNps+/PDDQJcEeIVwR7EtWbJEiYmJSk5O1rZt29SyZUv16tVLx44dC3RpgCHy8/PVsmVLvfjii4EuBfAJt8Kh2K655hq1b99eL7zwgqTfv+Gobt26SkhI0Pjx4wNcHWAsm82m9PR0DRgwINClAJdE545iOXv2rLZu3aobb7zRPRYSEqIbb7xRGRkZAawMAEC4o1iOHz+uwsJC1axZ02O8Zs2a7mclAwACg3AHAMBkCHcUS/Xq1VWuXDkdPXrUY/zo0aOqVatWgKoCAEiEO4opNDRUbdu21Zo1a9xjTqdTa9asUVxcXAArAwDwla8otsTERA0bNkzt2rVThw4dNHv2bOXn52vEiBGBLg0wRF5envbt2+f+9/79+7Vjxw5FRUUpJiYmgJUBf49b4VAiL7zwgmbMmKHs7Gy1atVKc+fO1TXXXBPosgBDrFu3Ttdff32R8WHDhmnhwoWlXxDgJcIdAACTYc0dAACTIdwBADAZwh0AAJMh3AEAMBnCHQAAkyHcAQAwGcIdAACTIdyBMqp+/fqaPXu21/svXLhQVatWLfF5bTabPvzwwxIfB4D/EO6AD2w2299ukyZNCnSJAMCz5QFfHDlyxP3zkiVLNHHiRGVmZrrHwsPD3T+7XC4VFhaqfHn+ZwagdNG5Az6oVauWe4uMjJTNZnP/e8+ePapSpYpWrlyptm3bym63a8OGDRo+fLgGDBjgcZwxY8aoW7du7n87nU6lpKSoQYMGqlixolq2bKmlS5f6VNvMmTPVvHlzVa5cWXXr1tXDDz+svLy8Ivt9+OGHatSokcLCwtSrVy8dOnTI4/Vly5apTZs2CgsLU8OGDTV58mSdP3/ep1oABBbhDhhs/PjxmjZtmn744Qe1aNHCq99JSUnRokWLNH/+fH333XcaO3ashgwZovXr13t93pCQEM2dO1ffffed0tLStHbtWj3xxBMe+5w+fVpPPfWUFi1apK+//lonT57UHXfc4X79q6++0tChQ/Xoo4/q+++/10svvaSFCxfqqaee8roOAIHHfCFgsClTpqhHjx5e7+9wOPT000/r888/V1xcnCSpYcOG2rBhg1566SV17drVq+OMGTPG/XP9+vX15JNPatSoUZo3b557/Ny5c3rhhRfc39yXlpampk2batOmTerQoYMmT56s8ePHa9iwYe46pk6dqieeeELJyclevycAgUW4AwZr166dT/vv27dPp0+fLvKB4OzZs2rdurXXx/n888+VkpKiPXv2KDc3V+fPn1dBQYFOnz6tSpUqSZLKly+v9u3bu3+nSZMmqlq1qn744Qd16NBBO3fu1Ndff+3RqRcWFhY5DoDgRrgDBqtcubLHv0NCQvTXb1Y+d+6c++c/1sU//vhjXX755R772e12r8554MAB3XTTTXrooYf01FNPKSoqShs2bNDIkSN19uxZr0M5Ly9PkydP1sCBA4u8FhYW5tUxAAQe4Q74WY0aNbR7926PsR07dqhChQqSpGbNmslutysrK8vrKfi/2rp1q5xOp5577jmFhPx+Kc27775bZL/z589ry5Yt6tChgyQpMzNTJ0+eVNOmTSVJbdq0UWZmpmJjY4tVB4DgQLgDfta9e3fNmDFDixYtUlxcnN58803t3r3bPeVepUoVjRs3TmPHjpXT6dS1116rnJwcff3114qIiHCvf/+d2NhYnTt3Ts8//7z69eunr7/+WvPnzy+yX4UKFZSQkKC5c+eqfPnyGj16tDp27OgO+4kTJ+qmm25STEyMbr31VoWEhGjnzp3avXu3nnzySWP/gwHgN1wtD/hZr169NGHCBD3xxBNq3769Tp06paFDh3rsM3XqVE2YMEEpKSlq2rSpevfurY8//lgNGjTw6hwtW7bUzJkzNX36dF199dV66623lJKSUmS/SpUq6R//+Ifuuusude7cWeHh4VqyZIlHrStWrNBnn32m9u3bq2PHjpo1a5bq1atXsv8QAJQqm+uvi4EAAKBMo3MHAMBkCHcAAEyGcAcAwGQIdwAATIZwBwDAZAh3AABMhnAHAMBkCHcAAEyGcAcAwGQIdwAATIZwBwDAZAh3AABM5v8BCsKucapzpt0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:\t0.547826\n",
      "F1-score:\t0.500000\n",
      "Precision:\t0.500000\n",
      "Recall:\t\t0.500000\n",
      "\n",
      "Classification performance:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.59      0.59        63\n",
      "           1       0.50      0.50      0.50        52\n",
      "\n",
      "    accuracy                           0.55       115\n",
      "   macro avg       0.54      0.54      0.54       115\n",
      "weighted avg       0.55      0.55      0.55       115\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mat_dnn = confusion_matrix(y, y_pred_dnn)\n",
    "sns.heatmap(mat_dnn.T, square=True, annot=True, fmt='d', cmap='RdPu')\n",
    "plt.xlabel('True label')\n",
    "plt.ylabel('Predicted label')\n",
    "plt.title(\"DNN\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Accuracy:\\t%f\" % accuracy_score(y, y_pred_dnn))\n",
    "print(\"F1-score:\\t%f\" % f1_score(y, y_pred_dnn))\n",
    "print(\"Precision:\\t%f\" % precision_score(y, y_pred_dnn))\n",
    "print(\"Recall:\\t\\t%f\" % recall_score(y, y_pred_dnn))\n",
    "print(\"\\nClassification performance:\\n%s\" % classification_report(y, y_pred_dnn))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c49dbde6",
   "metadata": {},
   "source": [
    "### Integrated Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "428d8796",
   "metadata": {},
   "source": [
    "### XGBoost（Extreme Gradient Boosting）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b8914e8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b645e495",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for XGBoost:  {'colsample_bytree': 0.9, 'learning_rate': 0.1, 'max_depth': 7, 'n_estimators': 500, 'subsample': 0.7}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "    'learning_rate': [0.01, 0.05, 0.1],\n",
    "    'n_estimators': [100, 500, 1000],\n",
    "    'max_depth': [3, 5, 7],\n",
    "    'subsample': [0.7, 0.9],\n",
    "    'colsample_bytree': [0.7, 0.9]\n",
    "}\n",
    "\n",
    "xgb_model = xgb.XGBClassifier()\n",
    "grid_search = GridSearchCV(xgb_model, param_grid, cv=10, scoring='accuracy')\n",
    "grid_search.fit(X, y)\n",
    "\n",
    "print(\"Best parameters for XGBoost: \", grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "112c50d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/quinne/miniforge3/envs/tf/lib/python3.8/site-packages/xgboost/sklearn.py:1395: UserWarning: `use_label_encoder` is deprecated in 1.7.0.\n",
      "  warnings.warn(\"`use_label_encoder` is deprecated in 1.7.0.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predicted Probabilities for XGBoost:\n",
      "[0.9113043  0.9959842  0.91978276 0.28968361 0.35409638 0.01283397\n",
      " 0.15150708 0.05639541 0.94458073 0.16305727 0.02597693 0.02691187\n",
      " 0.08442254 0.75941902 0.03914353 0.06646032 0.10853269 0.9909752\n",
      " 0.98267502 0.90153688 0.01033479 0.00144019 0.04755609 0.52820539\n",
      " 0.13906693 0.24220245 0.37389433 0.08736603 0.22018874 0.58416826\n",
      " 0.07046657 0.09912316 0.00813416 0.84126133 0.0158952  0.48588607\n",
      " 0.81343836 0.06485264 0.94100416 0.02418549 0.18227127 0.07036322\n",
      " 0.93836904 0.99571854 0.00548885 0.08118413 0.32153362 0.24564432\n",
      " 0.06310773 0.00507469 0.39936602 0.15304701 0.19440822 0.41769597\n",
      " 0.02686487 0.90269536 0.14639568 0.28707707 0.00456495 0.68693066\n",
      " 0.82390743 0.43291453 0.58942497 0.48311329 0.7396515  0.04211948\n",
      " 0.1665574  0.80980623 0.89834189 0.27003676 0.91332024 0.00544643\n",
      " 0.00752516 0.54873139 0.87709796 0.04681002 0.02046329 0.91806513\n",
      " 0.02186144 0.91676813 0.98522514 0.95572746 0.0439298  0.95810455\n",
      " 0.98518282 0.08727308 0.89419723 0.34006155 0.79460925 0.11114472\n",
      " 0.94375503 0.04093185 0.94443709 0.96305025 0.50971723 0.6239087\n",
      " 0.60347462 0.25989446 0.55297643 0.46677461 0.26694191 0.78727317\n",
      " 0.00998944 0.12998058 0.67367601 0.58649969 0.11950915 0.03633802\n",
      " 0.04370527 0.70269811 0.5783987  0.29482675 0.28721425 0.52628243\n",
      " 0.18003793]\n"
     ]
    }
   ],
   "source": [
    "#Best parameters for XGBoost:  {'colsample_bytree': 0.9, 'learning_rate': 0.1, 'max_depth': 7, 'n_estimators': 500, 'subsample': 0.7}\n",
    "\n",
    "xgb_model = xgb.XGBClassifier(\n",
    "    colsample_bytree=0.9, \n",
    "    learning_rate=0.1, \n",
    "    max_depth=7, \n",
    "    n_estimators=500, \n",
    "    subsample=0.7,\n",
    "    use_label_encoder=False\n",
    ")\n",
    "\n",
    "y_prob_xgb = np.zeros_like(y, dtype=float)\n",
    "\n",
    "for train_idx, test_idx in cv.split(X, y):\n",
    "    X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n",
    "    y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n",
    "    \n",
    "    xgb_model.fit(X_train, y_train)\n",
    "    y_prob_xgb[test_idx] = xgb_model.predict_proba(X_test)[:, 1]\n",
    "\n",
    "y_pred_xgb = (y_prob_xgb > 0.5).astype(int)\n",
    "\n",
    "print(\"\\nPredicted Probabilities for XGBoost:\")\n",
    "print(y_prob_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "66445808",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0,\n",
       "       0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0,\n",
       "       1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1,\n",
       "       1, 0, 0, 1, 0])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4281c004",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAHHCAYAAACiDxGKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0j0lEQVR4nO3deVxU9f7H8fegMCBrLogmuKSl5pKJC2laqbmkadKqXpese00klTZpMzNDb6VYaZp1XbpqLqWlZpZr6cVUXMpSyrLQFNS6gKCOCPP7o9+dmlCbwRkGz3k9H4/zeDDfc+acz3Dtvvl8zzIWu91uFwAAMAw/XxcAAAA8i3AHAMBgCHcAAAyGcAcAwGAIdwAADIZwBwDAYAh3AAAMhnAHAMBgCHcAAAyGcAfKiY0bN8pisWjp0qW+LgXAZY5wB9wwYMAABQYG6ttvvy2xbuLEibJYLFq5cqVjzGaz6bXXXlP79u11xRVXKCAgQDVr1tTtt9+uhQsXqqioqCzLvyQLFixQamqqr8sA4ALCHXDD5MmTValSJQ0bNsxp/ODBg3r++ecVHx+vnj17SpKOHz+udu3a6eGHH1ZISIiefvppzZw5U4mJiSooKFC/fv304osv+uJjlArhDlw+Kvq6AOByEhkZqUmTJunvf/+75s6dq0GDBkmShg8fLn9/f02dOtWx7d/+9jft2rVL7733nvr27eu0n+TkZO3YsUMZGRllWj8Ac6BzB9z0wAMPqF27dnr00Uf1yy+/6N1339XHH3+sF154QVdeeaUkKS0tTWvWrNHf//73EsH+P7Gxserfv3+J8aKiIj355JOKiopScHCwbr/9dh06dKjEdkuWLFHLli0VFBSkqlWrasCAAfr5559LbLd+/XrdeOONCg4OVkREhHr37q19+/Y5bXPy5EmNGjVKderUkdVqVWRkpLp06aKdO3dKkm666SatWrVKP/30kywWiywWi+rUqePurw5AGaFzB9xksVg0c+ZMtWjRQg899JA+//xzxcbGKiEhwbHNihUrJP12jt5dEyZMkMVi0RNPPKFjx44pNTVVnTt31u7duxUUFCRJmjNnjoYMGaJWrVopJSVF2dnZmjp1qrZs2aJdu3YpIiJCkrR27Vp1795d9erV03PPPafTp0/rtddeU7t27bRz505HQA8bNkxLly7ViBEj1LhxY/3yyy/avHmz9u3bp+uvv15PPfWUcnNzdfjwYU2ZMkWSFBIScgm/RQBeZQdQKsnJyXZJ9goVKtjT09Od1t1xxx12SfacnByn8dOnT9uPHz/uWP773/861m3YsMEuyX7llVfa8/LyHOOLFy+2S7JPnTrVbrfb7WfPnrVHRkbamzRpYj99+rRju5UrV9ol2Z999lnH2HXXXWePjIy0//LLL46xPXv22P38/OwDBw50jIWHh9sTEhIu+nlvu+02e+3atf/6FwPA55iWB0qpatWqkqSaNWuqSZMmTuvy8vIklexuZ8yYoWrVqjmW9u3bl9jvwIEDFRoa6nh95513qkaNGvroo48kSTt27NCxY8c0fPhwBQYGOra77bbb1LBhQ61atUqSdPToUe3evVuDBw9W5cqVHds1a9ZMXbp0cexPkiIiIvTFF1/oyJEjpfpdAChfCHegFA4dOqSxY8eqSZMmOnTokP75z386rf9fOOfn5zuNx8fH69NPP9Wnn36qZs2anXffDRo0cHptsVhUv359/fjjj5Kkn376SZJ0zTXXlHhvw4YNHesvtl2jRo104sQJFRQUSJL++c9/au/evYqOjlbr1q313HPP6Ycffrjo7wBA+UW4A6UwYsQISdLq1at11113acKECU5h2LBhQ0nS3r17nd4XHR2tzp07q3PnzrriiivKruC/cPfdd+uHH37Qa6+9ppo1a+qll17Stddeq9WrV/u6NAClQLgDblq2bJk+/PBDjR8/XrVq1VJqaqoCAgKcLqj7373u8+fPd3v/3333ndNru92uAwcOOC5+q127tiSd9za6jIwMx/qLbbd//35VrVpVwcHBjrEaNWpo+PDhWr58uQ4ePKgqVapowoQJjvUWi8XtzwLANwh3wA0nT57Uww8/rBYtWigxMVHSb+fcx48fr48//lhLliyRJLVr105dunTRm2++qQ8++OC8+7Lb7ecdnzdvnk6ePOl4vXTpUh09elTdu3eX9NstdJGRkZoxY4ZsNptju9WrV2vfvn267bbbJP0W1tddd53mzp2rnJwcx3Z79+7VJ598oh49ekj67da73NxcpxoiIyNVs2ZNp/0HBweX2A5A+WSxX+j/YQCUMHLkSL3++uvaunWrWrVq5RgvKipS69atlZWVpf379ys0NFTHjh1Tt27dtGvXLnXv3t0xFZ+VlaW1a9dq/fr16t69u+PCto0bN+rmm29W06ZNZbFYNGTIEGVnZys1NVW1atXSnj17VKlSJUm/3wrXpk0b3XfffY5b4SIjI897K1z9+vU1dOhQx61w586dU3p6uurWraucnBzVqlVLd955p5o3b66QkBCtXbtWixcv1iuvvKKkpCRJ0ksvvaTHH39co0ePVqtWrRQSEqJevXqV7f8AAFzj24v1gcvHjh077BUqVLCPGDHivOu3bdtm9/Pzsz/88MOOsdOnT9tTU1PtcXFx9rCwMHvFihXtUVFR9p49e9rnz59vP3funGPb/90Kt3DhQntycrI9MjLSHhQUZL/tttvsP/30U4njLVq0yN6iRQu71Wq1V65c2d6/f3/74cOHS2y3du1ae7t27exBQUH2sLAwe69evezffPONY73NZrM/9thj9ubNm9tDQ0PtwcHB9ubNm9unT5/utJ/8/Hx7v3797BEREXZJ3BYHlGN07gAAGAzn3AEAMBjCHQAAgyHcAQAwGMIdAACDIdwBADAYwh0AAIMh3AEAMJiKvi7AG3pYEv56I+AytiLtPl+XAHhVhbYlvw7Z0zyVFR/Zp3lkP55E5w4AgMEYsnMHAOCvGLm7JdwBAKZkkXG/xphwBwCYkpE7dyN/NgAATInOHQBgSkzLAwBgMEaeujbyZwMAwJTo3AEApmTcSXnCHQBgUn4Gjnem5QEAMBg6dwCAKRm3byfcAQAmxbQ8AAC4bNC5AwBMybh9O+EOADApI09dE+4AAFMy8uNnjfyHCwAApkTnDgAwJSN3t4Q7AMCUuBUOAABcNujcAQCmZNy+nXAHAJiUn8W48c60PAAABkPnDgAwJeP27YQ7AMCkjDx1beTPBgCAKdG5AwBMyciPnyXcAQCmZOSpa8IdAGBKRu7cjfyHCwAApkTnDgAwJSN3t4Q7AMCUjDspb+w/XAAAMCU6dwCAKRn5K18JdwCAKRk32pmWBwDAcOjcAQCmxLQ8AAAGY+SpayN/NgAATInOHQBgSsadlCfcAQAmxTl3AAAMxrjRzjl3AAAMh84dAGBKTMsDAGAwxo12puUBADAcOncAgCkZubsl3AEApsS0PAAAuGzQuQMATImr5QEAMBgjT10b+bMBAGBKdO4AAFMy7qQ8nTsAwKT8PLRciokTJ8pisWjUqFGOsTNnzighIUFVqlRRSEiI4uPjlZ2d7dZ+CXcAgCn5yeKRpbS2b9+umTNnqlmzZk7jo0eP1ooVK7RkyRJt2rRJR44cUd++fd38bAAAoEzl5+erf//+mjVrlq644grHeG5urt5++21NnjxZt9xyi1q2bKnZs2frP//5j7Zu3ery/gl3AIApWTy02Gw25eXlOS02m+2ix05ISNBtt92mzp07O42np6ersLDQabxhw4aKiYlRWlqay5+NcAcAmJKnzrmnpKQoPDzcaUlJSbngcd99913t3LnzvNtkZWUpICBAERERTuPVq1dXVlaWy5+Nq+UBALgEycnJSkpKchqzWq3n3fbQoUMaOXKkPv30UwUGBnqtJsIdAGBKnpq6tlqtFwzzP0tPT9exY8d0/fXXO8aKior02Wef6fXXX9eaNWt09uxZ5eTkOHXv2dnZioqKcrkmwh0AYEq+uM+9U6dO+uqrr5zGhgwZooYNG+qJJ55QdHS0/P39tW7dOsXHx0uSMjIylJmZqbi4OJePQ7gDAFBGQkND1aRJE6ex4OBgValSxTE+dOhQJSUlqXLlygoLC1NiYqLi4uLUtm1bl49DuAMATKm8fnHMlClT5Ofnp/j4eNlsNnXt2lXTp093ax+EOwDAlMpLtG/cuNHpdWBgoKZNm6Zp06aVep/cCgcAgMHQuQMATMnI3S3hDgAwJcIdAACDsZSbs+6eZ+Q/XAAAMCU6dwCAKRm5uyXcAQCmZORwN/JnAwDAlOjcAQCmZNzL6Qh3AIBJGXnq2sifDQAAU6JzBwCYkpHvcyfcAQCmZOSpayN/NgAATInOHQBgSkbubgl3AIApGfeMO+EOADApPwPHu5FnJQAAMCU6dwCAKRm5uyXcAQCmZNxJeWP/4QIAgCnRuaNU7nqii4ZM7KPlqev15uj3Sqx//qPhiu1+rcb3mam0D770QYWAe95csUpr03fqh6NHFegfoOsaXKVH7r5LdWtEObYZlPJPbd+f4fS+u2/uqOcGDyzrcuEBRu5uCXe4rUFsjLr/o71+2HP4vOv7jLpZdnsZFwVcoh0Z3+q+TjerSd26KiouVurS9/TAS69oRcoLqmS1Ora7q2MHjejbx/E6yBrgg2rhCUYOdyN/NnhBYLBVj88frFcfXKD8/54qsb5e81rq+0gnpd7/bx9UB5Tem4+O1h03tleDWleqYUy0XnxgqI7+8qu+Ofij03aB1gBViwh3LCFBQb4pGLgIn3buJ06c0L/+9S+lpaUpKytLkhQVFaUbbrhBgwcPVrVq1XxZHs5j+LS7tW3V19q9LkP3Pt3NaZ01yF+PLxis6QmL9d/sPB9VCHjGydO//fEaHhLsNL4ybatW/GerqoaH6abrrtNDvXsq6A+dPS4ffHGMF2zfvl1du3ZVpUqV1LlzZ1199dWSpOzsbL366quaOHGi1qxZo9jYWF+ViD/pcE9L1b8+WiNb/fO86x+ccqf2/ecHbf2Qc+y4vBUXF2vi/Hd1fYP6alCrlmP8trZtVLNqFUVGRCjj0GFNXrxUP2Zl6dWHE3xYLUrLyFPXPgv3xMRE3XXXXZoxY4YsFue/nux2u4YNG6bExESlpaVddD82m002m81prEhFqqAKHq/ZzKrWitA/pt6pp7q8pkLbuRLr2/Rqqua3XK3EFhN9UB3gWePnzdd3P/+sfz81xmn87ps7On6+OrqWqkWE6/5JLysz+5hiqkeWdZnABfks3Pfs2aM5c+aUCHZJslgsGj16tFq0aPGX+0lJSdG4ceOcxuorVg3U2mO1QmrQMkZXVA/Tazt//z+7ChUrqEmH+uo1oqNWvfG5alxVVUtyXnJ635PvPaivPz+gMTdPLeuSgVJ5Yd58bdqzR/OefEJRlStfdNtmV9WTJGUeI9wvR8adlPdhuEdFRWnbtm1q2LDheddv27ZN1atX/8v9JCcnKykpyWnsrvDHPVIjfrd7XYYeavKC09jo2X/T4f3ZWjLpE+WdKNDqmZud1r+x92nNGv2evljxVVmWCpSK3W7XhHcWaG36Ts1Jfly1XLjmZ/9PmZKkauHh3i4PXuDnZ9x491m4P/roo/r73/+u9PR0derUyRHk2dnZWrdunWbNmqWXX375L/djtVpl/dPFLEzJe97pfJt++vqo09iZApvyfsl3jJ/vIrrjmb8q+8dfyqRG4FKMn/dvrdr6hV4fmajgwEAdz8mVJIVWClJgQIAys49p1dYv1KFZU0WEhCjj0GFNWvCuYq+5WtfERPu4epSGhXD3vISEBFWtWlVTpkzR9OnTVVRUJEmqUKGCWrZsqTlz5ujuu+/2VXkATObd9Rsl/fagmj+a8MAQ3XFje/lXrKi0r7/RvDWf6vRZm6IqV1aXVi017PaePqgWuDiL3e77x40UFhbqxIkTkqSqVavK39//kvbXw8KVqzC2FWn3+boEwKsqtG3v9WOstD7vkf30tD3rkf14Url4Qp2/v79q1Kjh6zIAACZiMfC9cAb+aAAAmFO56NwBAChrfue5FdsoCHcAgCkZ+Wp5puUBADAYOncAgCnxEBsAAAzGwKfcmZYHAMBo6NwBAKbEtDwAAAZzvm8lNQrCHQBgSkbu3DnnDgCAwdC5AwBMycgPsSHcAQCmZOBsZ1oeAACjoXMHAJgS0/IAABiMkW+FY1oeAACDoXMHAJiSke9zJ9wBAKZk5HPuTMsDAGAwdO4AAFMycONOuAMAzMnI0/KEOwDAlPy4FQ4AAFwu6NwBAKbEtDwAAAbjZ+C5awN/NAAAzInOHQBgSkZ+tjzhDgAwJSM/fpZpeQAADIZwBwCYksVi8cjijjfeeEPNmjVTWFiYwsLCFBcXp9WrVzvW33TTTSX2P2zYMLc/G9PyAABT8sW0fK1atTRx4kQ1aNBAdrtdc+fOVe/evbVr1y5de+21kqQHH3xQzz//vOM9lSpVcvs4hDsAAGWkV69eTq8nTJigN954Q1u3bnWEe6VKlRQVFXVJx2FaHgBgShY/zyw2m015eXlOi81m+8vjFxUV6d1331VBQYHi4uIc4/Pnz1fVqlXVpEkTJScn69SpU25/NsIdAGBKfhaLR5aUlBSFh4c7LSkpKRc87ldffaWQkBBZrVYNGzZMy5YtU+PGjSVJ/fr107///W9t2LBBycnJeueddzRgwAC3P5vFbrfbS/2bKad6WBJ8XQLgVSvS7vN1CYBXVWjb3uvH+LbFdI/sp/bWoSU6davVKqvVet7tz549q8zMTOXm5mrp0qV66623tGnTJkfA/9H69evVqVMnHThwQFdddZXLNbl0zv3LL790eYfNmjVzeVsAAC53Fwvy8wkICFD9+vUlSS1bttT27ds1depUzZw5s8S2bdq0kSTvhPt1110ni8WiCzX5/1tnsVhUVFTk8sEBAPCV8vKVr8XFxRc8R797925JUo0aNdzap0vhfvDgQbd2CgBAeeeLb4VLTk5W9+7dFRMTo5MnT2rBggXauHGj1qxZo++//14LFixQjx49VKVKFX355ZcaPXq0OnTo4PasuEvhXrt27VJ9CAAA8Ltjx45p4MCBOnr0qMLDw9WsWTOtWbNGXbp00aFDh7R27VqlpqaqoKBA0dHRio+P19NPP+32cUp1n/s777yjGTNm6ODBg0pLS1Pt2rWVmpqqunXrqnfv3qXZJQAAZcrig/vF3n777Quui46O1qZNmzxyHLc/2htvvKGkpCT16NFDOTk5jnPsERERSk1N9UhRAAB4m8XimaU8cjvcX3vtNc2aNUtPPfWUKlSo4BiPjY3VV1995dHiAACA+9yelj948KBatGhRYtxqtaqgoMAjRQEA4G2+uKCurLjdudetW9dxaf4fffzxx2rUqJEnagIAwOs89fjZ8sjtzj0pKUkJCQk6c+aM7Ha7tm3bpoULFyolJUVvvfWWN2oEAMDjyuv5ck9wO9wfeOABBQUF6emnn9apU6fUr18/1axZU1OnTtW9997rjRoBAIAbSnUrXP/+/dW/f3+dOnVK+fn5ioyM9HRdAAB4l4HPuZf6+9yPHTumjIwMSb89frZatWoeKwoAAG8z8rS825cCnDx5Un/7299Us2ZNdezYUR07dlTNmjU1YMAA5ebmeqNGAADgBrfD/YEHHtAXX3yhVatWKScnRzk5OVq5cqV27Nihf/zjH96oEQAAj+Nq+T9YuXKl1qxZo/btf/+u3a5du2rWrFnq1q2bR4sDAMBbuM/9D6pUqaLw8PAS4+Hh4briiis8UhQAACg9t8P96aefVlJSkrKyshxjWVlZeuyxx/TMM894tDgAALzFyM+Wd2lavkWLFrL84RN89913iomJUUxMjCQpMzNTVqtVx48f57w7AODyUE7Pl3uCS+Hep08fL5cBAAA8xaVwHzt2rLfrAACgTBn5grpSP8QGAIDLWXk9X+4Jbod7UVGRpkyZosWLFyszM1Nnz551Wv/rr796rDgAALylvN6j7gluf7Rx48Zp8uTJuueee5Sbm6ukpCT17dtXfn5+eu6557xQIgAAcIfb4T5//nzNmjVLjzzyiCpWrKj77rtPb731lp599llt3brVGzUCAOBxFovFI0t55Ha4Z2VlqWnTppKkkJAQx/Pke/bsqVWrVnm2OgAAvMTIj591u6xatWrp6NGjkqSrrrpKn3zyiSRp+/btslqtnq0OAAC4ze1wv+OOO7Ru3TpJUmJiop555hk1aNBAAwcO1P333+/xAgEA8Ao/Dy3lkNtXy0+cONHx8z333KOYmBilpaWpQYMG6tWrl0eLAwDAW8rp6XKPuOT73OPi4hQXF+eJWgAAgAe4FO4ffvihyzu8/fbbS10MAABlxfRPqHP12fIWi0VFRUWXUg8AAGWivF7p7gkuhXtxcbG36wAAAB7Cs+UBAKbEBXUAABiN2c+5AwBgNEY+527gjwYAgDnRuQMATMn059zz8vJc3mFYWFipiwEAoKyY/j73iIgIl7/WjvvcAQDwLZfCfcOGDY6ff/zxR40ZM0aDBw92PHY2LS1Nc+fOVUpKineqBADAw0w/Ld+xY0fHz88//7wmT56s++67zzF2++23q2nTpnrzzTc1aNAgz1cJAICHcbX8H6SlpSk2NrbEeGxsrLZt2+aRogAAQOm5He7R0dGaNWtWifG33npL0dHRHikKAACv87N4ZimH3L4VbsqUKYqPj9fq1avVpk0bSdK2bdv03Xff6b333vN4gQAAeIORz7m73bn36NFD3377rXr16qVff/1Vv/76q3r16qVvv/1WPXr08EaNAADADaV6iE10dLRefPFFT9cCAECZ4YK6P/n88881YMAA3XDDDfr5558lSe+88442b97s0eIAAPAWi5/FI0t55Ha4v/fee+ratauCgoK0c+dO2Ww2SVJubi7dPADgsmGxeGYpj9wO9xdeeEEzZszQrFmz5O/v7xhv166ddu7c6dHiAACA+9w+556RkaEOHTqUGA8PD1dOTo4narpkH52e5OsSAK86tybN1yUAlz/Ouf8uKipKBw4cKDG+efNm1atXzyNFAQDgdQa+z93tcH/wwQc1cuRIffHFF7JYLDpy5Ijmz5+vRx99VA899JA3agQAAG5we1p+zJgxKi4uVqdOnXTq1Cl16NBBVqtVjz76qBITE71RIwAAnlder4bzALfD3WKx6KmnntJjjz2mAwcOKD8/X40bN1ZISIg36gMAwDs45/67+++/XydPnlRAQIAaN26s1q1bKyQkRAUFBbr//vu9USMAAHCD2+E+d+5cnT59usT46dOnNW/ePI8UBQCA1xn4gjqXp+Xz8vJkt9tlt9t18uRJBQYGOtYVFRXpo48+UmRkpFeKBADA0wx8yt31cI+IiJDFYpHFYtHVV19dYr3FYtG4ceM8WhwAAHCfy+G+YcMG2e123XLLLXrvvfdUuXJlx7qAgADVrl1bNWvW9EqRAAB4XDmdUvcEl8O9Y8eOkqSDBw8qJiZGFiPPZwAAjM/A4e72BXXr16/X0qVLS4wvWbJEc+fO9UhRAAB4nZ+HlnLI7bJSUlJUtWrVEuORkZF8KxwAAOWA2w+xyczMVN26dUuM165dW5mZmR4pCgAArzPw6WW3O/fIyEh9+eWXJcb37NmjKlWqeKQoAAC8zsD3ubsd7vfdd58efvhhbdiwQUVFRSoqKtL69es1cuRI3Xvvvd6oEQAAuMHtafnx48frxx9/VKdOnVSx4m9vLy4u1sCBAznnDgC4fJTTi+E8we1wDwgI0KJFizR+/Hjt2bNHQUFBatq0qWrXru2N+gAA8I5yOqXuCaX+u+Xqq6/WXXfdpZ49exLsAAC44I033lCzZs0UFhamsLAwxcXFafXq1Y71Z86cUUJCgqpUqaKQkBDFx8crOzvb7eO41LknJSVp/PjxCg4OVlJS0kW3nTx5sttFAABQ5nzQuNeqVUsTJ05UgwYNZLfbNXfuXPXu3Vu7du3Stddeq9GjR2vVqlVasmSJwsPDNWLECPXt21dbtmxx6zguhfuuXbtUWFjo+PlCeGodAOCy4YNp+V69ejm9njBhgt544w1t3bpVtWrV0ttvv60FCxbolltukSTNnj1bjRo10tatW9W2bVuXj+NSuG/YsOG8PwMAYHY2m002m81pzGq1ymq1XvR9RUVFWrJkiQoKChQXF6f09HQVFhaqc+fOjm0aNmyomJgYpaWluRXuBr5WEACAi/DQfe4pKSkKDw93WlJSUi542K+++kohISGyWq0aNmyYli1bpsaNGysrK0sBAQGKiIhw2r569erKyspy66O51Ln37dvX5R2+//77bhUAAIAveOpMcnJyconr0S7WtV9zzTXavXu3cnNztXTpUg0aNEibNm3yTDH/z6VwDw8Pd/xst9u1bNkyhYeHKzY2VpKUnp6unJwct/4IAADApzx0zt2VKfg/CggIUP369SVJLVu21Pbt2zV16lTdc889Onv2rHJycpy69+zsbEVFRblVk0vhPnv2bMfPTzzxhO6++27NmDFDFSpUkPTbeYPhw4crLCzMrYMDAGB2xcXFstlsatmypfz9/bVu3TrFx8dLkjIyMpSZmam4uDi39un2Q2z+9a9/afPmzY5gl6QKFSooKSlJN9xwg1566SV3dwkAQNnzwdXyycnJ6t69u2JiYnTy5EktWLBAGzdu1Jo1axQeHq6hQ4cqKSlJlStXVlhYmBITExUXF+fWxXRSKcL93Llz2r9/v6655hqn8f3796u4uNjd3QEA4Bs+uKT82LFjGjhwoI4eParw8HA1a9ZMa9asUZcuXSRJU6ZMkZ+fn+Lj42Wz2dS1a1dNnz7d7eO4He5DhgzR0KFD9f3336t169aSpC+++EITJ07UkCFD3C4AAACzePvtty+6PjAwUNOmTdO0adMu6Thuh/vLL7+sqKgovfLKKzp69KgkqUaNGnrsscf0yCOPXFIxAACUGQM/eM1it9vtpX1zXl6eJJW/C+nO5Pu6AsCrzq1J83UJgFdV7N3F68c4O3mZR/YTkHSHR/bjSaU643Du3DmtXbtWCxcudDxy9siRI8rPJ1QBAPA1t6flf/rpJ3Xr1k2ZmZmy2Wzq0qWLQkNDNWnSJNlsNs2YMcMbdQIA4Fl85evvRo4cqdjYWP33v/9VUFCQY/yOO+7QunXrPFocAABe46HHz5ZHbnfun3/+uf7zn/8oICDAabxOnTr6+eefPVYYAAAoHbfDvbi4WEVFRSXGDx8+rNDQUI8UBQCA15XPptsj3J6Wv/XWW5Wamup4bbFYlJ+fr7Fjx6pHjx6erA0AAO9hWv53L7/8srp166bGjRvrzJkz6tevn7777jtVrVpVCxcu9EaNAAB4XjkNZk9wO9yjo6O1Z88eLVq0SHv27FF+fr6GDh2q/v37O11gBwAAfMOtcC8sLFTDhg21cuVK9e/fX/379/dWXQAAeJWBH1DnXrj7+/vrzJkz3qoFAICyY+BpebcvqEtISNCkSZN07tw5b9QDAAAukdvn3Ldv365169bpk08+UdOmTRUcHOy0/v333/dYcQAAeI2BO3e3wz0iIkLx8fHeqAUAgLLjg+9zLytuh/vs2bO9UQcAAPAQl/9uKS4u1qRJk9SuXTu1atVKY8aM0enTp71ZGwAA3mOxeGYph1wO9wkTJujJJ59USEiIrrzySk2dOlUJCQnerA0AAO/x89BSDrlc1rx58zR9+nStWbNGy5cv14oVKzR//nwVFxd7sz4AAOAml8M9MzPT6dnxnTt3lsVi0ZEjR7xSGAAAXmXgaXmXL6g7d+6cAgMDncb8/f1VWFjo8aIAAPC68pnLHuFyuNvtdg0ePFhWq9UxdubMGQ0bNszpXnfucwcAXBYId2nQoEElxgYMGODRYgAAwKVzOdy5vx0AYCg8oQ4AAIMxbraX1zv0AABAadG5AwDMycCdO+EOADCncnqPuicwLQ8AgMHQuQMATMnAjTvhDgAwKQOHO9PyAAAYDJ07AMCceIgNAAAGY9xsJ9wBACZl4HDnnDsAAAZD5w4AMCcD3wtHuAMAzMm42c60PAAARkPnDgAwJwO3t4Q7AMCcDHzO3cB/twAAYE507gAAczJu4064AwBMysDhzrQ8AAAGQ+cOADAnA19QR7gDAMzJwHPXhDsAwJyM27gb+e8WAADMic4dAGBOnHMHAMBgjJvtTMsDAGA0dO4AAFMy8Kw84Q4AMCk/46Y70/IAABgMnTsAwJyM27gT7gAAkzLwSXem5QEAMBg6d7hke/pOvT1nnvbu26fjx09o2pSX1fmWmx3rT/zyi15OfVWb07bq5MmTir3+ej0z5nHVqR3jw6oB181av0af7t2jg8eyFejvr+vq1FNS996qG1ldkvTzr7/o1oljz/veyQPuV9dm15dlufAE4zbuhDtcc+r0aV1zzdWK73O7RiQ95rTObrcrYdQjqlixoqanTlZISLDmzJuvIf94SKveX6pKlYJ8VDXguu0/HNB9N3RQ01q1da64SFM/XqEH33pdHz76tCoFWBUVcYU2PvOi03uWbN2i2ZvWqv011/qoalwSwh1m17F9O3Vs3+686378KVO7v/xKK99brAb1r5IkPfd0strdcqtWffyx7up7R1mWCpTKmw8kOL2ecPcA3fh8sr45fEix9eqrgp+fqoWGOW2z7us96tb8egVbrWVZKjyFW+GACztbeFaSZLUGOMb8/PwUEBCg9F27fVQVcGlOnjkjSQqvVOm8678+nKn9Rw6rb6u4siwLcEm5DvdDhw7p/vvvv+g2NptNeXl5TovNZiujCiFJ9erUUc0aUXrl1deVm5ens4WFevNfc5SVna3jx0/4ujzAbcXFxZr04VK1qFNPDaJqnneb97anqV5klFrUqVfG1cFjLB5ayqFyHe6//vqr5s6de9FtUlJSFB4e7rSkvPRKGVUISfL399drk1/Wjz9lqvWNN+u6Nu30xfYd6tC+nSx+5fqfGHBeLyxfrO+yj+rlfkPOu/5M4Vl9tGuH4unaL28Wi2cWN6SkpKhVq1YKDQ1VZGSk+vTpo4yMDKdtbrrpJlksFqdl2LBhbh3Hp+fcP/zww4uu/+GHH/5yH8nJyUpKSnIas9oLL6kuuK9J40b6YPFCnTx5UoWF51S58hW6q/9ANbm2sa9LA9zywvLF2rRvr+Y+NEpREVecd5tPvtyt04VndXvL1mVcHS53mzZtUkJCglq1aqVz587pySef1K233qpvvvlGwcHBju0efPBBPf/8847XlS5weuhCfBruffr0kcVikd1uv+A2lr/4q8hqtcr654tZzuR7ojyUQmhoqKTfLrLb+80+jUx4yMcVAa6x2+2a8MESrdu7R3P+MVK1Kle94Lbvb/+Pbm7cVJVDQsuwQnhe2c+pf/zxx06v58yZo8jISKWnp6tDhw6O8UqVKikqKqrUx/HpnGmNGjX0/vvvq7i4+LzLzp07fVke/qDg1Cnt25+hfft/mz46/PMR7dufoSNHj0qSVn/yqb7YvkOHDh/W2g0bdf+w4ep8801qfwPTlrg8jF++WCt3btc/7xusSoGBOn4yT8dP5unM/18w+j8/nTiuHQe/V3zrG3xUKTzGQ9Pyl3LtV25uriSpcuXKTuPz589X1apV1aRJEyUnJ+vUqVNufTSfdu4tW7ZUenq6evfufd71f9XVo+zs/fobDXzgH47XKS9PliTdcXtPTRw/TsePn9DEl6fol19+UbVqVdW7520a/o8HfVUu4LZFaZ9LkgbPnOo0/sLdA3RHbFvH62Xb01Q9PELtGjQs0/pQfqWkpGjcuHFOY2PHjtVzzz130fcVFxdr1KhRateunZo0aeIY79evn2rXrq2aNWvqyy+/1BNPPKGMjAy9//77LtdksfswPT///HMVFBSoW7du511fUFCgHTt2qGPHju7tmGl5GNy5NWm+LgHwqoq9u3j9GEVfbPbIfs5d16pEp37eU8Z/8tBDD2n16tXavHmzatWqdcHt1q9fr06dOunAgQO66qqrXKrJp537jTfeeNH1wcHB7gc7AACu8NAXx7gS5H82YsQIrVy5Up999tlFg12S2rRpI0mXT7gDAGAmdrtdiYmJWrZsmTZu3Ki6dev+5Xt2794t6bfr1FxFuAMAzMkHX/makJCgBQsW6IMPPlBoaKiysrIkSeHh4QoKCtL333+vBQsWqEePHqpSpYq+/PJLjR49Wh06dFCzZs1cPo5Pz7l7DefcYXCcc4fRlck59x2e+e+oQqzrdwVd6Pbu2bNna/DgwTp06JAGDBigvXv3qqCgQNHR0brjjjv09NNPKyws7LzvPR86dwCAOfmgc/+rfjo6OlqbNm265OPwbFAAAAyGzh0AYFLl9FtfPIBwBwCYkw+m5csK0/IAABgMnTsAwJwM3LkT7gAAczJwuDMtDwCAwdC5AwDMycCdO+EOADAnA4c70/IAABgM4Q4AgMEwLQ8AMCcDT8sT7gAAU7rQN7QZAdPyAAAYDJ07AMCcDNy5E+4AAHMycLgzLQ8AgMHQuQMAzMnAnTvhDgAwJwOHO9PyAAAYDJ07AMCkjNu5E+4AAHNiWh4AAFwu6NwBAOZk4M6dcAcAmBPhDgCAwRg32znnDgCA0dC5AwDMiWl5AAAMxsDhzrQ8AAAGQ+cOADAp43buhDsAwJyYlgcAAJcLOncAgDkZuHMn3AEA5mTgcGdaHgAAg6FzBwCYE507AAC4XNC5AwDMic4dAABcLujcAQDmZODOnXAHAJiTcbOdaXkAAIyGzh0AYFLGbd0JdwCAORn4nDvT8gAAGAydOwDAnIzbuBPuAABzshg43ZmWBwDAYOjcAQDmZOAL6gh3AIA5Ee4AABiMcbOdc+4AABgNnTsAwKSM27oT7gAAczLwOXem5QEAMBg6dwCAORm3cSfcAQBmZdx0Z1oeAACDoXMHAJiTgS+oI9wBAOZk3GxnWh4AAKOhcwcAmJOBp+Xp3AEAJmXx0OK6lJQUtWrVSqGhoYqMjFSfPn2UkZHhtM2ZM2eUkJCgKlWqKCQkRPHx8crOznbrOIQ7AMCcyj7btWnTJiUkJGjr1q369NNPVVhYqFtvvVUFBQWObUaPHq0VK1ZoyZIl2rRpk44cOaK+ffu699HsdrvdvdIuA2fyfV0B4FXn1qT5ugTAqyr27uL1Y9hzTnhkP5aIqqV+7/HjxxUZGalNmzapQ4cOys3NVbVq1bRgwQLdeeedkqT9+/erUaNGSktLU9u2bV3aL507AMCcLBaPLDabTXl5eU6LzWZzqYTc3FxJUuXKlSVJ6enpKiwsVOfOnR3bNGzYUDExMUpLc/2PesIdAGBOHgr3lJQUhYeHOy0pKSl/efji4mKNGjVK7dq1U5MmTSRJWVlZCggIUEREhNO21atXV1ZWlssfjavlAQC4BMnJyUpKSnIas1qtf/m+hIQE7d27V5s3b/Z4TYQ7AACXwGq1uhTmfzRixAitXLlSn332mWrVquUYj4qK0tmzZ5WTk+PUvWdnZysqKsrl/TMtDwAwJYvF4pHFHXa7XSNGjNCyZcu0fv161a1b12l9y5Yt5e/vr3Xr1jnGMjIylJmZqbi4OJePQ+cOAEAZSUhI0IIFC/TBBx8oNDTUcR49PDxcQUFBCg8P19ChQ5WUlKTKlSsrLCxMiYmJiouLc/lKeYlb4YDLErfCwejK4lY45ed4Zj8hES5veqFOf/bs2Ro8eLCk3x5i88gjj2jhwoWy2Wzq2rWrpk+f7ta0POEOXIYIdxhd2YR7rmf2ExLumf14EOfcAQAwGM65AwDMybjfG0O4AwBMysDfCke4AwDMycDhzjl3AAAMhnAHAMBgmJYHAJgT0/IAAOByYcyH2KBM2Ww2paSkKDk52e0vTwDKO/5943JEuOOS5eXlKTw8XLm5uQoLC/N1OYBH8e8blyOm5QEAMBjCHQAAgyHcAQAwGMIdl8xqtWrs2LFcbARD4t83LkdcUAcAgMHQuQMAYDCEOwAABkO4AwBgMIQ7AAAGQ7jjkkybNk116tRRYGCg2rRpo23btvm6JMBjPvvsM/Xq1Us1a9aUxWLR8uXLfV0S4BLCHaW2aNEiJSUlaezYsdq5c6eaN2+url276tixY74uDfCIgoICNW/eXNOmTfN1KYBbuBUOpdamTRu1atVKr7/+uiSpuLhY0dHRSkxM1JgxY3xcHeBZFotFy5YtU58+fXxdCvCX6NxRKmfPnlV6ero6d+7sGPPz81Pnzp2Vlpbmw8oAAIQ7SuXEiRMqKipS9erVncarV6+urKwsH1UFAJAIdwAADIdwR6lUrVpVFSpUUHZ2ttN4dna2oqKifFQVAEAi3FFKAQEBatmypdatW+cYKy4u1rp16xQXF+fDygAAFX1dAC5fSUlJGjRokGJjY9W6dWulpqaqoKBAQ4YM8XVpgEfk5+frwIEDjtcHDx7U7t27VblyZcXExPiwMuDiuBUOl+T111/XSy+9pKysLF133XV69dVX1aZNG1+XBXjExo0bdfPNN5cYHzRokObMmVP2BQEuItwBADAYzrkDAGAwhDsAAAZDuAMAYDCEOwAABkO4AwBgMIQ7AAAGQ7gDAGAwhDtwmapTp45SU1Nd3n7OnDmKiIi45ONaLBYtX778kvcDwHsId8ANFovlostzzz3n6xIBgGfLA+44evSo4+dFixbp2WefVUZGhmMsJCTE8bPdbldRUZEqVuQ/MwBli84dcENUVJRjCQ8Pl8Vicbzev3+/QkNDtXr1arVs2VJWq1WbN2/W4MGD1adPH6f9jBo1SjfddJPjdXFxsVJSUlS3bl0FBQWpefPmWrp0qVu1TZ48WU2bNlVwcLCio6M1fPhw5efnl9hu+fLlatCggQIDA9W1a1cdOnTIaf0HH3yg66+/XoGBgapXr57GjRunc+fOuVULAN8i3AEPGzNmjCZOnKh9+/apWbNmLr0nJSVF8+bN04wZM/T1119r9OjRGjBggDZt2uTycf38/PTqq6/q66+/1ty5c7V+/Xo9/vjjTtucOnVKEyZM0Lx587Rlyxbl5OTo3nvvdaz//PPPNXDgQI0cOVLffPONZs6cqTlz5mjChAku1wHA95gvBDzs+eefV5cuXVze3maz6cUXX9TatWsVFxcnSapXr542b96smTNnqmPHji7tZ9SoUY6f69SpoxdeeEHDhg3T9OnTHeOFhYV6/fXXHd/cN3fuXDVq1Ejbtm1T69atNW7cOI0ZM0aDBg1y1DF+/Hg9/vjjGjt2rMufCYBvEe6Ah8XGxrq1/YEDB3Tq1KkSfxCcPXtWLVq0cHk/a9euVUpKivbv36+8vDydO3dOZ86c0alTp1SpUiVJUsWKFdWqVSvHexo2bKiIiAjt27dPrVu31p49e7RlyxanTr2oqKjEfgCUb4Q74GHBwcFOr/38/PTnb1YuLCx0/Py/8+KrVq3SlVde6bSd1Wp16Zg//vijevbsqYceekgTJkxQ5cqVtXnzZg0dOlRnz551OZTz8/M1btw49e3bt8S6wMBAl/YBwPcId8DLqlWrpr179zqN7d69W/7+/pKkxo0by2q1KjMz0+Up+D9LT09XcXGxXnnlFfn5/XYpzeLFi0tsd+7cOe3YsUOtW7eWJGVkZCgnJ0eNGjWSJF1//fXKyMhQ/fr1S1UHgPKBcAe87JZbbtFLL72kefPmKS4uTv/+97+1d+9ex5R7aGioHn30UY0ePVrFxcVq3769cnNztWXLFoWFhTnOf19M/fr1VVhYqNdee029evXSli1bNGPGjBLb+fv7KzExUa+++qoqVqyoESNGqG3bto6wf/bZZ9WzZ0/FxMTozjvvlJ+fn/bs2aO9e/fqhRde8OwvBoDXcLU84GVdu3bVM888o8cff1ytWrXSyZMnNXDgQKdtxo8fr2eeeUYpKSlq1KiRunXrplWrVqlu3bouHaN58+aaPHmyJk2apCZNmmj+/PlKSUkpsV2lSpX0xBNPqF+/fmrXrp1CQkK0aNEip1pXrlypTz75RK1atVLbtm01ZcoU1a5d+9J+CQDKlMX+55OBAADgskbnDgCAwRDuAAAYDOEOAIDBEO4AABgM4Q4AgMEQ7gAAGAzhDgCAwRDuAAAYDOEOAIDBEO4AABgM4Q4AgMEQ7gAAGMz/AZxUmU/bn1bXAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:\t0.617391\n",
      "F1-score:\t0.551020\n",
      "Precision:\t0.586957\n",
      "Recall:\t\t0.519231\n",
      "\n",
      "Classification performance:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.70      0.67        63\n",
      "           1       0.59      0.52      0.55        52\n",
      "\n",
      "    accuracy                           0.62       115\n",
      "   macro avg       0.61      0.61      0.61       115\n",
      "weighted avg       0.61      0.62      0.61       115\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mat_xgb = confusion_matrix(y, y_pred_xgb)\n",
    "sns.heatmap(mat_xgb.T, square=True, annot=True, fmt='d', cmap='RdPu')\n",
    "plt.xlabel('True label')\n",
    "plt.ylabel('Predicted label')\n",
    "plt.title(\"XGboost\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Accuracy:\\t%f\" % accuracy_score(y, y_pred_xgb))\n",
    "print(\"F1-score:\\t%f\" % f1_score(y, y_pred_xgb))\n",
    "print(\"Precision:\\t%f\" % precision_score(y, y_pred_xgb))\n",
    "print(\"Recall:\\t\\t%f\" % recall_score(y, y_pred_xgb))\n",
    "print(\"\\nClassification performance:\\n%s\" % classification_report(y, y_pred_xgb))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94102c0c",
   "metadata": {},
   "source": [
    "### CatBoost "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "268a1eeb",
   "metadata": {},
   "source": [
    "CatBoost's main advantage: It uses a special algorithm to encode classification features that often works better than traditional One-Hot Encoding or Label Encoding. Because it is not necessary to use the encoded dataset for training catboost, the original dataset can be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "32f840b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier, Pool, cv\n",
    "import catboost as cb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f7f6a515",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Result</th>\n",
       "      <th>match_number</th>\n",
       "      <th>team</th>\n",
       "      <th>opponent</th>\n",
       "      <th>gender</th>\n",
       "      <th>season</th>\n",
       "      <th>team_score</th>\n",
       "      <th>team_wickets</th>\n",
       "      <th>winner</th>\n",
       "      <th>toss_winner</th>\n",
       "      <th>home_advantage</th>\n",
       "      <th>venue_city</th>\n",
       "      <th>choose_to_bat</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>avg_win_rate</th>\n",
       "      <th>avg_wickets_out</th>\n",
       "      <th>last_match_result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>win</td>\n",
       "      <td>5</td>\n",
       "      <td>Oval Invincibles</td>\n",
       "      <td>London Spirit</td>\n",
       "      <td>female</td>\n",
       "      <td>2021</td>\n",
       "      <td>132</td>\n",
       "      <td>7</td>\n",
       "      <td>Oval Invincibles</td>\n",
       "      <td>London Spirit</td>\n",
       "      <td>no</td>\n",
       "      <td>Lord's, London London</td>\n",
       "      <td>no</td>\n",
       "      <td>139.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>win</td>\n",
       "      <td>6</td>\n",
       "      <td>Birmingham Phoenix</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>female</td>\n",
       "      <td>2021</td>\n",
       "      <td>113</td>\n",
       "      <td>9</td>\n",
       "      <td>Birmingham Phoenix</td>\n",
       "      <td>Birmingham Phoenix</td>\n",
       "      <td>no</td>\n",
       "      <td>Old Trafford, Manchester Manchester</td>\n",
       "      <td>yes</td>\n",
       "      <td>128.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>lose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>win</td>\n",
       "      <td>7</td>\n",
       "      <td>Northern Superchargers</td>\n",
       "      <td>Trent Rockets</td>\n",
       "      <td>female</td>\n",
       "      <td>2021</td>\n",
       "      <td>149</td>\n",
       "      <td>7</td>\n",
       "      <td>Northern Superchargers</td>\n",
       "      <td>Northern Superchargers</td>\n",
       "      <td>no</td>\n",
       "      <td>Trent Bridge, Nottingham Nottingham</td>\n",
       "      <td>yes</td>\n",
       "      <td>131.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lose</td>\n",
       "      <td>8</td>\n",
       "      <td>Welsh Fire</td>\n",
       "      <td>Southern Brave</td>\n",
       "      <td>female</td>\n",
       "      <td>2021</td>\n",
       "      <td>110</td>\n",
       "      <td>7</td>\n",
       "      <td>Southern Brave</td>\n",
       "      <td>Southern Brave</td>\n",
       "      <td>yes</td>\n",
       "      <td>Sophia Gardens, Cardiff Cardiff</td>\n",
       "      <td>no</td>\n",
       "      <td>130.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>lose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>win</td>\n",
       "      <td>10</td>\n",
       "      <td>Trent Rockets</td>\n",
       "      <td>London Spirit</td>\n",
       "      <td>female</td>\n",
       "      <td>2021</td>\n",
       "      <td>151</td>\n",
       "      <td>4</td>\n",
       "      <td>Trent Rockets</td>\n",
       "      <td>Trent Rockets</td>\n",
       "      <td>no</td>\n",
       "      <td>Lord's, London London</td>\n",
       "      <td>yes</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>lose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>lose</td>\n",
       "      <td>30</td>\n",
       "      <td>London Spirit</td>\n",
       "      <td>Birmingham Phoenix</td>\n",
       "      <td>male</td>\n",
       "      <td>2022</td>\n",
       "      <td>139</td>\n",
       "      <td>6</td>\n",
       "      <td>Birmingham Phoenix</td>\n",
       "      <td>Birmingham Phoenix</td>\n",
       "      <td>yes</td>\n",
       "      <td>Lord's, London London</td>\n",
       "      <td>no</td>\n",
       "      <td>140.642857</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>6.571429</td>\n",
       "      <td>lose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>win</td>\n",
       "      <td>31</td>\n",
       "      <td>Northern Superchargers</td>\n",
       "      <td>Southern Brave</td>\n",
       "      <td>male</td>\n",
       "      <td>2022</td>\n",
       "      <td>135</td>\n",
       "      <td>8</td>\n",
       "      <td>Northern Superchargers</td>\n",
       "      <td>Southern Brave</td>\n",
       "      <td>yes</td>\n",
       "      <td>Headingley, Leeds Leeds</td>\n",
       "      <td>no</td>\n",
       "      <td>152.714286</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>lose</td>\n",
       "      <td>32</td>\n",
       "      <td>Oval Invincibles</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>male</td>\n",
       "      <td>2022</td>\n",
       "      <td>143</td>\n",
       "      <td>9</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>Oval Invincibles</td>\n",
       "      <td>no</td>\n",
       "      <td>Old Trafford, Manchester Manchester</td>\n",
       "      <td>yes</td>\n",
       "      <td>145.500000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>5.785714</td>\n",
       "      <td>win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>lose</td>\n",
       "      <td>40</td>\n",
       "      <td>London Spirit</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>male</td>\n",
       "      <td>2022</td>\n",
       "      <td>150</td>\n",
       "      <td>7</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>London Spirit</td>\n",
       "      <td>no</td>\n",
       "      <td>The Rose Bowl, Southampton Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>140.533333</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>6.533333</td>\n",
       "      <td>lose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>lose</td>\n",
       "      <td>50</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>Trent Rockets</td>\n",
       "      <td>male</td>\n",
       "      <td>2022</td>\n",
       "      <td>120</td>\n",
       "      <td>9</td>\n",
       "      <td>Trent Rockets</td>\n",
       "      <td>Manchester Originals</td>\n",
       "      <td>no</td>\n",
       "      <td>Lord's, London London</td>\n",
       "      <td>yes</td>\n",
       "      <td>148.800000</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>win</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>115 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Result  match_number                    team              opponent  \\\n",
       "0      win             5        Oval Invincibles         London Spirit   \n",
       "1      win             6      Birmingham Phoenix  Manchester Originals   \n",
       "2      win             7  Northern Superchargers         Trent Rockets   \n",
       "3     lose             8              Welsh Fire        Southern Brave   \n",
       "4      win            10           Trent Rockets         London Spirit   \n",
       "..     ...           ...                     ...                   ...   \n",
       "110   lose            30           London Spirit    Birmingham Phoenix   \n",
       "111    win            31  Northern Superchargers        Southern Brave   \n",
       "112   lose            32        Oval Invincibles  Manchester Originals   \n",
       "113   lose            40           London Spirit  Manchester Originals   \n",
       "114   lose            50    Manchester Originals         Trent Rockets   \n",
       "\n",
       "     gender  season  team_score  team_wickets                  winner  \\\n",
       "0    female    2021         132             7        Oval Invincibles   \n",
       "1    female    2021         113             9      Birmingham Phoenix   \n",
       "2    female    2021         149             7  Northern Superchargers   \n",
       "3    female    2021         110             7          Southern Brave   \n",
       "4    female    2021         151             4           Trent Rockets   \n",
       "..      ...     ...         ...           ...                     ...   \n",
       "110    male    2022         139             6      Birmingham Phoenix   \n",
       "111    male    2022         135             8  Northern Superchargers   \n",
       "112    male    2022         143             9    Manchester Originals   \n",
       "113    male    2022         150             7    Manchester Originals   \n",
       "114    male    2022         120             9           Trent Rockets   \n",
       "\n",
       "                toss_winner home_advantage  \\\n",
       "0             London Spirit             no   \n",
       "1        Birmingham Phoenix             no   \n",
       "2    Northern Superchargers             no   \n",
       "3            Southern Brave            yes   \n",
       "4             Trent Rockets             no   \n",
       "..                      ...            ...   \n",
       "110      Birmingham Phoenix            yes   \n",
       "111          Southern Brave            yes   \n",
       "112        Oval Invincibles             no   \n",
       "113           London Spirit             no   \n",
       "114    Manchester Originals             no   \n",
       "\n",
       "                                 venue_city choose_to_bat   avg_score  \\\n",
       "0                     Lord's, London London            no  139.000000   \n",
       "1       Old Trafford, Manchester Manchester           yes  128.000000   \n",
       "2       Trent Bridge, Nottingham Nottingham           yes  131.000000   \n",
       "3           Sophia Gardens, Cardiff Cardiff            no  130.000000   \n",
       "4                     Lord's, London London           yes  116.000000   \n",
       "..                                      ...           ...         ...   \n",
       "110                   Lord's, London London            no  140.642857   \n",
       "111                 Headingley, Leeds Leeds            no  152.714286   \n",
       "112     Old Trafford, Manchester Manchester           yes  145.500000   \n",
       "113  The Rose Bowl, Southampton Southampton           yes  140.533333   \n",
       "114                   Lord's, London London           yes  148.800000   \n",
       "\n",
       "     avg_win_rate  avg_wickets_out last_match_result  \n",
       "0        1.000000         5.000000               win  \n",
       "1        0.000000         6.000000              lose  \n",
       "2        1.000000         4.000000               win  \n",
       "3        0.000000         8.000000              lose  \n",
       "4        0.000000         7.000000              lose  \n",
       "..            ...              ...               ...  \n",
       "110      0.428571         6.571429              lose  \n",
       "111      0.428571         6.000000               win  \n",
       "112      0.571429         5.785714               win  \n",
       "113      0.400000         6.533333              lose  \n",
       "114      0.533333         6.000000               win  \n",
       "\n",
       "[115 rows x 17 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bat= pd.read_csv('/Users/quinne/Desktop/all_code/dissertation_dataset/halftime_dataframe_R.csv')\n",
    "df_bat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "962cd78f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 115 entries, 0 to 114\n",
      "Data columns (total 17 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   Result             115 non-null    object \n",
      " 1   match_number       115 non-null    int64  \n",
      " 2   team               115 non-null    object \n",
      " 3   opponent           115 non-null    object \n",
      " 4   gender             115 non-null    object \n",
      " 5   season             115 non-null    int64  \n",
      " 6   team_score         115 non-null    int64  \n",
      " 7   team_wickets       115 non-null    int64  \n",
      " 8   winner             115 non-null    object \n",
      " 9   toss_winner        115 non-null    object \n",
      " 10  home_advantage     115 non-null    object \n",
      " 11  venue_city         115 non-null    object \n",
      " 12  choose_to_bat      115 non-null    object \n",
      " 13  avg_score          115 non-null    float64\n",
      " 14  avg_win_rate       115 non-null    float64\n",
      " 15  avg_wickets_out    115 non-null    float64\n",
      " 16  last_match_result  115 non-null    object \n",
      "dtypes: float64(3), int64(4), object(10)\n",
      "memory usage: 15.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df_bat.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "2906a2eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_keep = ['Result', \"match_number\", \"team\", \"opponent\", \n",
    "                   \"gender\", 'season', 'home_advantage', 'venue_city',\n",
    "                   'avg_score', 'avg_win_rate', 'avg_wickets_out', 'last_match_result']\n",
    "df = df_bat[columns_to_keep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3460d0fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(115, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a55dcf94",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/1v/gbft9lh97dggng6ytnnwkd9c0000gn/T/ipykernel_49168/3400818731.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['Result'] = df['Result'].apply(lambda x: 1 if x == 'win' else 0)\n"
     ]
    }
   ],
   "source": [
    "# Transforming the 'Result' column to binary format\n",
    "df['Result'] = df['Result'].apply(lambda x: 1 if x == 'win' else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5531a81b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting the data into X (features) and y (target)\n",
    "X = df.drop(columns=['Result'])\n",
    "y = df['Result']\n",
    "\n",
    "# List of categorical features\n",
    "cat_features = [\n",
    "    'team', 'opponent', 'gender', 'home_advantage', 'venue_city', 'last_match_result'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c238e75d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "bestTest = 0.6930869783\n",
      "bestIteration = 2\n",
      "\n",
      "0:\tloss: 0.6930870\tbest: 0.6930870 (0)\ttotal: 186ms\tremaining: 14.8s\n",
      "\n",
      "bestTest = 0.688026097\n",
      "bestIteration = 21\n",
      "\n",
      "1:\tloss: 0.6880261\tbest: 0.6880261 (1)\ttotal: 305ms\tremaining: 12s\n",
      "\n",
      "bestTest = 0.6915828762\n",
      "bestIteration = 2\n",
      "\n",
      "2:\tloss: 0.6915829\tbest: 0.6880261 (1)\ttotal: 393ms\tremaining: 10.2s\n",
      "\n",
      "bestTest = 0.6932450779\n",
      "bestIteration = 2\n",
      "\n",
      "3:\tloss: 0.6932451\tbest: 0.6880261 (1)\ttotal: 500ms\tremaining: 9.63s\n",
      "\n",
      "bestTest = 0.6940503282\n",
      "bestIteration = 2\n",
      "\n",
      "4:\tloss: 0.6940503\tbest: 0.6880261 (1)\ttotal: 604ms\tremaining: 9.18s\n",
      "\n",
      "bestTest = 0.6939549389\n",
      "bestIteration = 1\n",
      "\n",
      "5:\tloss: 0.6939549\tbest: 0.6880261 (1)\ttotal: 717ms\tremaining: 8.96s\n",
      "\n",
      "bestTest = 0.6931860882\n",
      "bestIteration = 2\n",
      "\n",
      "6:\tloss: 0.6931861\tbest: 0.6880261 (1)\ttotal: 798ms\tremaining: 8.44s\n",
      "\n",
      "bestTest = 0.6937300813\n",
      "bestIteration = 2\n",
      "\n",
      "7:\tloss: 0.6937301\tbest: 0.6880261 (1)\ttotal: 941ms\tremaining: 8.59s\n",
      "\n",
      "bestTest = 0.6952326105\n",
      "bestIteration = 56\n",
      "\n",
      "8:\tloss: 0.6952326\tbest: 0.6880261 (1)\ttotal: 1.05s\tremaining: 8.42s\n",
      "\n",
      "bestTest = 0.6930487631\n",
      "bestIteration = 0\n",
      "\n",
      "9:\tloss: 0.6930488\tbest: 0.6880261 (1)\ttotal: 1.65s\tremaining: 11.7s\n",
      "\n",
      "bestTest = 0.69247039\n",
      "bestIteration = 2\n",
      "\n",
      "10:\tloss: 0.6924704\tbest: 0.6880261 (1)\ttotal: 2.29s\tremaining: 14.6s\n",
      "\n",
      "bestTest = 0.6930564318\n",
      "bestIteration = 0\n",
      "\n",
      "11:\tloss: 0.6930564\tbest: 0.6880261 (1)\ttotal: 2.85s\tremaining: 16.4s\n",
      "\n",
      "bestTest = 0.6913999322\n",
      "bestIteration = 2\n",
      "\n",
      "12:\tloss: 0.6913999\tbest: 0.6880261 (1)\ttotal: 3.41s\tremaining: 17.9s\n",
      "\n",
      "bestTest = 0.6939484332\n",
      "bestIteration = 0\n",
      "\n",
      "13:\tloss: 0.6939484\tbest: 0.6880261 (1)\ttotal: 3.99s\tremaining: 19.1s\n",
      "\n",
      "bestTest = 0.6950789152\n",
      "bestIteration = 0\n",
      "\n",
      "14:\tloss: 0.6950789\tbest: 0.6880261 (1)\ttotal: 4.53s\tremaining: 19.9s\n",
      "\n",
      "bestTest = 0.6913908145\n",
      "bestIteration = 6\n",
      "\n",
      "15:\tloss: 0.6913908\tbest: 0.6880261 (1)\ttotal: 5.14s\tremaining: 20.9s\n",
      "\n",
      "bestTest = 0.6944583123\n",
      "bestIteration = 0\n",
      "\n",
      "16:\tloss: 0.6944583\tbest: 0.6880261 (1)\ttotal: 5.74s\tremaining: 21.6s\n",
      "\n",
      "bestTest = 0.6960290673\n",
      "bestIteration = 0\n",
      "\n",
      "17:\tloss: 0.6960291\tbest: 0.6880261 (1)\ttotal: 6.33s\tremaining: 22.1s\n",
      "\n",
      "bestTest = 0.6930487631\n",
      "bestIteration = 0\n",
      "\n",
      "18:\tloss: 0.6930488\tbest: 0.6880261 (1)\ttotal: 7.42s\tremaining: 24.2s\n",
      "\n",
      "bestTest = 0.69247039\n",
      "bestIteration = 2\n",
      "\n",
      "19:\tloss: 0.6924704\tbest: 0.6880261 (1)\ttotal: 8.61s\tremaining: 26.3s\n",
      "\n",
      "bestTest = 0.6930564318\n",
      "bestIteration = 0\n",
      "\n",
      "20:\tloss: 0.6930564\tbest: 0.6880261 (1)\ttotal: 9.68s\tremaining: 27.7s\n",
      "\n",
      "bestTest = 0.6913999322\n",
      "bestIteration = 2\n",
      "\n",
      "21:\tloss: 0.6913999\tbest: 0.6880261 (1)\ttotal: 10.9s\tremaining: 29.2s\n",
      "\n",
      "bestTest = 0.6939484332\n",
      "bestIteration = 0\n",
      "\n",
      "22:\tloss: 0.6939484\tbest: 0.6880261 (1)\ttotal: 12.3s\tremaining: 31.1s\n",
      "\n",
      "bestTest = 0.6950789152\n",
      "bestIteration = 0\n",
      "\n",
      "23:\tloss: 0.6950789\tbest: 0.6880261 (1)\ttotal: 13.5s\tremaining: 32s\n",
      "\n",
      "bestTest = 0.6913908145\n",
      "bestIteration = 6\n",
      "\n",
      "24:\tloss: 0.6913908\tbest: 0.6880261 (1)\ttotal: 14.7s\tremaining: 32.8s\n",
      "\n",
      "bestTest = 0.6944583123\n",
      "bestIteration = 0\n",
      "\n",
      "25:\tloss: 0.6944583\tbest: 0.6880261 (1)\ttotal: 15.7s\tremaining: 33.1s\n",
      "\n",
      "bestTest = 0.6960290673\n",
      "bestIteration = 0\n",
      "\n",
      "26:\tloss: 0.6960291\tbest: 0.6880261 (1)\ttotal: 16.7s\tremaining: 33.4s\n",
      "\n",
      "bestTest = 0.6937983577\n",
      "bestIteration = 0\n",
      "\n",
      "27:\tloss: 0.6937984\tbest: 0.6880261 (1)\ttotal: 16.8s\tremaining: 31.9s\n",
      "\n",
      "bestTest = 0.6967319825\n",
      "bestIteration = 0\n",
      "\n",
      "28:\tloss: 0.6967320\tbest: 0.6880261 (1)\ttotal: 17.1s\tremaining: 30.6s\n",
      "\n",
      "bestTest = 0.7011354642\n",
      "bestIteration = 0\n",
      "\n",
      "29:\tloss: 0.7011355\tbest: 0.6880261 (1)\ttotal: 17.2s\tremaining: 29.3s\n",
      "\n",
      "bestTest = 0.6939982591\n",
      "bestIteration = 0\n",
      "\n",
      "30:\tloss: 0.6939983\tbest: 0.6880261 (1)\ttotal: 17.4s\tremaining: 28s\n",
      "\n",
      "bestTest = 0.6975753544\n",
      "bestIteration = 0\n",
      "\n",
      "31:\tloss: 0.6975754\tbest: 0.6880261 (1)\ttotal: 17.5s\tremaining: 26.8s\n",
      "\n",
      "bestTest = 0.7024347685\n",
      "bestIteration = 0\n",
      "\n",
      "32:\tloss: 0.7024348\tbest: 0.6880261 (1)\ttotal: 17.7s\tremaining: 25.7s\n",
      "\n",
      "bestTest = 0.6940591226\n",
      "bestIteration = 0\n",
      "\n",
      "33:\tloss: 0.6940591\tbest: 0.6880261 (1)\ttotal: 17.8s\tremaining: 24.6s\n",
      "\n",
      "bestTest = 0.6921832241\n",
      "bestIteration = 4\n",
      "\n",
      "34:\tloss: 0.6921832\tbest: 0.6880261 (1)\ttotal: 17.9s\tremaining: 23.6s\n",
      "\n",
      "bestTest = 0.7017836153\n",
      "bestIteration = 2\n",
      "\n",
      "35:\tloss: 0.7017836\tbest: 0.6880261 (1)\ttotal: 18.1s\tremaining: 22.6s\n",
      "\n",
      "bestTest = 0.6884555973\n",
      "bestIteration = 20\n",
      "\n",
      "36:\tloss: 0.6884556\tbest: 0.6880261 (1)\ttotal: 18.8s\tremaining: 22.3s\n",
      "\n",
      "bestTest = 0.6917287589\n",
      "bestIteration = 2\n",
      "\n",
      "37:\tloss: 0.6917288\tbest: 0.6880261 (1)\ttotal: 19.5s\tremaining: 22.1s\n",
      "\n",
      "bestTest = 0.6528944006\n",
      "bestIteration = 3\n",
      "\n",
      "38:\tloss: 0.6528944\tbest: 0.6528944 (38)\ttotal: 20.5s\tremaining: 22.1s\n",
      "\n",
      "bestTest = 0.6925246057\n",
      "bestIteration = 2\n",
      "\n",
      "39:\tloss: 0.6925246\tbest: 0.6528944 (38)\ttotal: 21.2s\tremaining: 21.7s\n",
      "\n",
      "bestTest = 0.6906832353\n",
      "bestIteration = 2\n",
      "\n",
      "40:\tloss: 0.6906832\tbest: 0.6528944 (38)\ttotal: 21.9s\tremaining: 21.4s\n",
      "\n",
      "bestTest = 0.6934005191\n",
      "bestIteration = 0\n",
      "\n",
      "41:\tloss: 0.6934005\tbest: 0.6528944 (38)\ttotal: 22.7s\tremaining: 21s\n",
      "\n",
      "bestTest = 0.691408487\n",
      "bestIteration = 4\n",
      "\n",
      "42:\tloss: 0.6914085\tbest: 0.6528944 (38)\ttotal: 23.4s\tremaining: 20.7s\n",
      "\n",
      "bestTest = 0.6905629876\n",
      "bestIteration = 2\n",
      "\n",
      "43:\tloss: 0.6905630\tbest: 0.6528944 (38)\ttotal: 24.1s\tremaining: 20.3s\n",
      "\n",
      "bestTest = 0.6924365974\n",
      "bestIteration = 0\n",
      "\n",
      "44:\tloss: 0.6924366\tbest: 0.6528944 (38)\ttotal: 24.9s\tremaining: 19.9s\n",
      "\n",
      "bestTest = 0.6884555973\n",
      "bestIteration = 20\n",
      "\n",
      "45:\tloss: 0.6884556\tbest: 0.6528944 (38)\ttotal: 26.4s\tremaining: 20.1s\n",
      "\n",
      "bestTest = 0.6917287589\n",
      "bestIteration = 2\n",
      "\n",
      "46:\tloss: 0.6917288\tbest: 0.6528944 (38)\ttotal: 28.2s\tremaining: 20.4s\n",
      "\n",
      "bestTest = 0.6528944006\n",
      "bestIteration = 3\n",
      "\n",
      "47:\tloss: 0.6528944\tbest: 0.6528944 (38)\ttotal: 29.7s\tremaining: 20.4s\n",
      "\n",
      "bestTest = 0.6925246057\n",
      "bestIteration = 2\n",
      "\n",
      "48:\tloss: 0.6925246\tbest: 0.6528944 (38)\ttotal: 31.2s\tremaining: 20.4s\n",
      "\n",
      "bestTest = 0.6906832353\n",
      "bestIteration = 2\n",
      "\n",
      "49:\tloss: 0.6906832\tbest: 0.6528944 (38)\ttotal: 32.7s\tremaining: 20.3s\n",
      "\n",
      "bestTest = 0.6934005191\n",
      "bestIteration = 0\n",
      "\n",
      "50:\tloss: 0.6934005\tbest: 0.6528944 (38)\ttotal: 34.4s\tremaining: 20.2s\n",
      "\n",
      "bestTest = 0.691408487\n",
      "bestIteration = 4\n",
      "\n",
      "51:\tloss: 0.6914085\tbest: 0.6528944 (38)\ttotal: 36.1s\tremaining: 20.2s\n",
      "\n",
      "bestTest = 0.6905629876\n",
      "bestIteration = 2\n",
      "\n",
      "52:\tloss: 0.6905630\tbest: 0.6528944 (38)\ttotal: 37.7s\tremaining: 19.9s\n",
      "\n",
      "bestTest = 0.6924365974\n",
      "bestIteration = 0\n",
      "\n",
      "53:\tloss: 0.6924366\tbest: 0.6528944 (38)\ttotal: 39.3s\tremaining: 19.6s\n",
      "\n",
      "bestTest = 0.6919176929\n",
      "bestIteration = 1\n",
      "\n",
      "54:\tloss: 0.6919177\tbest: 0.6528944 (38)\ttotal: 39.4s\tremaining: 18.6s\n",
      "\n",
      "bestTest = 0.7005593846\n",
      "bestIteration = 0\n",
      "\n",
      "55:\tloss: 0.7005594\tbest: 0.6528944 (38)\ttotal: 39.6s\tremaining: 17.7s\n",
      "\n",
      "bestTest = 0.7084231846\n",
      "bestIteration = 0\n",
      "\n",
      "56:\tloss: 0.7084232\tbest: 0.6528944 (38)\ttotal: 39.7s\tremaining: 16.7s\n",
      "\n",
      "bestTest = 0.6925393103\n",
      "bestIteration = 1\n",
      "\n",
      "57:\tloss: 0.6925393\tbest: 0.6528944 (38)\ttotal: 39.9s\tremaining: 15.8s\n",
      "\n",
      "bestTest = 0.6992134984\n",
      "bestIteration = 0\n",
      "\n",
      "58:\tloss: 0.6992135\tbest: 0.6528944 (38)\ttotal: 40s\tremaining: 14.9s\n",
      "\n",
      "bestTest = 0.7055751421\n",
      "bestIteration = 0\n",
      "\n",
      "59:\tloss: 0.7055751\tbest: 0.6528944 (38)\ttotal: 40.2s\tremaining: 14.1s\n",
      "\n",
      "bestTest = 0.6927833859\n",
      "bestIteration = 1\n",
      "\n",
      "60:\tloss: 0.6927834\tbest: 0.6528944 (38)\ttotal: 40.3s\tremaining: 13.2s\n",
      "\n",
      "bestTest = 0.6984752718\n",
      "bestIteration = 0\n",
      "\n",
      "61:\tloss: 0.6984753\tbest: 0.6528944 (38)\ttotal: 40.4s\tremaining: 12.4s\n",
      "\n",
      "bestTest = 0.7040271784\n",
      "bestIteration = 0\n",
      "\n",
      "62:\tloss: 0.7040272\tbest: 0.6528944 (38)\ttotal: 40.6s\tremaining: 11.6s\n",
      "\n",
      "bestTest = 0.6911920454\n",
      "bestIteration = 18\n",
      "\n",
      "63:\tloss: 0.6911920\tbest: 0.6528944 (38)\ttotal: 41.5s\tremaining: 11s\n",
      "\n",
      "bestTest = 0.6964307156\n",
      "bestIteration = 0\n",
      "\n",
      "64:\tloss: 0.6964307\tbest: 0.6528944 (38)\ttotal: 42.5s\tremaining: 10.5s\n",
      "\n",
      "bestTest = 0.6843586036\n",
      "bestIteration = 13\n",
      "\n",
      "65:\tloss: 0.6843586\tbest: 0.6528944 (38)\ttotal: 43.6s\tremaining: 9.91s\n",
      "\n",
      "bestTest = 0.6932852177\n",
      "bestIteration = 0\n",
      "\n",
      "66:\tloss: 0.6932852\tbest: 0.6528944 (38)\ttotal: 45.2s\tremaining: 9.45s\n",
      "\n",
      "bestTest = 0.6944397076\n",
      "bestIteration = 0\n",
      "\n",
      "67:\tloss: 0.6944397\tbest: 0.6528944 (38)\ttotal: 46.6s\tremaining: 8.91s\n",
      "\n",
      "bestTest = 0.6972341119\n",
      "bestIteration = 0\n",
      "\n",
      "68:\tloss: 0.6972341\tbest: 0.6528944 (38)\ttotal: 48.3s\tremaining: 8.39s\n",
      "\n",
      "bestTest = 0.6931691902\n",
      "bestIteration = 5\n",
      "\n",
      "69:\tloss: 0.6931692\tbest: 0.6528944 (38)\ttotal: 49.8s\tremaining: 7.82s\n",
      "\n",
      "bestTest = 0.6937966082\n",
      "bestIteration = 0\n",
      "\n",
      "70:\tloss: 0.6937966\tbest: 0.6528944 (38)\ttotal: 51s\tremaining: 7.19s\n",
      "\n",
      "bestTest = 0.691812311\n",
      "bestIteration = 15\n",
      "\n",
      "71:\tloss: 0.6918123\tbest: 0.6528944 (38)\ttotal: 52s\tremaining: 6.51s\n",
      "\n",
      "bestTest = 0.6911920454\n",
      "bestIteration = 18\n",
      "\n",
      "72:\tloss: 0.6911920\tbest: 0.6528944 (38)\ttotal: 53.9s\tremaining: 5.91s\n",
      "\n",
      "bestTest = 0.6964307156\n",
      "bestIteration = 0\n",
      "\n",
      "73:\tloss: 0.6964307\tbest: 0.6528944 (38)\ttotal: 55.9s\tremaining: 5.29s\n",
      "\n",
      "bestTest = 0.6843586036\n",
      "bestIteration = 13\n",
      "\n",
      "74:\tloss: 0.6843586\tbest: 0.6528944 (38)\ttotal: 57.8s\tremaining: 4.62s\n",
      "\n",
      "bestTest = 0.6932852177\n",
      "bestIteration = 0\n",
      "\n",
      "75:\tloss: 0.6932852\tbest: 0.6528944 (38)\ttotal: 60s\tremaining: 3.95s\n",
      "\n",
      "bestTest = 0.6944397076\n",
      "bestIteration = 0\n",
      "\n",
      "76:\tloss: 0.6944397\tbest: 0.6528944 (38)\ttotal: 1m 1s\tremaining: 3.22s\n",
      "\n",
      "bestTest = 0.6972341119\n",
      "bestIteration = 0\n",
      "\n",
      "77:\tloss: 0.6972341\tbest: 0.6528944 (38)\ttotal: 1m 3s\tremaining: 2.46s\n",
      "\n",
      "bestTest = 0.6931691902\n",
      "bestIteration = 5\n",
      "\n",
      "78:\tloss: 0.6931692\tbest: 0.6528944 (38)\ttotal: 1m 5s\tremaining: 1.66s\n",
      "\n",
      "bestTest = 0.6937966082\n",
      "bestIteration = 0\n",
      "\n",
      "79:\tloss: 0.6937966\tbest: 0.6528944 (38)\ttotal: 1m 7s\tremaining: 845ms\n",
      "\n",
      "bestTest = 0.691812311\n",
      "bestIteration = 15\n",
      "\n",
      "80:\tloss: 0.6918123\tbest: 0.6528944 (38)\ttotal: 1m 9s\tremaining: 0us\n",
      "Estimating final quality...\n",
      "Training on fold [0/10]\n",
      "\n",
      "bestTest = 0.5197941884\n",
      "bestIteration = 50\n",
      "\n",
      "Training on fold [1/10]\n",
      "\n",
      "bestTest = 0.6427248766\n",
      "bestIteration = 9\n",
      "\n",
      "Training on fold [2/10]\n",
      "\n",
      "bestTest = 0.6740699934\n",
      "bestIteration = 0\n",
      "\n",
      "Training on fold [3/10]\n",
      "\n",
      "bestTest = 0.6007810316\n",
      "bestIteration = 72\n",
      "\n",
      "Training on fold [4/10]\n",
      "\n",
      "bestTest = 0.7093777261\n",
      "bestIteration = 4\n",
      "\n",
      "Training on fold [5/10]\n",
      "\n",
      "bestTest = 0.5916537441\n",
      "bestIteration = 135\n",
      "\n",
      "Training on fold [6/10]\n",
      "\n",
      "bestTest = 0.6644488597\n",
      "bestIteration = 6\n",
      "\n",
      "Training on fold [7/10]\n",
      "\n",
      "bestTest = 0.6232083785\n",
      "bestIteration = 79\n",
      "\n",
      "Training on fold [8/10]\n",
      "\n",
      "bestTest = 0.5929521619\n",
      "bestIteration = 68\n",
      "\n",
      "Training on fold [9/10]\n",
      "\n",
      "bestTest = 0.7223540954\n",
      "bestIteration = 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Initialize a base model\n",
    "model = CatBoostClassifier(cat_features=cat_features, verbose=0)\n",
    "\n",
    "# Parameter grid\n",
    "grid = {\n",
    "    'learning_rate': [0.01, 0.05, 0.1],\n",
    "    'iterations': [100, 500, 1000],\n",
    "    'depth': [3, 5, 7],\n",
    "    'l2_leaf_reg': [1, 3, 5]\n",
    "}\n",
    "\n",
    "# Grid search\n",
    "grid_search_result = model.grid_search(grid, \n",
    "                                       X=Pool(X, label=y, cat_features=cat_features), \n",
    "                                       cv=10,  \n",
    "                                       plot=False)\n",
    "\n",
    "# Best parameters from grid search\n",
    "best_params = grid_search_result['params']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "dc4625c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for CatBoost:  {'depth': 5, 'l2_leaf_reg': 1, 'iterations': 500, 'learning_rate': 0.1}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best parameters for CatBoost: \", best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "971b6baa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x29e3373a0>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Best parameters for CatBoost:  {'depth': 5, 'l2_leaf_reg': 1, 'iterations': 500, 'learning_rate': 0.1}\n",
    "\n",
    "# Initialize the CatBoost model with the best parameters\n",
    "model = CatBoostClassifier(\n",
    "    depth=5, \n",
    "    l2_leaf_reg=1, \n",
    "    iterations=500, \n",
    "    learning_rate=0.1,\n",
    "    cat_features=cat_features,\n",
    "    verbose=0  \n",
    ")\n",
    "\n",
    "# Train the model on the entire dataset\n",
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "ae012cb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opponent: 17.853987415406856\n",
      "avg_wickets_out: 17.776588058339183\n",
      "avg_win_rate: 14.413931397684264\n",
      "venue_city: 12.681705747461843\n",
      "team: 12.244530142956396\n",
      "avg_score: 11.132025881234739\n",
      "match_number: 8.612799357587768\n",
      "home_advantage: 2.1303112643920166\n",
      "last_match_result: 1.6174817839278959\n",
      "gender: 0.7733074421379836\n",
      "season: 0.7633315088710431\n"
     ]
    }
   ],
   "source": [
    "feature_importances = model.get_feature_importance()\n",
    "feature_names = X.columns.tolist()\n",
    "features = sorted(zip(feature_importances, feature_names), reverse=True)\n",
    "for importance, name in features:\n",
    "    print(f\"{name}: {importance}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "3492bb44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predicted Probabilities for CatBoost:\n",
      "[5.30821119e-01 9.90198143e-01 8.96523455e-01 5.70900785e-01\n",
      " 8.42460865e-01 1.07165807e-02 9.66864931e-01 2.81015267e-01\n",
      " 4.42639626e-01 7.63238143e-02 8.61152583e-03 7.27750373e-02\n",
      " 7.02036720e-02 9.72015700e-01 5.21781272e-01 1.67995043e-02\n",
      " 5.32299120e-01 9.62548061e-01 9.98758252e-01 5.29480096e-01\n",
      " 9.88192766e-02 3.88980725e-04 1.22805217e-01 8.47843570e-01\n",
      " 3.99444890e-02 2.60487769e-01 8.00472520e-01 8.76183607e-02\n",
      " 3.77574013e-02 9.88338510e-01 2.02515964e-02 4.79287327e-01\n",
      " 2.48700712e-03 5.50481727e-01 4.82150043e-01 4.10422022e-01\n",
      " 8.35270132e-01 1.52535619e-01 9.47657736e-01 5.69036775e-02\n",
      " 8.11789630e-01 4.59919315e-03 9.87252903e-01 9.23411412e-01\n",
      " 2.13596578e-02 2.98337167e-02 7.20632877e-01 6.64932682e-02\n",
      " 1.88593141e-01 1.46726010e-01 4.20736413e-03 4.29970148e-03\n",
      " 7.71317456e-02 2.77910119e-01 3.44051672e-02 9.72929009e-01\n",
      " 7.57069805e-01 3.07924205e-01 3.88336058e-02 3.05030644e-01\n",
      " 9.11478710e-01 4.12040289e-01 4.96100466e-01 1.43629508e-01\n",
      " 8.76261392e-01 2.56083778e-02 2.87725275e-01 9.90564408e-01\n",
      " 9.96991662e-01 8.73125896e-02 6.82226665e-01 2.35857118e-03\n",
      " 3.20521877e-03 7.69429296e-01 6.45056079e-01 4.79077382e-01\n",
      " 4.32492824e-02 1.32763380e-01 2.83031571e-01 9.12337599e-01\n",
      " 9.33720072e-01 9.90230926e-01 6.70064268e-02 9.81458055e-01\n",
      " 9.95856532e-01 8.58188247e-01 3.11622832e-01 3.08766239e-01\n",
      " 9.14128153e-01 5.67647414e-02 8.41335787e-01 9.37374793e-02\n",
      " 9.05421895e-01 9.95608251e-01 9.62506719e-01 9.51917311e-01\n",
      " 2.69394797e-01 5.56965501e-01 2.56258346e-01 7.47589417e-01\n",
      " 6.45348238e-01 9.84395966e-01 2.25960142e-03 8.75169871e-01\n",
      " 4.74596515e-01 7.25317686e-01 1.91311749e-01 1.56111964e-02\n",
      " 3.35741176e-01 9.95091687e-01 3.04767673e-01 4.18071311e-02\n",
      " 7.49934480e-01 2.06972818e-01 7.26584745e-02]\n"
     ]
    }
   ],
   "source": [
    "# Train# 10-fold cross-validation\n",
    "cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=5)\n",
    "y_prob_catboost = np.zeros_like(y, dtype=float)\n",
    "\n",
    "for train_idx, test_idx in cv.split(X, y):\n",
    "    X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n",
    "    y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n",
    "    \n",
    "    model.fit(X_train, y_train)\n",
    "    y_prob_catboost[test_idx] = model.predict_proba(X_test)[:, 1]\n",
    "\n",
    "y_pred_catboost = (y_prob_catboost > 0.5).astype(int)\n",
    "\n",
    "print(\"\\nPredicted Probabilities for CatBoost:\")\n",
    "print(y_prob_catboost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "287f36a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0,\n",
       "       0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1,\n",
       "       0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0,\n",
       "       0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0,\n",
       "       1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1,\n",
       "       0, 0, 1, 0, 0])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8d7704e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAHHCAYAAACiDxGKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA640lEQVR4nO3de5yN5f7/8fcazJoxY8Z2GIfNGIwwQk7VULSdz5Q2HeQQhYZiKtvsnZA0RA6lRpFDB0XKIZIcotiTUyiqcUgoxiTMmGEGs+7fH31bv71C1hr3mjVzr9fz8bgfj7Wu+1rX/Vk19Vmf67oPNsMwDAEAAMsI8HUAAADAXCR3AAAshuQOAIDFkNwBALAYkjsAABZDcgcAwGJI7gAAWAzJHQAAiyG5AwBgMSR3oACaP3++bDabduzY4etQABRCJHfAQ4cOHdKgQYNUrVo1BQUFKSwsTM2aNdOMGTN04cIFj8Z67bXXNH/+fO8E6kWFNW7AXxT1dQBAYbJq1Sr985//lN1uV58+fXTzzTfr4sWL2rx5s55++mnt27dPb7zxhtvjvfbaaypTpoz69evnvaC9oLDGDfgLkjvgpsOHD+u+++5TlSpVtGHDBlWoUMG5Ly4uTgcPHtSqVat8GCEA/I5pecBNL774ojIzM/Xmm2+6JPY/REdH64knnpAkzZs3Ty1btlRERITsdrtiYmKUlJTk0j8qKkr79u3Tpk2bZLPZZLPZdNddd7n0OX/+vAYNGqTSpUsrLCxMffr00ZkzZ6449muvvaY6derIbrerYsWKiouL09mzZ6/o98EHH6hRo0YKDg5WmTJl1Lt3b/3yyy8ufVJTU9W/f39VqlRJdrtdFSpUULdu3fTTTz+5HTcA36JyB9z08ccfq1q1amratOl1+yYlJalOnTrq2rWrihYtqo8//liPPfaYHA6H4uLiJEnTp0/XsGHDFBoaqv/85z+SpHLlyrmMM3ToUJUsWVJjx45VSkqKkpKSdOTIEW3cuFE2m02SNHbsWI0bN06tW7fWkCFDnP22b9+uLVu2qFixYpJ+P0mvf//+atKkiRITE3Xy5EnNmDFDW7Zs0a5du1SyZElJUo8ePbRv3z4NGzZMUVFRSktL09q1a3X06FFFRUW5FTcAHzMAXFd6erohyejWrZtb/c+fP39FW7t27Yxq1aq5tNWpU8do0aLFFX3nzZtnSDIaNWpkXLx40dn+4osvGpKM5cuXG4ZhGGlpaUZgYKDRtm1bIzc319lv5syZhiRj7ty5hmEYxsWLF42IiAjj5ptvNi5cuODst3LlSkOS8eyzzxqGYRhnzpwxJBmTJ0/+y+93rbgBFAxMywNuyMjIkCSVKFHCrf7BwcHO1+np6Tp16pRatGihH3/8Uenp6W4f99FHH3VW3pI0ZMgQFS1aVJ988okkad26dbp48aKGDx+ugID//5/zI488orCwMOc5ADt27FBaWpoee+wxBQUFOft16tRJtWrVcvYLDg5WYGCgNm7ceNXpfwCFA8kdcENYWJgk6dy5c27137Jli1q3bq2QkBCVLFlSZcuW1b///W9J8ii516hRw+V9aGioKlSo4Fz/PnLkiCSpZs2aLv0CAwNVrVo15/5r9ZOkWrVqOffb7XZNmjRJq1evVrly5dS8eXO9+OKLSk1NdTtmAL5HcgfcEBYWpooVK2rv3r3X7Xvo0CG1atVKp06d0tSpU7Vq1SqtXbtWI0aMkCQ5HA5vh3tDhg8frv379ysxMVFBQUEaPXq0ateurV27dvk6NABuIrkDburcubMOHTqk5OTkv+z38ccfKycnRytWrNCgQYPUsWNHtW7d2mWq/g9/nBR3LQcOHHB5n5mZqRMnTigqKkqSVKVKFUlSSkqKS7+LFy/q8OHDzv3X6vdH2x/7/1C9enU9+eST+uyzz7R3715dvHhRL730kttxA/AtkjvgppEjRyokJEQDBw7UyZMnr9h/6NAhzZgxQ0WKFJEkGYbh3Jeenq558+Zd8ZmQkJCrXrL2hzfeeEOXLl1yvk9KStLly5fVoUMHSVLr1q0VGBiol19+2eV4b775ptLT09WpUydJUuPGjRUREaFZs2YpJyfH2W/16tX6/vvvnf3Onz+v7OxslxiqV6+uEiVKuHzuenED8C0uhQPcVL16dS1cuFC9evVS7dq1Xe5Q99///lcffPCB+vXrp/j4eAUGBqpLly4aNGiQMjMzNXv2bEVEROjEiRMuYzZq1EhJSUl6/vnnFR0drYiICLVs2dK5/+LFi2rVqpV69uyplJQUvfbaa7rjjjvUtWtXSVLZsmWVkJCgcePGqX379uratauzX5MmTdS7d29JUrFixTRp0iT1799fLVq00P333++8FC4qKsq5ZLB//37n8WJiYlS0aFEtXbpUJ0+e1H333ed23AB8zNen6wOFzf79+41HHnnEiIqKMgIDA40SJUoYzZo1M1555RUjOzvbMAzDWLFihVGvXj0jKCjIiIqKMiZNmmTMnTvXkGQcPnzYOVZqaqrRqVMno0SJEoYk5+Vlf1wKt2nTJuPRRx81/va3vxmhoaHGgw8+aPz2229XxDRz5kyjVq1aRrFixYxy5coZQ4YMMc6cOXNFv0WLFhkNGjQw7Ha7UapUKePBBx80fv75Z+f+U6dOGXFxcUatWrWMkJAQIzw83LjtttuMxYsXu4xzrbgBFAw2w/ifuTwAAFDoseYOAIDFkNwBALAYkjsAABZDcgcAIJ8kJSWpXr16CgsLU1hYmGJjY7V69Wrn/tTUVD300EMqX768QkJC1LBhQ3344YceH4fkDgBAPqlUqZImTpyonTt3aseOHWrZsqW6deumffv2SZL69OmjlJQUrVixQt9++63uuece9ezZ0+M7RHK2PAAAPlSqVClNnjxZAwYMUGhoqJKSkvTQQw8595cuXVqTJk3SwIED3R6Tyh0AgBuQk5OjjIwMl+1/7+h4Lbm5uXr//feVlZWl2NhYSVLTpk21aNEinT59Wg6HQ++//76ys7N11113eRSTJe9Q19EW5+sQAK9adfIZX4cAeJUtooLXj2FWrrh1TFmNGzfOpW3MmDEaO3bsVft/++23io2NVXZ2tkJDQ7V06VLFxMRIkhYvXqxevXqpdOnSKlq0qIoXL66lS5cqOjrao5gsmdwBAMgvCQkJio+Pd2mz2+3X7F+zZk3t3r1b6enpWrJkifr27atNmzYpJiZGo0eP1tmzZ7Vu3TqVKVNGy5YtU8+ePfXll1+qbt26bsdkyTV3KndYHZU7rC4/KvfOJuWKlcarN/T51q1bq3r16ho5cqSio6O1d+9e1alTx2V/dHS0Zs2a5faYVO4AAL9kU8F4dLHD4VBOTo7Onz8vSQoIcD0drkiRInI4HB6NSXIHAPglX5xRnpCQoA4dOigyMlLnzp3TwoULtXHjRq1Zs0a1atVSdHS0Bg0apClTpqh06dJatmyZ1q5dq5UrV3p0HJI7AAD5JC0tTX369NGJEycUHh6uevXqac2aNWrTpo0k6ZNPPtGoUaPUpUsXZWZmKjo6WgsWLFDHjh09Og7JHQDgl3wxLf/mm2/+5f4aNWrk6Y50f0ZyBwD4JSvf6MXK3w0AAL9E5Q4A8EsF41x57yC5AwD8UoCF0zvT8gAAWAyVOwDAL1m3bie5AwD8FNPyAACg0KByBwD4JevW7SR3AICfsvLUNckdAOCXCspT4bzByj9cAADwS1TuAAC/ZOXqluQOAPBLXAoHAAAKDSp3AIBfsm7dTnIHAPipAJt10zvT8gAAWAyVOwDAL1m3bie5AwD8lJWnrq383QAA8EtU7gAAv2Tl28+S3AEAfsnKU9ckdwCAX7Jy5W7lHy4AAPglKncAgF+ycnVLcgcA+CXrTspb+4cLAAB+icodAOCXrPzIV5I7AMAvWTe1My0PAIDlULkDAPwS0/IAAFiMlaeurfzdAADwS1TuAAC/ZN1JeZI7AMBPseYOAIDFWDe1s+YOAIDlULkDAPwS0/IAAFiMdVM70/IAAFgOlTsAwC9ZuboluQMA/BLT8gAAoNCgcgcA+CXOlgcAwGKsPHVt5e8GAECBkpSUpHr16iksLExhYWGKjY3V6tWrXfokJyerZcuWCgkJUVhYmJo3b64LFy54dBySOwDAL9lM2jxRqVIlTZw4UTt37tSOHTvUsmVLdevWTfv27ZP0e2Jv37692rZtq23btmn79u0aOnSoAgI8S9dMywMA/JIvqtsuXbq4vJ8wYYKSkpL01VdfqU6dOhoxYoQef/xxjRo1ytmnZs2aHh+Hyh0A4JcCZDNly8nJUUZGhsuWk5Nz3ePn5ubq/fffV1ZWlmJjY5WWlqatW7cqIiJCTZs2Vbly5dSiRQtt3rw5D98NAADkWWJiosLDw122xMTEa/b/9ttvFRoaKrvdrsGDB2vp0qWKiYnRjz/+KEkaO3asHnnkEX366adq2LChWrVqpQMHDngUE9PyAAC/ZNaFcAkJCYqPj3dps9vt1+xfs2ZN7d69W+np6VqyZIn69u2rTZs2yeFwSJIGDRqk/v37S5IaNGig9evXa+7cuX/5g+HPSO4AAL9k1tS13W7/y2T+Z4GBgYqOjpYkNWrUSNu3b9eMGTOc6+wxMTEu/WvXrq2jR496FBPT8gAA+JDD4VBOTo6ioqJUsWJFpaSkuOzfv3+/qlSp4tGYVO4AAL/ki+o2ISFBHTp0UGRkpM6dO6eFCxdq48aNWrNmjWw2m55++mmNGTNG9evX1y233KIFCxbohx9+0JIlSzw6DskdAOCXfHHz2bS0NPXp00cnTpxQeHi46tWrpzVr1qhNmzaSpOHDhys7O1sjRozQ6dOnVb9+fa1du1bVq1f36Dg2wzAMb3wBX+poi/N1CIBXrTr5jK9DALzKFlHB68d4KeDfpozzpOMFU8YxE5U7AMAv8eAYAAAsxrqpnbPlAQCwHCp3AIBfsnJ1S3IHAPglkjsAABZjs/Cqu5V/uAAA4Jeo3AEAfsnK1S3JHQDgl6yc3K383QAA8EtU7gAAv2Td0+lI7gAAP2XlqWsrfzcAAPwSlTsAwC9Z+Tp3kjsAwC9Zeerayt8NAAC/ROUOAPBLVq5uSe4AAL9k3RV3kjsAwE8FWDi9W3lWAgAAv0TlDgDwS1aubknuAAC/ZN1JeWv/cAEAwC9RucMtHQffqU5D7lS5qFKSpCP7Tui951Zrx6ffSZL+Vi5MAybfrVva1FLxEnb9nHJSiyas0ZaPdvswasB9r7/9rtZ+8YV+PHJUQXa7GtxcR08OGaRqkZEu/Xbt3afps+fom+++V0BAgGrXiNaclyYryG73UeTIKytXtyR3uOXUz2c0b9RyHT+QJpvNplZ9b9Po5YM0rMFEHf3uhJ58q49CSgbrua6zlHEqU3c90ESjFg/QE40n6cfdP/s6fOC6tu/erQfu7q66tWspNzdX016fo4HxT2vl2/NVPDhY0u+J/ZGnRurR3g/omeGPq0iRIko5eEgBNitP8FoXyR1+b9vKvS7v33rmY3Uacqdq3R6lo9+dUO2m1fTqkPe1f/sRSdL7Ez5V9xH/UI1GkSR3FApzXprs8j7x36PUtGt37UvZrya31JckTXxlph669x492vtBZ78/V/ZAQeDT5H7q1CnNnTtXycnJSk1NlSSVL19eTZs2Vb9+/VS2bFlfhodrCAiw6Y5/NlRQSKC+Tz4sSfr+vz+qea+G2rZqr7LOXtCdPRsqMKiYvtl4wMfRAnlzLitTkhQeVkKS9NuZM9rz3ffq3KaN7hsSp2O/HFfVyEiNeHSAGtWr58tQkUc8OMYLtm/frnbt2ql48eJq3bq1brrpJknSyZMn9fLLL2vixIlas2aNGjdu7KsQ8SdRN1fUS8lPKTCoqC5k5mj83bN17Pvff5Ql9nxToxY9rMWnJ+vypVzlnL+o8Xe/oROHfvVx1IDnHA6HXnh5phrWvVk3VasmSTp2/Lgkaea8+Rr52BDVrhGt5Z+uUb/hT+rjBfMUVbmSL0NGHjAt7wXDhg3TP//5T82aNUu2P61XGYahwYMHa9iwYUpOTv7LcXJycpSTk+PSlqtcFVER02P2dz+nnNTQWxIVEh6kO+5toCcXPKSRLabr2Pepemh8Z4WWLK6EVi8r41SmYrvXV8LiARp55zT9tPe4r0MHPPLc1Ok6cPiwFr76irPN4TAkSb26dlGPTh0kSTE31VDyzq/14apP9OTgR30SK3A1PvvhsmfPHo0YMeKKxC5JNptNI0aM0O7du687TmJiosLDw122H7XTCxHj8qVcnTj0qw5+fUzz/71CP+75Rd2e+IfKVyujrsPu0rSH39GeDSk6/M0vWvjcJzqw46g6xzX3ddiAR56bNl0bk5P11ozpKh8R4WyPKF1akhQdVcWlf/WoKjqRlpavMcIcNpO2gshnyb18+fLatm3bNfdv27ZN5cqVu+44CQkJSk9Pd9mqqZGZoeIaAgJsKmYvqqDigZIkw+Fw2e/IdcgWUFD/9AFXhmHouWnTte6LzZo/fZoqVazgsv/vFcorokwZHT52zKX9p2PHVNGN/1eh4AkIsJmyFUQ+m5Z/6qmn9Oijj2rnzp1q1aqVM5GfPHlS69ev1+zZszVlypTrjmO322X/0/WlTMmbr98LXbVj9XdKO3paxUsE6a4HGqvuXTU0ut2rOvZDqn45kKZhrz+gOU99pIzfshTbvb4atKmlsZ1n+Tp0wC3PTZ2ulevW6dUXJiikeLB+/e03SVKJ0FAF2e2y2WwacH8vvTJ3vmpWr67aNaK17NM1+vHIUc0YP87H0SMvrFx82AzDMHx18EWLFmnatGnauXOncnNzJUlFihRRo0aNFB8fr549e+Zp3I62ODPDhKQn5jyoW1rVVKkKYcpKz9bhb37RkklrtWvdD5KkitFl1X9iN8XcUV3BoXYdP/irPpqyXhveufbsDPJu1clnfB2C5dS6866rtr+Q8C/d07GD8/0b77yrhUuXKT3jnGpGV9fTQwZxtrwX2CIqXL/TDVoRaM6Psq4Xx5gyjpl8mtz/cOnSJZ06dUqSVKZMGRUrVuyGxiO5w+pI7rC6/EjuK+3PmTJO55xnTRnHTAXiJjbFihVThQre/xcJAMAfbBa+Fs7CXw0AAP9UICp3AADym5WfCUByBwD4JSufLc+0PAAAFkPlDgDwSwX1BjRmILkDAPyShZfcmZYHAMBqqNwBAH6JaXkAACzmak8ltQqSOwDAL1m5cmfNHQAAiyG5AwD8ki3AZsrmiaSkJNWrV09hYWEKCwtTbGysVq9efUU/wzDUoUMH2Ww2LVu2zOPvxrQ8AMAv+WJWvlKlSpo4caJq1KghwzC0YMECdevWTbt27VKdOnWc/aZPn35D5wSQ3AEAyCddunRxeT9hwgQlJSXpq6++cib33bt366WXXtKOHTvy/MRUkjsAwC+ZdW/5nJwc5eTkuLTZ7XbZ7fa//Fxubq4++OADZWVlKTY2VpJ0/vx5PfDAA3r11VdVvnz5PMfEmjsAwC/ZbDZTtsTERIWHh7tsiYmJ1zzut99+q9DQUNntdg0ePFhLly5VTEyMJGnEiBFq2rSpunXrdkPfjcodAIAbkJCQoPj4eJe2v6raa9asqd27dys9PV1LlixR3759tWnTJh08eFAbNmzQrl27bjgmkjsAwC+ZdZ27O1Pw/yswMFDR0dGSpEaNGmn79u2aMWOGgoODdejQIZUsWdKlf48ePXTnnXdq48aNbh+D5A4A8EsF5XnuDodDOTk5GjdunAYOHOiyr27dupo2bdoVJ+JdD8kdAIB8kpCQoA4dOigyMlLnzp3TwoULtXHjRq1Zs0bly5e/6kl0kZGRqlq1qkfHIbkDAPySLwr3tLQ09enTRydOnFB4eLjq1aunNWvWqE2bNqYeh+QOAPBLvpiWf/PNNz3qbxhGno5DcgcA+KUACz8VjuvcAQCwGCp3AIBfKihny3sDyR0A4JcCLDx3beGvBgCAf6JyBwD4pRt5pGpBR3IHAPgls24/WxAxLQ8AgMVQuQMA/BLT8gAAWAzT8gAAoNCgcgcA+CWbhctbkjsAwC9Z+d7yJHcAgF/y+9vPfvPNN24PWK9evTwHAwAAbpxbyf2WW26RzWa75nNl/9hns9mUm5traoAAAHiD30/LHz582NtxAACQr/x+Wr5KlSrejgMAAJgkTxcCvP3222rWrJkqVqyoI0eOSJKmT5+u5cuXmxocAADeYgswZyuIPA4rKSlJ8fHx6tixo86ePetcYy9ZsqSmT59udnwAAHiFzWbOVhB5nNxfeeUVzZ49W//5z39UpEgRZ3vjxo317bffmhocAADwnMfXuR8+fFgNGjS4ot1utysrK8uUoAAA8DYrn1DnceVetWpV7d69+4r2Tz/9VLVr1zYjJgAAvM7Ka+4eV+7x8fGKi4tTdna2DMPQtm3b9N577ykxMVFz5szxRowAAJiuoK6Xm8Hj5D5w4EAFBwfrmWee0fnz5/XAAw+oYsWKmjFjhu677z5vxAgAADyQp3vLP/jgg3rwwQd1/vx5ZWZmKiIiwuy4AADwLguvuef5wTFpaWlKSUmR9PvtZ8uWLWtaUAAAeJuVp+U9PhXg3Llzeuihh1SxYkW1aNFCLVq0UMWKFdW7d2+lp6d7I0YAAOABj5P7wIEDtXXrVq1atUpnz57V2bNntXLlSu3YsUODBg3yRowAAJiOs+X/x8qVK7VmzRrdcccdzrZ27dpp9uzZat++vanBAQDgLVzn/j9Kly6t8PDwK9rDw8P1t7/9zZSgAABA3nmc3J955hnFx8crNTXV2Zaamqqnn35ao0ePNjU4AAC8xcr3lndrWr5Bgway/c83OHDggCIjIxUZGSlJOnr0qOx2u3799VfW3QEAhUMBXS83g1vJvXv37l4OAwAAmMWt5D5mzBhvxwEAQL6y8gl1eb6JDQAAhVlBXS83g8fJPTc3V9OmTdPixYt19OhRXbx40WX/6dOnTQsOAABvKajXqJvB4682btw4TZ06Vb169VJ6erri4+N1zz33KCAgQGPHjvVCiAAAwBMeJ/d3331Xs2fP1pNPPqmiRYvq/vvv15w5c/Tss8/qq6++8kaMAACYzmazmbIVRB4n99TUVNWtW1eSFBoa6ryffOfOnbVq1SpzowMAwEusfPtZj8OqVKmSTpw4IUmqXr26PvvsM0nS9u3bZbfbzY0OAAB4zOPkfvfdd2v9+vWSpGHDhmn06NGqUaOG+vTpo4cfftj0AAEA8IoAk7YCyOOz5SdOnOh83atXL0VGRio5OVk1atRQly5dTA0OAABvKaDL5aa44evcY2NjFRsba0YsAADABG4l9xUrVrg9YNeuXfMcDAAA+cXv71Dn7r3lbTabcnNzbyQeAADyhS/OdE9KSlJSUpJ++uknSVKdOnX07LPPqkOHDjp9+rTGjBmjzz77TEePHlXZsmXVvXt3jR8//qqPWv8rbiV3h8Ph8RcAAACuKlWqpIkTJ6pGjRoyDEMLFixQt27dtGvXLhmGoePHj2vKlCmKiYnRkSNHNHjwYB0/flxLlizx6Dg2wzAML30Hn+loi/N1CIBXrTr5jK9DALzKFlHB68c403OuKeP8bfGNXSlWqlQpTZ48WQMGDLhi3wcffKDevXsrKytLRYu6f5ocD44BAPgnH6+55+bm6oMPPlBWVtY1T0xPT09XWFiYR4ldIrkDAPyUWWvuOTk5ysnJcWmz2+3XvLHbt99+q9jYWGVnZys0NFRLly5VTEzMFf1OnTql8ePH69FHH/U4pgJ6+T0AAIVDYmKiwsPDXbbExMRr9q9Zs6Z2796trVu3asiQIerbt6++++47lz4ZGRnq1KmTYmJi8vRQNtbcgUKINXdYXX6suac/OM+UcYLmPuBR5f5nrVu3VvXq1fX6669Lks6dO6d27dqpePHiWrlypYKCgjyOya1p+YyMDLcHDAsL8zgIAADym1nXuXuSyK/G4XA4fxxkZGSoXbt2stvtWrFiRZ4Su+Rmci9ZsqTbj7XjOncAAK4uISFBHTp0UGRkpM6dO6eFCxdq48aNWrNmjTIyMtS2bVudP39e77zzjjIyMpzFddmyZVWkSBG3j+NWcv/888+dr3/66SeNGjVK/fr1c57dl5ycrAULFvzlGgMAAAWJL+4tn5aWpj59+ujEiRMKDw9XvXr1tGbNGrVp00YbN27U1q1bJUnR0dEunzt8+LCioqLcPo7Ha+6tWrXSwIEDdf/997u0L1y4UG+88YY2btzoyXBewZo7rI41d1hdfqy5Z/abb8o4ofP7mTKOmTw+Wz45OVmNGze+or1x48batm2bKUEBAIC88zi5V65cWbNnz76ifc6cOapcubIpQQEA4HUBNnO2Asjjm9hMmzZNPXr00OrVq3XbbbdJkrZt26YDBw7oww8/ND1AAAC8wcrPc/e4cu/YsaP279+vLl266PTp0zp9+rS6dOmi/fv3q2PHjt6IEQAAeCBPt5+tXLmyXnjhBbNjAQAg3/jika/5JU9f7csvv1Tv3r3VtGlT/fLLL5Kkt99+W5s3bzY1OAAAvMUWYDNlK4g8Tu4ffvih2rVrp+DgYH399dfOu+qkp6dTzQMACg2bzZytIPI4uT///POaNWuWZs+erWLFijnbmzVrpq+//trU4AAAgOc8XnNPSUlR8+bNr2gPDw/X2bNnzYjphn1yYZKvQwC8ysg45+sQgMKPNff/r3z58jp48OAV7Zs3b1a1atVMCQoAAK+z8HXuHif3Rx55RE888YS2bt0qm82m48eP691339VTTz2lIUOGeCNGAADgAY+n5UeNGiWHw6FWrVrp/Pnzat68uex2u5566ikNGzbMGzECAGC+gno2nAk8fnDMHy5evKiDBw8qMzNTMTExCg0NNTu2vMvO9HUEgFex5g6ry48Hx2Q/8Y4p4wTN6G3KOGbyeFr+4Ycf1rlz5xQYGKiYmBjdeuutCg0NVVZWlh5++GFvxAgAADzgcXJfsGCBLly4cEX7hQsX9NZbb5kSFAAAXmfhE+rcXnPPyMiQYRgyDEPnzp1TUFCQc19ubq4++eQTRUREeCVIAADMZuEld/eTe8mSJWWz2WSz2XTTTTddsd9ms2ncuHGmBgcAADzndnL//PPPZRiGWrZsqQ8//FClSpVy7gsMDFSVKlVUsWJFrwQJAIDpCuiUuhncTu4tWrSQJB0+fFiRkZGyWXk+AwBgfRZO7h6fULdhwwYtWbLkivYPPvhACxYsMCUoAAC8LsCkrQDyOKzExESVKVPmivaIiAieCgcAQAHg8R3qjh49qqpVq17RXqVKFR09etSUoAAA8DoLLy97XLlHRETom2++uaJ9z549Kl26tClBAQDgdRa+zt3j5H7//ffr8ccf1+eff67c3Fzl5uZqw4YNeuKJJ3Tfffd5I0YAAOABj6flx48fr59++kmtWrVS0aK/f9zhcKhPnz6suQMACo8CejKcGfL84Jj9+/drz549Cg4OVt26dVWlShWzY8s7HhwDi+PBMbC6/HhwTM6YxaaMYx/X05RxzORx5f6Hm2666ap3qgMAAL7lVnKPj4/X+PHjFRISovj4+L/sO3XqVFMCAwDAqwrmuXCmcCu579q1S5cuXXK+vhbuWgcAKDQK6JnuZnAruX/++edXfQ0AAAqePK+5AwBQqPl75X7PPfe4PeBHH32U52AAAMgvVl5Jdiu5h4eHO18bhqGlS5cqPDxcjRs3liTt3LlTZ8+e9ehHAAAAPuXvlfu8efOcr//1r3+pZ8+emjVrlooUKSJJys3N1WOPPaawsDDvRAkAANzm8U1sypYtq82bN6tmzZou7SkpKWratKl+++03UwPME25iA4vjJjawuvy4ic3FSeYsIwf+q+DNWnt8873Lly/rhx9+uKL9hx9+kMPhMCUoAAC8zsLPc/f4bPn+/ftrwIABOnTokG699VZJ0tatWzVx4kT179/f9AABAIBnPE7uU6ZMUfny5fXSSy/pxIkTkqQKFSro6aef1pNPPml6gAAAeIWFT5fP84NjJCkjI0OSCt6JdKy5w+JYc4fV5cua+9SlpowTGH+3KeOYKU+rBZcvX9a6dev03nvvOW85e/z4cWVmklQBAPA1j6fljxw5ovbt2+vo0aPKyclRmzZtVKJECU2aNEk5OTmaNWuWN+IEAMBcFr7O3ePK/YknnlDjxo115swZBQcHO9vvvvturV+/3tTgAADwmgCbOVsB5HHl/uWXX+q///2vAgMDXdqjoqL0yy+/mBYYAADIG4+Tu8PhUG5u7hXtP//8s0qUKGFKUAAAeF3BLLpN4fG0fNu2bTV9+nTne5vNpszMTI0ZM0YdO3Y0MzYAALyHafn/b8qUKWrfvr1iYmKUnZ2tBx54QAcOHFCZMmX03nvveSNGAADMV0ATsxk8rtwrV66sPXv26D//+Y9GjBihBg0aaOLEidq1a5ciIiK8ESMAAJaQlJSkevXqKSwsTGFhYYqNjdXq1aud+7OzsxUXF6fSpUsrNDRUPXr00MmTJz0+jkc3sbl06ZJq1aqllStXqnbt2h4fLN9wExtYHDexgdXlx01sLs362JRxig3u4nbfjz/+WEWKFFGNGjVkGIYWLFigyZMna9euXapTp46GDBmiVatWaf78+QoPD9fQoUMVEBCgLVu2eBSTx3eo+/vf/65169aR3AEfIrnD6vIlub+x0pRxij3a+YY+X6pUKU2ePFn33nuvypYtq4ULF+ree++V9PtD2WrXrq3k5GTdfvvtbo/p8bR8XFycJk2apMuXL3v6UQAA8H9yc3P1/vvvKysrS7Gxsdq5c6cuXbqk1q1bO/vUqlVLkZGRSk5O9mhsj0+o2759u9avX6/PPvtMdevWVUhIiMv+jz4y5/m4AAB4lUkn1OXk5CgnJ8elzW63y263X7X/t99+q9jYWGVnZys0NFRLly5VTEyMdu/ercDAQJUsWdKlf7ly5ZSamupRTB4n95IlS6pHjx6efgwAgILFpGexJyYmaty4cS5tY8aM0dixY6/av2bNmtq9e7fS09O1ZMkS9e3bV5s2bTInmP9zQ0+FK7BYc4fFseYOq8uXNfe5q0wZx/Fga48q9z9r3bq1qlevrl69eqlVq1Y6c+aMS/VepUoVDR8+XCNGjHA7Jrd/tzgcDk2aNEnNmjVTkyZNNGrUKF24cMHtAwEAUKDYbKZsdrvdeWnbH5u7iV36Pb/m5OSoUaNGKlasmMtzWlJSUnT06FHFxsZ69NXcnpafMGGCxo4dq9atWys4OFgzZsxQWlqa5s6d69EBAQAoEEyalvdEQkKCOnTooMjISJ07d04LFy7Uxo0btWbNGoWHh2vAgAGKj49XqVKlFBYWpmHDhik2NtajM+UlD5L7W2+9pddee02DBg2SJK1bt06dOnXSnDlzFBDgg39CAAAUMmlpaerTp49OnDih8PBw1atXT2vWrFGbNm0kSdOmTVNAQIB69OihnJwctWvXTq+99prHx3F7zd1ut+vgwYOqXLmysy0oKEgHDx5UpUqVPD6wV7HmDotjzR1Wly9r7m+tvn4nNxTr08GUcczkduV++fJlBQUFubQVK1ZMly5dMj0oAAC8zrq3lnc/uRuGoX79+rmcJJCdna3Bgwe7XOvOde4AgEKB5C717dv3irbevXubGgwAALhxbif3efPmeTMOAADyl4Uf+erxHeoAALAE6+Z2X1zlBwAAvInKHQDgnyxcuZPcAQD+yWbd7M60PAAAFkPlDgDwSxYu3EnuAAA/ZeHkzrQ8AAAWQ+UOAPBP3MQGAACLsW5uJ7kDAPyUhZM7a+4AAFgMlTsAwD9Z+Fo4kjsAwD9ZN7czLQ8AgNVQuQMA/JOFy1uSOwDAP1l4zd3Cv1sAAPBPVO4AAP9k3cKd5A4A8FMWTu5MywMAYDFU7gAA/2ThE+pI7gAA/2ThuWuSOwDAP1m3cLfy7xYAAPwTlTsAwD+x5g4AgMVYN7czLQ8AgNVQuQMA/JKFZ+VJ7gAAPxVg3ezOtDwAABZD5Q4A8E/WLdxJ7gAAP2XhRXem5QEAsBgqd7jl9Tfn6rP1n+vHwz8pyG5Xg1vq6anhj6taVJSzz0MDHtW2HTtdPtfr3h56bvS/8zlawHOvv/2u1n7xhX48cvT3v/Gb6+jJIYNULTLSpd+uvfs0ffYcffPd9woICFDtGtGa89JkBdntPooceWbdwp3kDvds2/G1Huz1T9WtU0e5ubma+spMDRgcp1UfLVHx4sHOfj173K3HHxvsfB8cFOSLcAGPbd+9Ww/c3V11a9dSbm6upr0+RwPjn9bKt+erePDvf+O79u7TI0+N1KO9H9Azwx9XkSJFlHLwkAIsPL1raRb+12YzDMPwdRCmy870dQSWd/r0GcX+o7XemTtbTRo1lPR75V6r5k36z8infByd9RkZ53wdguWdPnNWTbt219uvzFCTW+pLknoNGqKmTRrriYEDfByd9dkiKnj9GLlbNpkyTpFmLUwZx0ysuSNPzmX+/gMqPCzMpf3jT1brthYt1fmennppxiu6cOGCL8IDbti5rD/+xktIkn47c0Z7vvtepUr+TfcNiVOzrner99AntPObb3wZJnBVBXpa/tixYxozZozmzp17zT45OTnKyclxabMbl2Rn/ctrHA6HXnhxihreUl831Yh2tnfu0F4VK5RXRERZpew/oCnTX9Hhn45o5rQpPowW8JzD4dALL89Uw7o366Zq1SRJx44flyTNnDdfIx8boto1orX80zXqN/xJfbxgnqIqV/JlyMgLC0/LF+jK/fTp01qwYMFf9klMTFR4eLjLljj5pXyK0D+Ne2GiDhw6pGkvJrq097r3Ht3ZrKlq1qihrp06atLz47R2w+c6euyYjyIF8ua5qdN14PBhTR37rLPN4fh9BbNX1y7q0amDYm6qoYTHh6pq5cr6cNUnvgoVN8JmM2crgHxaua9YseIv9//444/XHSMhIUHx8fEubXbj0g3FhWt77oVJ2vjFZr0zd7bKlyv3l33r160rSTpy9JgiK1fOj/CAG/bctOnamJysd155WeUjIpztEaVLS5Kio6q49K8eVUUn0tLyNUbgenya3Lt37y6bzaa/OqfPdp1fRXa7/copeE6oM51hGBqf+KLWbvhcb7/5hipX+vt1P/N9SookqWzZst4OD7hhhmFo/PQZWvfFZr318nRVquh6QtffK5RXRJkyOvynmaifjh3Tnbfdlp+hwjQFs+o2g0+n5StUqKCPPvpIDofjqtvXX3/ty/DwP8a9MFErPvlEL02coJCQ4vr11Cn9euqUsrOzJUlHjx3Tq6/P1t7vvtfPvxzX+o2b9K9nnlWTRg1V66YaPo4euL7npk7Xx5+t1ZRnn1FI8WD9+ttv+vW335T9f+f02Gw2Dbi/l95e8pE+/Xyjjvz8s2bMeVM/Hjmqezt39HH0yBMfTMsnJiaqSZMmKlGihCIiItS9e3el/F8h9IfU1FQ99NBDKl++vEJCQtSwYUN9+OGHHh3Hp5V7o0aNtHPnTnXr1u2q+69X1SP/vLd4iaTfL3f7X4nPjdE93bqqWLFiSt66TW+9+57OX7igCuXLqW3rVnrsES4ZQuHw3rLlkqQ+jw93aX8h4V+6p2MHSVLfnv9UzsWLmjjzVaVnnFPN6OqaO22KIv9+/ZksQJI2bdqkuLg4NWnSRJcvX9a///1vtW3bVt99951CQkIkSX369NHZs2e1YsUKlSlTRgsXLlTPnj21Y8cONWjQwK3j+PQ69y+//FJZWVlq3779VfdnZWVpx44datHCw2sImZaHxXGdO6wuX65z37rZlHGK3HZHnj/766+/KiIiQps2bVLz5s0lSaGhoUpKStJDDz3k7Fe6dGlNmjRJAwcOdGtcn1bud95551/uDwkJ8TyxAwDgDpPOdL/qJdlXOx/sKtLT0yVJpUqVcrY1bdpUixYtUqdOnVSyZEktXrxY2dnZuuuuu9yOqUBfCgcAQEF31UuyExOv+zmHw6Hhw4erWbNmuvnmm53tixcv1qVLl1S6dGnZ7XYNGjRIS5cuVXR09F+M5qpA38QGAACvMalyv+ol2W5U7XFxcdq7d682b3ZdHhg9erTOnj2rdevWqUyZMlq2bJl69uypL7/8UnX/7xLj6+He8kAhxJo7rC5f1tx3JJsyTpHGsR5/ZujQoVq+fLm++OILVa1a1dl+6NAhRUdHa+/evapTp46zvXXr1oqOjtasWbPcGp/KHQDgn3xwdznDMDRs2DAtXbpUGzdudEnsknT+/HlJUkCA66p5kSJF5HA43D4OyR0AgHwSFxenhQsXavny5SpRooRSU1MlSeHh4QoODlatWrUUHR2tQYMGacqUKSpdurSWLVumtWvXauXKlW4fh2l5oBBiWh5Wly/T8ju3mjJOkUbu36HwWnddnTdvnvr16ydJOnDggEaNGqXNmzcrMzNT0dHReuqpp1wujbvucUjuQOFDcofV5Uty/3qbKeMUaXirKeOYiUvhAACwGNbcAQD+qYA+rtUMJHcAgH+ycHJnWh4AAIuhcgcA+CcLV+4kdwCAf7JwcmdaHgAAiyG5AwBgMUzLAwD8k4Wn5UnuAAC/dK1bwVoB0/IAAFgMlTsAwD9ZuHInuQMA/JOFkzvT8gAAWAyVOwDAP1m4cie5AwD8k4WTO9PyAABYDJU7AMBPWbdyJ7kDAPwT0/IAAKCwoHIHAPgnC1fuJHcAgH8iuQMAYDHWze2suQMAYDVU7gAA/8S0PAAAFmPh5M60PAAAFkPlDgDwU9at3EnuAAD/xLQ8AAAoLKjcAQD+ycKVO8kdAOCfLJzcmZYHAMBiqNwBAP6Jyh0AABQWVO4AAP9E5Q4AAAoLKncAgH+ycOVOcgcA+Cfr5nam5QEAsBoqdwCAn7Ju6U5yBwD4JwuvuTMtDwCAxVC5AwD8k3ULd5I7AMA/2Syc3ZmWBwDAYkjuAAD/ZLOZs3kgMTFRTZo0UYkSJRQREaHu3bsrJSXlin7Jyclq2bKlQkJCFBYWpubNm+vChQtuH4fkDgDwTz5I7ps2bVJcXJy++uorrV27VpcuXVLbtm2VlZXl7JOcnKz27durbdu22rZtm7Zv366hQ4cqIMD9lG0zDMPwKLLCIDvT1xEAXmVknPN1CIBX2SIqeP0YRtpxU8axRVTM82d//fVXRUREaNOmTWrevLkk6fbbb1ebNm00fvz4PI9L5Q4AwA3IyclRRkaGy5aTk+PWZ9PT0yVJpUqVkiSlpaVp69atioiIUNOmTVWuXDm1aNFCmzdv9igmkjsAwE/ZTNkSExMVHh7usiUmJl736A6HQ8OHD1ezZs108803S5J+/PFHSdLYsWP1yCOP6NNPP1XDhg3VqlUrHThwwP1vxrQ8UPgwLQ+ry5dp+V9TTRnnYtjfrqjU7Xa77Hb7X35uyJAhWr16tTZv3qxKlSpJkv773/+qWbNmSkhI0AsvvODsW69ePXXq1MmtHw0S17kDAHBD3EnkfzZ06FCtXLlSX3zxhTOxS1KFCr//qImJiXHpX7t2bR09etTt8ZmWBwD4J3Nm5T1iGIaGDh2qpUuXasOGDapatarL/qioKFWsWPGKy+P279+vKlWquH0cKncAgJ/K/zvUxcXFaeHChVq+fLlKlCih1NTflwbCw8MVHBwsm82mp59+WmPGjFH9+vV1yy23aMGCBfrhhx+0ZMkSt4/DmjtQCLHmDqvLlzX3UydNGcdWppz7fa9xXfy8efPUr18/5/uJEyfq1Vdf1enTp1W/fn29+OKLuuOOO9w/DskdKHxI7rC6fEnuv6WZMo6tdIQp45iJaXkAgH+y7nNjOKEOAACroXIHAPgnD+8LX5iQ3AEAforkDgCAtVg3t7PmDgCA1VC5AwD8E2vuAABYjIWTO9PyAABYDMkdAACLYVoeAOCXrnWfdyugcgcAwGKo3AEA/snClTvJHQDgp6yb3JmWBwDAYqjcAQD+ybqFO8kdAOCnWHMHAMBiLJzcWXMHAMBiSO4AAFgM0/IAAP/EtDwAACgsbIZhGL4OAoVbTk6OEhMTlZCQILvd7utwAFPx943CiOSOG5aRkaHw8HClp6crLCzM1+EApuLvG4UR0/IAAFgMyR0AAIshuQMAYDEkd9wwu92uMWPGcLIRLIm/bxRGnFAHAIDFULkDAGAxJHcAACyG5A4AgMWQ3AEAsBiSO27Iq6++qqioKAUFBem2227Ttm3bfB0SYJovvvhCXbp0UcWKFWWz2bRs2TJfhwS4heSOPFu0aJHi4+M1ZswYff3116pfv77atWuntLQ0X4cGmCIrK0v169fXq6++6utQAI9wKRzy7LbbblOTJk00c+ZMSZLD4VDlypU1bNgwjRo1ysfRAeay2WxaunSpunfv7utQgOuickeeXLx4UTt37lTr1q2dbQEBAWrdurWSk5N9GBkAgOSOPDl16pRyc3NVrlw5l/Zy5copNTXVR1EBACSSOwAAlkNyR56UKVNGRYoU0cmTJ13aT548qfLly/soKgCARHJHHgUGBqpRo0Zav369s83hcGj9+vWKjY31YWQAgKK+DgCFV3x8vPr27avGjRvr1ltv1fTp05WVlaX+/fv7OjTAFJmZmTp48KDz/eHDh7V7926VKlVKkZGRPowM+GtcCocbMnPmTE2ePFmpqam65ZZb9PLLL+u2227zdViAKTZu3Kh//OMfV7T37dtX8+fPz/+AADeR3AEAsBjW3AEAsBiSOwAAFkNyBwDAYkjuAABYDMkdAACLIbkDAGAxJHcAACyG5A4UUlFRUZo+fbrb/efPn6+SJUve8HFtNpuWLVt2w+MA8B6SO+ABm832l9vYsWN9HSIAcG95wBMnTpxwvl60aJGeffZZpaSkONtCQ0Odrw3DUG5urooW5T8zAPmLyh3wQPny5Z1beHi4bDab8/0PP/ygEiVKaPXq1WrUqJHsdrs2b96sfv36qXv37i7jDB8+XHfddZfzvcPhUGJioqpWrarg4GDVr19fS5Ys8Si2qVOnqm7dugoJCVHlypX12GOPKTMz84p+y5YtU40aNRQUFKR27drp2LFjLvuXL1+uhg0bKigoSNWqVdO4ceN0+fJlj2IB4Fskd8Bko0aN0sSJE/X999+rXr16bn0mMTFRb731lmbNmqV9+/ZpxIgR6t27tzZt2uT2cQMCAvTyyy9r3759WrBggTZs2KCRI0e69Dl//rwmTJigt956S1u2bNHZs2d13333Ofd/+eWX6tOnj5544gl99913ev311zV//nxNmDDB7TgA+B7zhYDJnnvuObVp08bt/jk5OXrhhRe0bt06xcbGSpKqVaumzZs36/XXX1eLFi3cGmf48OHO11FRUXr++ec1ePBgvfbaa872S5cuaebMmc4n9y1YsEC1a9fWtm3bdOutt2rcuHEaNWqU+vbt64xj/PjxGjlypMaMGeP2dwLgWyR3wGSNGzf2qP/Bgwd1/vz5K34QXLx4UQ0aNHB7nHXr1ikxMVE//PCDMjIydPnyZWVnZ+v8+fMqXry4JKlo0aJq0qSJ8zO1atVSyZIl9f333+vWW2/Vnj17tGXLFpdKPTc394pxABRsJHfAZCEhIS7vAwIC9OcnK1+6dMn5+o918VWrVunvf/+7Sz+73e7WMX/66Sd17txZQ4YM0YQJE1SqVClt3rxZAwYM0MWLF91OypmZmRo3bpzuueeeK/YFBQW5NQYA3yO5A15WtmxZ7d2716Vt9+7dKlasmCQpJiZGdrtdR48edXsK/s927twph8Ohl156SQEBv59Ks3jx4iv6Xb58WTt27NCtt94qSUpJSdHZs2dVu3ZtSVLDhg2VkpKi6OjoPMUBoGAguQNe1rJlS02ePFlvvfWWYmNj9c4772jv3r3OKfcSJUroqaee0ogRI+RwOHTHHXcoPT1dW7ZsUVhYmHP9+69ER0fr0qVLeuWVV9SlSxdt2bJFs2bNuqJfsWLFNGzYML388ssqWrSohg4dqttvv92Z7J999ll17txZkZGRuvfeexUQEKA9e/Zo7969ev755839BwPAazhbHvCydu3aafTo0Ro5cqSaNGmic+fOqU+fPi59xo8fr9GjRysxMVG1a9dW+/bttWrVKlWtWtWtY9SvX19Tp07VpEmTdPPNN+vdd99VYmLiFf2KFy+uf/3rX3rggQfUrFkzhYaGatGiRS6xrly5Up999pmaNGmi22+/XdOmTVOVKlVu7B8CgHxlM/68GAgAAAo1KncAACyG5A4AgMWQ3AEAsBiSOwAAFkNyBwDAYkjuAABYDMkdAACLIbkDAGAxJHcAACyG5A4AgMWQ3AEAsBiSOwAAFvP/ALv0NHBj9ogtAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:\t0.556522\n",
      "F1-score:\t0.504854\n",
      "Precision:\t0.509804\n",
      "Recall:\t\t0.500000\n",
      "\n",
      "Classification performance:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.60      0.60        63\n",
      "           1       0.51      0.50      0.50        52\n",
      "\n",
      "    accuracy                           0.56       115\n",
      "   macro avg       0.55      0.55      0.55       115\n",
      "weighted avg       0.56      0.56      0.56       115\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mat_catboost = confusion_matrix(y, y_pred_catboost)  \n",
    "sns.heatmap(mat_catboost.T, square=True, annot=True, fmt='d', cmap='RdPu')\n",
    "plt.xlabel('True label')\n",
    "plt.ylabel('Predicted label')\n",
    "plt.title(\"Catboost\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Accuracy:\\t%f\" % accuracy_score(y, y_pred_catboost))  \n",
    "print(\"F1-score:\\t%f\" % f1_score(y, y_pred_catboost))  \n",
    "print(\"Precision:\\t%f\" % precision_score(y, y_pred_catboost))  \n",
    "print(\"Recall:\\t\\t%f\" % recall_score(y, y_pred_catboost))\n",
    "print(\"\\nClassification performance:\\n%s\" % classification_report(y, y_pred_catboost))  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}