NMMH_TH4_21C29001.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import requests\n",
    "import io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv( \"https://github.com/huynhthanh98/ML/raw/ML-2022/Lab04/lienminh.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>killsDiff</th>\n",
       "      <th>minionsKilledDiff</th>\n",
       "      <th>wardPlacedDiff</th>\n",
       "      <th>firstBlood</th>\n",
       "      <th>heralds</th>\n",
       "      <th>dragons</th>\n",
       "      <th>teamWins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>-2</td>\n",
       "      <td>13</td>\n",
       "      <td>blue</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>-66</td>\n",
       "      <td>0</td>\n",
       "      <td>red</td>\n",
       "      <td>red</td>\n",
       "      <td>red</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4</td>\n",
       "      <td>-17</td>\n",
       "      <td>0</td>\n",
       "      <td>red</td>\n",
       "      <td>none</td>\n",
       "      <td>blue</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1</td>\n",
       "      <td>-34</td>\n",
       "      <td>28</td>\n",
       "      <td>red</td>\n",
       "      <td>blue</td>\n",
       "      <td>none</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>-15</td>\n",
       "      <td>58</td>\n",
       "      <td>red</td>\n",
       "      <td>none</td>\n",
       "      <td>red</td>\n",
       "      <td>red</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   killsDiff  minionsKilledDiff  wardPlacedDiff firstBlood heralds dragons  \\\n",
       "0          3                 -2              13       blue    none    none   \n",
       "1          0                -66               0        red     red     red   \n",
       "2         -4                -17               0        red    none    blue   \n",
       "3         -1                -34              28        red    blue    none   \n",
       "4          0                -15              58        red    none     red   \n",
       "\n",
       "  teamWins  \n",
       "0      red  \n",
       "1      red  \n",
       "2      red  \n",
       "3      red  \n",
       "4      red  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## UTILS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onehot(x):\n",
    "    _dict = dict([(i[1],i[0]) for i in enumerate(np.sort(np.unique(x)))]) # adding sort to make sure target var are properly encoding \n",
    "    nb_classes = len(np.unique(x))\n",
    "    _x = np.vectorize(_dict.get)(x)\n",
    "    one_hot_vector = np.eye(nb_classes)[_x.astype(int)]\n",
    "    return (one_hot_vector , _dict) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CALCULATE  P(X|Y) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- For Categorical variable: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def categorical_likelihood_x_given_y(x, y, label):\n",
    "    '''\n",
    "    - input: vector feature x, vector target y, label of target (0,1)\n",
    "    - output: vector p(x|yi)\n",
    "    '''\n",
    "    \n",
    "    filter_yi=(y==label)\n",
    "    if label==1:\n",
    "        y_temp = y\n",
    "    else:\n",
    "        y_temp = 1-y\n",
    "    _x_onehot=onehot(x)[0]\n",
    "    _p_x_given_y =np.dot(_x_onehot.T, (y_temp.reshape(-1, 1))) / y[filter_yi].shape[0] # k scalar with k is \n",
    "    _p_xj_given_y = np.dot(_x_onehot, _p_x_given_y).reshape(-1)\n",
    "    return _p_xj_given_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- For Numerical Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gaussian_likelihood_x_given_y (x, y, label):\n",
    "    '''\n",
    "    - input: vector feature x, vector target y, lable of target (0,1)\n",
    "    - output: vector p(x|yi)\n",
    "    '''\n",
    "    filter_yi=(y==label)\n",
    "    mean, std = np.mean(x[filter_yi]), np.std(x[filter_yi])\n",
    "    p=(1/(np.sqrt(2*np.pi)*std)) * np.exp(-np.square(x-mean)/(2*np.square(std)))\n",
    "    return p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def likelihood_calc(y, X_categorical=None, X_numerical=None):\n",
    "    # calculating the likelihood P(x|y) matrix by using categorical and gaussian likelihood function given each yi\n",
    "    num_labels = len(list(np.unique(y)))\n",
    "    num_observations=y.shape[0]\n",
    "    _likelihood=np.zeros([num_observations, num_labels]) \n",
    "    \n",
    "    for label in list(np.unique(y)):\n",
    "        if isinstance(X_categorical, np.ndarray): \n",
    "            _Likelihood_X_categorical=np.apply_along_axis(\n",
    "                lambda x: categorical_likelihood_x_given_y (x, y, label) ,0 , X_categorical)\n",
    "        else: \n",
    "             _Likelihood_X_categorical=np.zeros([y.shape[0],0]) \n",
    "                \n",
    "        if isinstance(X_numerical, np.ndarray):  \n",
    "            _Likelihood_X_numerical=np.apply_along_axis(\n",
    "                lambda x: gaussian_likelihood_x_given_y (x, y, label) ,0, X_numerical)\n",
    "        else: \n",
    "            _Likelihood_X_numerical=np.zeros([y.shape[0],0]) \n",
    "\n",
    "        _likelihood_matrix=np.concatenate(\n",
    "            [_Likelihood_X_categorical, _Likelihood_X_numerical], axis=1) \n",
    "        \n",
    "        _likelihood[:, label]=_likelihood_matrix[:, 0]\n",
    "        for var in range(1, _likelihood_matrix.shape[1]):\n",
    "            _likelihood[:, label]*=_likelihood_matrix[:, var] \n",
    "            \n",
    "    return _likelihood  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Build the Classifier, calculate  P(Y|X) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Posterior density ∝ Likelihood × Prior density**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def posterior_probability_calc(likelihood, prior, _dict_label2idx):\n",
    "    '''\n",
    "    Posterior density ∝ Likelihood × Prior density  \n",
    "    '''\n",
    "    \n",
    "    prior_dict = dict([(_dict_label2idx.get(i), j)  for i, j in prior.items()]) \n",
    "    posterior = np.zeros([likelihood.shape[0], likelihood.shape[1]])\n",
    "    \n",
    "    for i in range(likelihood.shape[1]):\n",
    "        posterior[:, i] =prior_dict.get(i)*likelihood[:, i] \n",
    "    return posterior "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classifying(posterior, _y_idx_label):\n",
    "    r=np.argmax(posterior, axis=1).reshape(-1)\n",
    "    return  np.vectorize(lambda x: _y_idx_label.get(x))(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def naive_bayes_prediction(\n",
    "    df, \n",
    "    target_variable, \n",
    "    prior, \n",
    "    categorical_variables=None,\n",
    "    numerical_variables=None):\n",
    "    \n",
    "    # convert pandas table to seperated numpy array of categorical and numerical variable \n",
    "    if categorical_variables != None: \n",
    "        X_categorical=df[categorical_variables].values\n",
    "    else:\n",
    "        X_categorical=None\n",
    "    if numerical_variables != None: \n",
    "        X_numerical=df[numerical_variables].values\n",
    "    else:\n",
    "        X_numerical=None\n",
    "    target=df[target_variable].values\n",
    "    \n",
    "    _dict_label2idx = dict([(i[1],i[0]) for i in enumerate(np.sort(np.unique(target)))])\n",
    "    _dict_idx2label = dict([(i[0],i[1]) for i in enumerate(np.sort(np.unique(target)))])     \n",
    "    \n",
    "    y = np.vectorize(lambda x: _dict_label2idx.get(x))(target).astype(int) \n",
    "    \n",
    "    likelihood=likelihood_calc(y, X_categorical, X_numerical) \n",
    "    posterior=posterior_probability_calc(likelihood, prior, _dict_label2idx)\n",
    "    y_pred=classifying(posterior, _dict_idx2label) \n",
    "    return y_pred\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_accuracy(y_target, y_predict):\n",
    "    '''\n",
    "    y_predict_score_matrix: result is list if float in (0,1)\n",
    "    '''\n",
    "    acc_score=np.mean((y_target.reshape(-1)==y_predict))\n",
    "    return acc_score  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction and Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_variables=['firstBlood', 'heralds', 'dragons']\n",
    "numerical_variables=['killsDiff', 'minionsKilledDiff', 'wardPlacedDiff'] \n",
    "target_variable='teamWins'\n",
    "y_target=df[target_variable].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Chỉ sử dụng các biến categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy if using categical variable only:  0.6298208320680231\n"
     ]
    }
   ],
   "source": [
    "prior={'blue': 0.5, 'red': 0.5} \n",
    "prior \n",
    "y_nb_categical_pred = naive_bayes_prediction(\n",
    "    df, \n",
    "    target_variable, \n",
    "    prior,  \n",
    "    categorical_variables, \n",
    "    numerical_variables=None)  \n",
    "acc = get_accuracy(y_target, y_nb_categical_pred) \n",
    "print(\"Accuracy if using categical variable only: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Chỉ sử dụng các biến continuous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy if using continuous variable only:  0.7250733879947363\n"
     ]
    }
   ],
   "source": [
    "prior={'blue': 0.5, 'red': 0.5} \n",
    "prior \n",
    "y_nb_gaussian_pred = naive_bayes_prediction(\n",
    "    df, \n",
    "    target_variable, \n",
    "    prior,   \n",
    "    numerical_variables)  \n",
    "acc = get_accuracy(y_target, y_nb_gaussian_pred)  \n",
    "print(\"Accuracy if using continuous variable only: \", acc) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Sử dụng cả biến categorical và continuous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy if using both:  0.7116104868913857\n"
     ]
    }
   ],
   "source": [
    "prior={'blue': 0.5, 'red': 0.5} \n",
    "prior \n",
    "y_nb_pred = naive_bayes_prediction(\n",
    "    df, \n",
    "    target_variable, \n",
    "    prior,  \n",
    "    categorical_variables, \n",
    "    numerical_variables) \n",
    "acc =  get_accuracy(y, y_nb_pred) \n",
    "print(\"Accuracy if using both: \", get_accuracy(y, y_nb_pred))  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SKlearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['killsDiff', 'minionsKilledDiff', 'wardPlacedDiff', 'firstBlood',\n",
       "       'heralds', 'dragons', 'teamWins'], dtype=object)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df[[col for col in df.columns.values if col != 'teamWins']].values\n",
    "X_categorical=df[categorical_variables].values  \n",
    "X_numerical=df[numerical_variables].values    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=df['teamWins'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB \n",
    "from sklearn.naive_bayes import BernoulliNB \n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[categorical_variables]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_encoder=LabelEncoder()\n",
    "label_encoded=label_encoder.fit_transform(df['teamWins'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lap010760/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_encoders.py:110: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  X_int = np.zeros((n_samples, n_features), dtype=np.int)\n",
      "/Users/lap010760/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/_encoders.py:111: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  X_mask = np.ones((n_samples, n_features), dtype=np.bool)\n"
     ]
    }
   ],
   "source": [
    "encoder = OrdinalEncoder()\n",
    "X_categorical_encoded = encoder.fit_transform(df[categorical_variables])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BernoulliNB(alpha=1.0, binarize=0.0, class_prior=[0.5, 0.5], fit_prior=True)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bernoulli_clf = BernoulliNB(class_prior=[0.5, 0.5]) \n",
    "bernoulli_clf.fit(X_categorical_encoded, y) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Chỉ sử dụng biến categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6133211863548942"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_categorical = bernoulli_clf.predict(X_categorical_encoded)\n",
    "np.mean(y_pred_categorical==y) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Chỉ sử dụng biến continous "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7079663933596518"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gaussian_clf = GaussianNB(priors=[0.5, 0.5]) \n",
    "gaussian_clf.fit(X_numerical, y)\n",
    "y_pred_gaussian = gaussian_clf.predict(X_numerical) \n",
    "np.mean(y_pred_gaussian==y) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob_pred_categorical = bernoulli_clf.predict_proba(X_categorical_encoded) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob_pred_gaussian = gaussian_clf.predict_proba(X_numerical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.4977507 , 0.5022493 ],\n",
       "       [0.30427725, 0.69572275],\n",
       "       [0.52018688, 0.47981312],\n",
       "       ...,\n",
       "       [0.30427725, 0.69572275],\n",
       "       [0.7107041 , 0.2892959 ],\n",
       "       [0.4977507 , 0.5022493 ]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob_pred_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.70490043, 0.29509957],\n",
       "       [0.2087349 , 0.7912651 ],\n",
       "       [0.17662248, 0.82337752],\n",
       "       ...,\n",
       "       [0.20840452, 0.79159548],\n",
       "       [0.2703737 , 0.7296263 ],\n",
       "       [0.53971591, 0.46028409]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob_pred_gaussian"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Sử dụng cả categorical và numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob_pred_combined=y_prob_pred_categorical*y_prob_pred_gaussian/0.5\n",
    "y_pred_argmax = np.argmax(y_prob_pred_combined, axis=1) \n",
    "y_pred=label_encoder.inverse_transform(y_pred_argmax)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7100921145864966"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y_pred==y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}