Apply SVM to Amazon reviews data set TF IDF word2vec [M].ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting imblearn\n",
      "  Using cached https://files.pythonhosted.org/packages/81/a7/4179e6ebfd654bd0eac0b9c06125b8b4c96a9d0a8ff9e9507eb2a26d2d7e/imblearn-0.0-py2.py3-none-any.whl\n",
      "Collecting imbalanced-learn (from imblearn)\n",
      "  Using cached https://files.pythonhosted.org/packages/80/a4/900463a3c0af082aed9c5a43f4ec317a9469710c5ef80496c9abc26ed0ca/imbalanced_learn-0.3.3-py3-none-any.whl\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn)\n",
      "Installing collected packages: imbalanced-learn, imblearn\n",
      "Successfully installed imbalanced-learn-0.3.3 imblearn-0.0\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 10.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install --user imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vishal/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/home/vishal/anaconda3/lib/python3.6/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import norm\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "import sqlite3\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import gensim\n",
    "\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "con = sqlite3.connect('final.sqlite') # this is cleaned dataset\n",
    "final = pd.read_sql_query(\"\"\"\n",
    "SELECT Score, Text_not_included\n",
    "FROM reviews\n",
    "\"\"\", con)[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, seq in enumerate(final['Text_not_included']):\n",
    "  final['Text_not_included'][i]=final['Text_not_included'][i].decode('UTF-8')\n",
    "X_train, X_test, y_train , y_test = train_test_split(final['Text_not_included'], final['Score'], test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate TF IDF weighted word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_idf_vect=TfidfVectorizer(ngram_range=(1,2), min_df=10, dtype=float)\n",
    "tf_idf_vect.fit(X_train)\n",
    "tf_idf_train=tf_idf_vect.transform(X_train)\n",
    "tf_idf_test=tf_idf_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2vec_model=gensim.models.word2vec.Word2Vec(sentences, min_count=10)\n",
    "\n",
    "avg_w2vec_train=np.zeros(shape=(len(X_train), 100), dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences=[]\n",
    "for review in X_train:\n",
    "  sentence=[]\n",
    "  for word in review.split():\n",
    "    sentence.append(word)\n",
    "  sentences.append(sentence)\n",
    "  \n",
    "tf_idf_w2vec_train=np.zeros((len(X_train), 100), dtype=float)\n",
    "feat=tf_idf_vect.get_feature_names()\n",
    "for i, sentence in enumerate(sentences):\n",
    "  tf_idf_sum=0\n",
    "  for word in sentence:\n",
    "    try:\n",
    "      tf_idf_w2vec_train[i]+=w2vec_model.wv[word]*tf_idf_train[i, feat.index(word)]\n",
    "      tf_idf_sum+=tf_idf_train[i, feat.index(word)]  \n",
    "    except KeyError:\n",
    "      pass\n",
    "    except ValueError:\n",
    "      pass\n",
    "  tf_idf_w2vec_train[i]/=tf_idf_sum\n",
    "  \n",
    "sentences=[]\n",
    "for review in X_test:\n",
    "  sentence=[]\n",
    "  for word in review.split():\n",
    "    sentence.append(word)\n",
    "  sentences.append(sentence)\n",
    "  \n",
    "tf_idf_w2vec_test=np.zeros((len(X_test), 100), dtype=float)\n",
    "\n",
    "for i, sentence in enumerate(sentences):\n",
    "  tf_idf_sum=0\n",
    "  for word in sentence:\n",
    "    try:\n",
    "      tf_idf_w2vec_test[i]+=w2vec_model.wv[word]*tf_idf_test[i, feat.index(word)]\n",
    "      tf_idf_sum+=tf_idf_test[i, feat.index(word)]  \n",
    "    except KeyError:\n",
    "      pass\n",
    "    except ValueError:\n",
    "      pass\n",
    "  tf_idf_w2vec_test[i]/=tf_idf_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Upsampling minority class\n",
    "over_sampler = SMOTE(ratio='minority')\n",
    "\n",
    "tf_idf_train_resampled, y_train_resampled = over_sampler.fit_sample(tf_idf_train, y_train)\n",
    "\n",
    "tf_idf_w2vec_train_resampled, y_train_resampled = over_sampler.fit_sample(tf_idf_w2vec_train, y_train)\n",
    "\n",
    "scaler_tf_idf=StandardScaler(with_mean=False)\n",
    "\n",
    "scaler_tf_w2vec=StandardScaler()\n",
    "\n",
    "\n",
    "scaler_tf_idf.fit(tf_idf_train_resampled)\n",
    "\n",
    "scaler_tf_w2vec.fit(tf_idf_w2vec_train_resampled)\n",
    "\n",
    "\n",
    "tf_idf_train_scaled=scaler_tf_idf.transform(tf_idf_train_resampled)\n",
    "\n",
    "tf_idf_w2vec_train_scaled=scaler_tf_w2vec.transform(tf_idf_w2vec_train_resampled)\n",
    "\n",
    "\n",
    "tf_idf_test_scaled=scaler_tf_idf.transform(tf_idf_test)\n",
    "\n",
    "tf_idf_w2vec_test_scaled=scaler_tf_w2vec.transform(tf_idf_w2vec_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GridSearchCV(cv=5, error_score='raise',\n",
      "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "       fit_params={}, iid=True, n_jobs=1,\n",
      "       param_grid={'C': array([ 10.     ,  10.52632,  11.05263,  11.57895,  12.10526,  12.63158,\n",
      "        13.15789,  13.68421,  14.21053,  14.73684,  15.26316,  15.78947,\n",
      "        16.31579,  16.84211,  17.36842,  17.89474,  18.42105,  18.94737,\n",
      "        19.47368,  20.     ]), 'gamma': array([ 0.001  ,  0.0535...937,\n",
      "        0.63195,  0.68453,  0.73711,  0.78968,  0.84226,  0.89484,\n",
      "        0.94742,  1.     ])},\n",
      "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)\n",
      "RandomizedSearchCV(cv=5, error_score='raise',\n",
      "          estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "          fit_params={}, iid=True, n_iter=10, n_jobs=1,\n",
      "          param_distributions={'C': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fe45c1d7438>, 'gamma': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fe45c356390>},\n",
      "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
      "          scoring='accuracy', verbose=0)\n"
     ]
    }
   ],
   "source": [
    "tuned_parameters = {'C': np.linspace(10, 20, 20, dtype=float),'gamma' : np.linspace(0.001, 1, 20, dtype=float)}\n",
    "\n",
    "#Using GridSearchCV\n",
    "gscv = GridSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5)\n",
    "\n",
    "print(gscv.fit(tf_idf_w2vec_train_scaled, y_train_resampled))\n",
    "\n",
    "tuned_parameters = {'C' : uniform(10,20), 'gamma' : uniform(0, 1)}\n",
    "\n",
    "#Using RandomizedSearchCV\n",
    "rscv = RandomizedSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5, n_iter=10)\n",
    "\n",
    "print(rscv.fit(tf_idf_w2vec_train_scaled, y_train_resampled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.57      0.05      0.09        87\n",
      "   positive       0.79      0.99      0.88       313\n",
      "\n",
      "avg / total       0.74      0.79      0.71       400\n",
      "\n",
      "[[  4   3]\n",
      " [ 83 310]]\n",
      "TPR = 0.9904153354632588\n",
      " TNR = 0.04597701149425287\n",
      " FPR = 0.9540229885057471\n",
      " FNR = 0.009584664536741214\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.21      0.03      0.06        87\n",
      "   positive       0.78      0.96      0.86       313\n",
      "\n",
      "avg / total       0.66      0.76      0.69       400\n",
      "\n",
      "[[  3  11]\n",
      " [ 84 302]]\n",
      "TPR = 0.9648562300319489\n",
      " TNR = 0.034482758620689655\n",
      " FPR = 0.9655172413793104\n",
      " FNR = 0.03514376996805112\n"
     ]
    }
   ],
   "source": [
    "predictions = gscv.best_estimator_.predict(tf_idf_w2vec_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))\n",
    "\n",
    "predictions = rscv.best_estimator_.predict(tf_idf_w2vec_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=10.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=0.63194736842105259,\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=20.582199592274449, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=0.49642755839081232,\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rscv.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion\n",
    "\n",
    "SVMs take a lot of time to train even on small datasets with less dimensions.<br>\n",
    "Best gamma is about 0.5<br>\n",
    "TNR as found by GSCV as well as RSCV is very poor<br>\n",
    "C can be taken to be between 10 and 20 "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}