Apply_Logistic_regression_to_Amazon_reviews_data_set_[M].ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 207
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4100,
     "status": "ok",
     "timestamp": 1527592611807,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "sQJ806dEZUdC",
    "outputId": "4b2e0a3f-3a90-40fa-c386-7631071eaa6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PyDrive in /usr/local/lib/python3.6/dist-packages (1.3.1)\r\n",
      "Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (3.12)\r\n",
      "Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (4.1.2)\r\n",
      "Requirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (1.6.7)\n",
      "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.2.1)\n",
      "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (3.4.2)\n",
      "Requirement already satisfied: httplib2>=0.9.1 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.11.3)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.4.2)\n",
      "Requirement already satisfied: six>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (1.11.0)\n",
      "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (3.0.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install PyDrive\n",
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "qX2e2K2tPe2K"
   },
   "outputs": [],
   "source": [
    "# 1. Authenticate and create the PyDrive client.\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1498,
     "status": "ok",
     "timestamp": 1527592615004,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "lorL77R7aETd",
    "outputId": "29183067-f76e-4df5-aa24-6628e01acf34"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "title: Apply Logistic regression to Amazon reviews data set. [M].ipynb, id: 1Es1wP2edJ0vrKasA5wYJEO-zeZvrqeBI\n",
      "title: Apply Naive Bayes to Amazon reviews [M].ipynb, id: 1qPxAZeYQUM-eqaKnOSM5ubK2IPIVmdyo\n",
      "title: clean_final.sqlite, id: 1T0HyUqaVFyD8HfIQEM6WN8jF8SpEOsAo\n",
      "title: KNN on Credit Card fraud detection.ipynb, id: 1CkA-RBfXqvubKkQrpnjbYUKVsC7VHlTl\n",
      "title: creditcard.csv, id: 1VpeqlS0lPVrlzlMIqvQTzc3Pno_Cj4SV\n",
      "title: creditcard.csv, id: 1bnZktEq3N_5wjoCH85oIXHxNwXUW_jx-\n",
      "title: Untitled, id: 1K0wwkizWx3WO8d-zw-YewWIUrPdINYmp\n",
      "title: final.sqlite, id: 1OzLc3k6-T55I-XRMq47ERyCbQbVw4caF\n",
      "title: HeavyComputations.ipynb, id: 1aBORe3gqeFY-iNhzMtr-TIkzEyEvFxcG\n",
      "title: LogisticRegression.ipynb, id: 1WcVTklMZBMu9VTCIWeupOK0r2aYbHk8p\n"
     ]
    }
   ],
   "source": [
    "file_list = drive.ListFile({'q': \"'1pbLvjcsi6UtFm3sPciCJGbCG4NK3uyuS' in parents and trashed=false\"}).GetList()\n",
    "for file1 in file_list:\n",
    "  print('title: %s, id: %s' % (file1['title'], file1['id']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "SBPEqHOqaUuC"
   },
   "outputs": [],
   "source": [
    "sql = drive.CreateFile({'id': '1OzLc3k6-T55I-XRMq47ERyCbQbVw4caF'})\n",
    "sql.GetContentFile('final.sqlite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3023,
     "status": "ok",
     "timestamp": 1527592620490,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "XuqNF0xAebob",
    "outputId": "7814af0c-57ca-4b69-e493-bd6a6ff6c798"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: imblearn in /usr/local/lib/python3.6/dist-packages (0.0)\r\n",
      "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.6/dist-packages (from imblearn) (0.3.3)\r\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (0.19.1)\r\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (1.14.3)\r\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (0.19.1)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "mgwG-rtMb2ha"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sqlite3\n",
    "\n",
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "3Ro_daubeZp1"
   },
   "outputs": [],
   "source": [
    "con = sqlite3.connect('final.sqlite') # this is cleaned dataset\n",
    "final = pd.read_sql_query(\"\"\"\n",
    "SELECT Score, Text_not_included\n",
    "FROM reviews\n",
    "\"\"\", con)[:12000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "hHf3fALWe1l8"
   },
   "outputs": [],
   "source": [
    "for i, seq in enumerate(final['Text_not_included']):\n",
    "  final['Text_not_included'][i]=final['Text_not_included'][i].decode('UTF-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "nk_-OJdQe5Zl"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train , y_test = train_test_split(final['Text_not_included'], final['Score'], test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2537,
     "status": "ok",
     "timestamp": 1527592656076,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "vroxoh-WgF68",
    "outputId": "c34f3011-1c1b-41e6-ce22-5f8ccffb0ac4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'float'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=5,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "        stop_words=None, strip_accents=None, sublinear_tf=False,\n",
       "        token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 42,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer(ngram_range=(1,2), min_df=5   , dtype=float) #play around with this min_df\n",
    "vectorizer.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "kJW4hzt0j4xV"
   },
   "outputs": [],
   "source": [
    "train_vectors = vectorizer.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1149,
     "status": "ok",
     "timestamp": 1527592658976,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "Pj027CI5kFbg",
    "outputId": "4d1673d3-159f-4016-a7c8-9932a40a4144"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9600, 13399)"
      ]
     },
     "execution_count": 44,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_vectors.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "fDUTJF7He9_D"
   },
   "outputs": [],
   "source": [
    "# Oversampling train set\n",
    "over_sampler = SMOTE(ratio='minority')\n",
    "X_train_resampled, y_train_resampled = over_sampler.fit_sample(train_vectors, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1221,
     "status": "ok",
     "timestamp": 1527592666489,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "Hvu7zFkTkn3t",
    "outputId": "7fabd04c-9958-42a1-9400-2ddc7500b582"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16006, 13399)"
      ]
     },
     "execution_count": 46,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_resampled.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BkeBv8ick47D"
   },
   "outputs": [],
   "source": [
    "test_vectors = vectorizer.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "yAdhujZ8mF1x"
   },
   "outputs": [],
   "source": [
    "tuned_parameters = [{'C': np.linspace(1.00000000e-05, 1.66581253e+00, 100, dtype=float)}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "S5ebb49T8Hsm"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler=StandardScaler(with_mean=False)\n",
    "scaler.fit(X_train_resampled)\n",
    "X_train_scaled = scaler.transform(X_train_resampled)\n",
    "X_test_scaled = scaler.transform(test_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ww3jwKZ5tY6A"
   },
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "lgLJEbSWt7lp"
   },
   "outputs": [],
   "source": [
    "model = LogisticRegression(penalty='l2')\n",
    "\n",
    "#Using GridSearchCV\n",
    "gscv = GridSearchCV(model, tuned_parameters, scoring = 'accuracy', cv=5)\n",
    "\n",
    "t0 = datetime.now()\n",
    "print(gscv.fit(X_train_scaled, y_train_resampled))\n",
    "t1=datetime.now()\n",
    "\n",
    "print(\"Execution time = {}\".format(t1-t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1031,
     "status": "ok",
     "timestamp": 1527581648826,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "pdX60pBjYE1-",
    "outputId": "f084593f-9ce2-4924-d9c4-a0f1b8a55ad5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.01683628818181818, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='ovr', n_jobs=1, penalty='l2', random_state=None,\n",
       "          solver='liblinear', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 50,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2937,
     "status": "ok",
     "timestamp": 1527581651886,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "qABJy_AdNqBV",
    "outputId": "4b01d4c8-6f95-4c6e-ddb1-993be91c6559"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.01683628818181818, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='ovr', n_jobs=1, penalty='l2', random_state=None,\n",
       "          solver='liblinear', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 51,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv.best_estimator_.fit(X_train_scaled, y_train_resampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "FGdVxwNvNwcK"
   },
   "outputs": [],
   "source": [
    "predictions = gscv.best_estimator_.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BtduYJhfN7dd"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1170,
     "status": "ok",
     "timestamp": 1527581655186,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "FW-qlux_OZ7b",
    "outputId": "4c6f24bc-87fb-4ff4-ba7b-f4ab92f2b5ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.69      0.63      0.66       733\n",
      "   positive       0.93      0.95      0.94      4058\n",
      "\n",
      "avg / total       0.90      0.90      0.90      4791\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1247,
     "status": "ok",
     "timestamp": 1527581745065,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "zKctXiMzThNy",
    "outputId": "31323506-c505-4f20-d868-46412053de70"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 465  207]\n",
      " [ 268 3851]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1477,
     "status": "ok",
     "timestamp": 1527581748053,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "SHn5N7tLT5mq",
    "outputId": "de5d1610-6189-4f81-9a98-10ea7f682868"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR = 0.948989650073928\n",
      " TNR = 0.6343792633015006\n",
      " FPR = 0.3656207366984993\n",
      " FNR = 0.05101034992607196\n"
     ]
    }
   ],
   "source": [
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ylCnOcYwOmJa"
   },
   "outputs": [],
   "source": [
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from scipy.stats import uniform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "YX-EQlZCRHts"
   },
   "outputs": [],
   "source": [
    "tuned_parameters = {'C' : uniform(1.00000000e-05, 1.66581253e+00)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 887084,
     "status": "ok",
     "timestamp": 1527582650520,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "MaKU1VBTQkh6",
    "outputId": "ef9287be-6583-4936-8859-2a783e3dc35d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomizedSearchCV(cv=5, error_score='raise',\n",
      "          estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False),\n",
      "          fit_params={}, iid=True, n_iter=100, n_jobs=1,\n",
      "          param_distributions={'C': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fd85d3c4d30>},\n",
      "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
      "          scoring='accuracy', verbose=0)\n",
      "Execution time = 0:14:46.027157\n"
     ]
    }
   ],
   "source": [
    "rscv = RandomizedSearchCV(model, tuned_parameters, scoring = 'accuracy', cv=5, n_iter=100)\n",
    "\n",
    "t0=datetime.now()\n",
    "print(rscv.fit(X_train_scaled, y_train_resampled))\n",
    "t1=datetime.now()\n",
    "\n",
    "print(\"Execution time = {}\".format(t1-t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1069,
     "status": "ok",
     "timestamp": 1527582651741,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "q128pJm6XiEZ",
    "outputId": "20dbf909-1895-4820-a62b-7262547ece21"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.033579382735105565, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='ovr', n_jobs=1, penalty='l2', random_state=None,\n",
       "          solver='liblinear', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 61,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rscv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1004,
     "status": "ok",
     "timestamp": 1527582652925,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "PtMN-ESjYGuk",
    "outputId": "f7a3f6f4-e394-482a-9f01-fbbb188160c7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.68      0.63      0.66       733\n",
      "   positive       0.93      0.95      0.94      4058\n",
      "\n",
      "avg / total       0.90      0.90      0.90      4791\n",
      "\n",
      "[[ 464  214]\n",
      " [ 269 3844]]\n"
     ]
    }
   ],
   "source": [
    "predictions = rscv.best_estimator_.predict(X_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1081,
     "status": "ok",
     "timestamp": 1527582654182,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "NKNhggfPcujh",
    "outputId": "cec71e65-da60-4106-a0f6-52193c1d9644"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPR = 0.9472646623952686\n",
      " TNR = 0.6330150068212824\n",
      " FPR = 0.3669849931787176\n",
      " FNR = 0.0527353376047314\n"
     ]
    }
   ],
   "source": [
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h7qipoqxdco_"
   },
   "source": [
    "## Remarks\n",
    "Huge improvement in performance over Naive Bayes (note the improved TNR) and not much difference between GridSearch and RandomSearch although the latter is somewhat faster (note the time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5pQsKsU0B76N"
   },
   "source": [
    "## Performance with L1 regulariser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 168356,
     "status": "ok",
     "timestamp": 1527583206601,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "ndd53OFHCBMZ",
    "outputId": "d8725d5e-1a4e-4255-d207-492ed3098d37"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.71      0.71       733\n",
      "   positive       0.95      0.95      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 522  220]\n",
      " [ 211 3838]]\n",
      "\n",
      "\n",
      "TPR = 0.9457861015278463\n",
      " TNR = 0.7121418826739427\n",
      " FPR = 0.2878581173260573\n",
      " FNR = 0.05421389847215377\n"
     ]
    }
   ],
   "source": [
    "rscv = RandomizedSearchCV(LogisticRegression(penalty='l1'), tuned_parameters, scoring = 'accuracy', cv=5, n_iter=50)\n",
    "rscv.fit(X_train_scaled, y_train_resampled)\n",
    "predictions = rscv.best_estimator_.predict(X_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(\"\\n\")\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "print(\"\\n\")\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CajgLShzDmyd"
   },
   "source": [
    "L1 regulariser outperforms L2 regulariser (higher TNR) only with 50 iterations. When done with 100 iterations of random search, this result is bound to improve."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tQpB917FDKwb"
   },
   "source": [
    "## Effect of increasing lambda in L1 regularised Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1384,
     "status": "ok",
     "timestamp": 1527583336148,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "YurcZ0u6C1qC",
    "outputId": "c27a5b05-6f9a-4ad4-a4ac-07578eed73d7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.4158263696229512, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='ovr', n_jobs=1, penalty='l1', random_state=None,\n",
       "          solver='liblinear', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 65,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rscv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "p8AVR9obEESz"
   },
   "outputs": [],
   "source": [
    "best_C = rscv.best_estimator_.C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 2754
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7706,
     "status": "ok",
     "timestamp": 1527584648982,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "owIf5b0QD-EM",
    "outputId": "e6da53e8-7c1b-427b-eed9-9127e85fcc65"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "results for 1.5 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.72      0.71       733\n",
      "   positive       0.95      0.95      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 527  221]\n",
      " [ 206 3837]]\n",
      "\n",
      "\n",
      "TPR = 0.9455396747166092\n",
      " TNR = 0.7189631650750341\n",
      " FPR = 0.2810368349249659\n",
      " FNR = 0.054460325283390836\n",
      "number of nonzero components = 6181\n",
      "sparsity = 0.7694258962211362\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n",
      "results for 2.0 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.71      0.72      0.71       733\n",
      "   positive       0.95      0.95      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 526  220]\n",
      " [ 207 3838]]\n",
      "\n",
      "\n",
      "TPR = 0.9457861015278463\n",
      " TNR = 0.7175989085948158\n",
      " FPR = 0.28240109140518416\n",
      " FNR = 0.05421389847215377\n",
      "number of nonzero components = 6032\n",
      "sparsity = 0.7749841459320327\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n",
      "results for 2.5 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.71      0.71       733\n",
      "   positive       0.95      0.94      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 524  225]\n",
      " [ 209 3833]]\n",
      "\n",
      "\n",
      "TPR = 0.9445539674716609\n",
      " TNR = 0.7148703956343793\n",
      " FPR = 0.28512960436562074\n",
      " FNR = 0.05544603252833908\n",
      "number of nonzero components = 5900\n",
      "sparsity = 0.7799082329242362\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n",
      "results for 3.0 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.72      0.71       733\n",
      "   positive       0.95      0.94      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 527  224]\n",
      " [ 206 3834]]\n",
      "\n",
      "\n",
      "TPR = 0.944800394282898\n",
      " TNR = 0.7189631650750341\n",
      " FPR = 0.2810368349249659\n",
      " FNR = 0.05519960571710202\n",
      "number of nonzero components = 5743\n",
      "sparsity = 0.7857649121498116\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n",
      "results for 3.5 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.72      0.71       733\n",
      "   positive       0.95      0.95      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 529  223]\n",
      " [ 204 3835]]\n",
      "\n",
      "\n",
      "TPR = 0.9450468210941351\n",
      " TNR = 0.7216916780354706\n",
      " FPR = 0.2783083219645293\n",
      " FNR = 0.05495317890586496\n",
      "number of nonzero components = 5711\n",
      "sparsity = 0.7869586302085276\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n",
      "results for 4.0 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.72      0.71       733\n",
      "   positive       0.95      0.95      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 530  222]\n",
      " [ 203 3836]]\n",
      "\n",
      "\n",
      "TPR = 0.9452932479053721\n",
      " TNR = 0.723055934515689\n",
      " FPR = 0.27694406548431105\n",
      " FNR = 0.05470675209462789\n",
      "number of nonzero components = 5624\n",
      "sparsity = 0.7902040511806617\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n",
      "results for 4.5 times best lambda\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.70      0.72      0.71       733\n",
      "   positive       0.95      0.94      0.95      4058\n",
      "\n",
      "avg / total       0.91      0.91      0.91      4791\n",
      "\n",
      "\n",
      "\n",
      "[[ 528  225]\n",
      " [ 205 3833]]\n",
      "\n",
      "\n",
      "TPR = 0.9445539674716609\n",
      " TNR = 0.7203274215552524\n",
      " FPR = 0.27967257844474763\n",
      " FNR = 0.05544603252833908\n",
      "number of nonzero components = 5555\n",
      "sparsity = 0.7927780057447682\n",
      "\n",
      "**************************************************************************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(3, 10, 1):\n",
    "  C = best_C/float(i/2)\n",
    "  model = LogisticRegression(C=C, penalty = 'l1')\n",
    "  model.fit(X_train_scaled, y_train_resampled)\n",
    "  predictions = model.predict(X_test_scaled)\n",
    "  print(\"results for {} times best lambda\".format(i/2))\n",
    "  print(classification_report(y_test, predictions))\n",
    "  print(\"\\n\")\n",
    "  print(confusion_matrix(y_test, predictions).T)\n",
    "  tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "  print(\"\\n\")\n",
    "  print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))\n",
    "  \n",
    "  print(\"number of nonzero components = {}\".format(np.count_nonzero(model.coef_)))\n",
    "  print(\"sparsity = {}\".format(float((len(vectorizer.get_feature_names())-np.count_nonzero(model.coef_))/len(vectorizer.get_feature_names()))))\n",
    "  print('\\n**************************************************************************************************************\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eBRrS8KVXhpX"
   },
   "source": [
    "## Remarks\n",
    "With increase in lambda (decrease in C) number of nonzero components decrease however there is no appreciable change in performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "F9bSxGfzXwq9"
   },
   "source": [
    "## Check for multicollinearity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "sGvvpUfPGCUh"
   },
   "outputs": [],
   "source": [
    "noise = np.reshape(np.random.normal(loc=0, scale=0.01, size=X_train_scaled.get_shape()[0]*X_train_scaled.get_shape()[1]), X_train_scaled.get_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "aWs2KaVlYtr6"
   },
   "outputs": [],
   "source": [
    "#X_noisy = X_train_scaled.todense() + noise\n",
    "\n",
    "model = LogisticRegression(C=0.0168)\n",
    "model.fit(X_train_scaled, y_train_resampled)\n",
    "\n",
    "w_non_noisy = model.coef_\n",
    "\n",
    "#model.fit(X_noisy, y_train_resampled)\n",
    "#w_noisy = model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "FjEK_nuXi15k"
   },
   "outputs": [],
   "source": [
    "diff = np.linalg.norm(w_non_noisy-w_noisy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1349,
     "status": "ok",
     "timestamp": 1527592305798,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "SEgRI3Dul7Ji",
    "outputId": "fe6dfae1-5b0f-41c5-f9c6-b42ed9357839"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.6978174895120945"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(w_non_noisy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 759,
     "status": "ok",
     "timestamp": 1527592324044,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "Zr-9YRx1mL7R",
    "outputId": "3cd2a093-2c3d-4b8a-e706-bfc709ff09f1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.698050250116066"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(w_noisy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1137,
     "status": "ok",
     "timestamp": 1527592325917,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "2k6riY1AmQhM",
    "outputId": "70df6a4b-2188-4870-c068-12363a5f7882"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.04061884783793817"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hZC_b6zLl_eQ"
   },
   "source": [
    "## Remarks\n",
    "Since difference vector has very small magnitude as compared to w_non_noisy we can conclude, there is very low multicollinearity between features<br>\n",
    "<br>\n",
    "## Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "m2S0SQg3m1we"
   },
   "outputs": [],
   "source": [
    "important = np.abs(w_non_noisy[0]).argsort()[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "wzNjxNpbnB0i"
   },
   "outputs": [],
   "source": [
    "# top 100 important feature names\n",
    "imp = np.array(vectorizer.get_feature_names())[important]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 374
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1059,
     "status": "ok",
     "timestamp": 1527593633636,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "CSrjOEMOogQY",
    "outputId": "f711b01e-e5f4-4fb5-afb4-f3a3b7cb6cc1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['standard poodl', 'easi grow', 'love brand', 'coffe creamer',\n",
       "       'mash', 'easi find', 'not total', 'ziplock bag', 'pulp',\n",
       "       'texa bbq', 'pleas use', 'supplier', 'experi', 'also moder',\n",
       "       'also keep', 'brew tea', 'like ive', 'sweeter', 'chip product',\n",
       "       'packag one', 'not requir', 'loos stool', 'premium brand', 'bounc',\n",
       "       'food help', 'west coast', 'nice tast', 'eat howev', 'bait',\n",
       "       'vitamin supplement', 'msg artifici', 'tri hook', 'drastic',\n",
       "       'keebler', 'boil water', 'sever varieti', 'delici healthi',\n",
       "       'bag make', 'amazon quick', 'week ago', 'pet', 'new formula',\n",
       "       'enjoy best', 'not firm', 'bun', 'serv packag', 'amount water',\n",
       "       'cost bit', 'enjoy coffe', 'price definit', 'treat pet',\n",
       "       'recommend item', 'chihuahua', 'pack make', 'know much',\n",
       "       'like mani', 'littl honey', 'good old', 'order futur', 'far far',\n",
       "       'husband absolut', 'didnt mind', 'seed butter', 'began search',\n",
       "       'oili', 'know well', 'good packag', 'anyth ive', 'urin', 'bush',\n",
       "       'price make', 'tuna', 'minti', 'live room', 'use type', 'versus',\n",
       "       'product tri', 'almost not', 'seattl', 'mail', 'break apart',\n",
       "       'line organ', 'think get', 'formul', 'older one', 'good like',\n",
       "       'food yet', 'ect', 'first tast', 'bacon', 'health issu', 'includ',\n",
       "       'carri happi', 'sure expect', 'twine english', 'cooki best',\n",
       "       'also say', 'unhealthi', 'snack bag', 'use need'], dtype='<U22')"
      ]
     },
     "execution_count": 82,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "Apply Logistic regression to Amazon reviews data set. [M].ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}