Apply k-NN on Amazon reviews dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "con = sqlite3.connect('./clean_final') # this is cleaned dataset\n",
    "final = pd.read_sql_query(\"\"\"\n",
    "SELECT *\n",
    "FROM reviews\n",
    "\"\"\", con)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanhtml(sentence): #function to clean the word of any html-tags\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, ' ', sentence)\n",
    "    return cleantext\n",
    "def cleanpunc(sentence): #function to clean the word of any punctuation or special characters\n",
    "    cleaned = re.sub(r'[?|!|\\'|\"|#]',r'',sentence)\n",
    "    cleaned = re.sub(r'[.|,|)|(|\\|/]',r' ',cleaned)\n",
    "    return  cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level_0</th>\n",
       "      <th>index</th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_not_included</th>\n",
       "      <th>Text_not_excluded</th>\n",
       "      <th>clean_review</th>\n",
       "      <th>stemmed_review</th>\n",
       "      <th>average_Word2Vec</th>\n",
       "      <th>tfidf_Word2Vec</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10618</td>\n",
       "      <td>16395</td>\n",
       "      <td>17899</td>\n",
       "      <td>B001EO653M</td>\n",
       "      <td>A21BMUL4UEY53N</td>\n",
       "      <td>chocoholic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>positive</td>\n",
       "      <td>1294444800</td>\n",
       "      <td>Can't get much better than this.</td>\n",
       "      <td>This awesome mix is not your usual Hot Chocola...</td>\n",
       "      <td>b'awesom mix not usual hot chocol look not buy...</td>\n",
       "      <td>b'awesom mix usual hot chocol look buy product...</td>\n",
       "      <td>this awesome mix not your usual hot chocolate ...</td>\n",
       "      <td>b'this awesom mix not your usual hot chocol th...</td>\n",
       "      <td>b'\\xfa\\x82\\xbe\\xf4)O\\xe0\\xbf\\xd0\\x17t\\xdeO\\x94...</td>\n",
       "      <td>b'\\xb6\\xbdl\\xac~\\xb7\\xdd\\xbfXI\\xf0;\\x04&amp;\\xa4?\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5581</td>\n",
       "      <td>23132</td>\n",
       "      <td>25312</td>\n",
       "      <td>B000MDCDOG</td>\n",
       "      <td>A1E1AX7DE5S18X</td>\n",
       "      <td>Nirac</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>positive</td>\n",
       "      <td>1288224000</td>\n",
       "      <td>Wonderful...Too Expensive Here</td>\n",
       "      <td>I've never been a fan of tuna, however, after ...</td>\n",
       "      <td>b'ive never fan tuna howev tri sundri tomato b...</td>\n",
       "      <td>b'ive never fan tuna howev tri sundri tomato b...</td>\n",
       "      <td>ive never been fan tuna however after trying t...</td>\n",
       "      <td>b'ive never been fan tuna howev after tri this...</td>\n",
       "      <td>b'\\xd1E\\x17\\x914\\x9f\\xdd\\xbft\\xd1E\\x85\\xd8\\xc5...</td>\n",
       "      <td>b'\\xfc\\xd1jVr)\\xda\\xbfK]9*0\\x89\\x95?\\x06$\\xc2[...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10460</td>\n",
       "      <td>17867</td>\n",
       "      <td>19477</td>\n",
       "      <td>B001EO5PWO</td>\n",
       "      <td>A2YAHDOYQK3JJJ</td>\n",
       "      <td>Roger D. Lauzon \"coaterfreak\"</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>positive</td>\n",
       "      <td>1285459200</td>\n",
       "      <td>great spice</td>\n",
       "      <td>these are the same spices thare used on the fo...</td>\n",
       "      <td>b'spice thare use food network alway fresh gre...</td>\n",
       "      <td>b'spice thare use food network alway fresh gre...</td>\n",
       "      <td>these are the same spices thare used the food ...</td>\n",
       "      <td>b'these are the same spice thare use the food ...</td>\n",
       "      <td>b'\\xdb\\xb6m\\xfb\\x9f\\xa5\\xe3\\xbf%I\\x92$/G\\xa1?I...</td>\n",
       "      <td>b\"\\xf75\\xd4\\x9b\\xf1m\\xe3\\xbfo\\xf0m\\xa5\\xb0l\\xa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6356</td>\n",
       "      <td>8644</td>\n",
       "      <td>9475</td>\n",
       "      <td>B000UBDKII</td>\n",
       "      <td>A2VRSADIZLOAU3</td>\n",
       "      <td>Anne Cummings \"annebanana\"</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>positive</td>\n",
       "      <td>1278115200</td>\n",
       "      <td>YUM</td>\n",
       "      <td>Great coffee.  We mix it with Godiva chocolate...</td>\n",
       "      <td>b'great coffe mix godiva chocol coffe regular ...</td>\n",
       "      <td>b'great coffe mix godiva chocol coffe regular ...</td>\n",
       "      <td>great coffee mix with godiva chocolate coffee ...</td>\n",
       "      <td>b'great coffe mix with godiva chocol coffe reg...</td>\n",
       "      <td>b\"\\x1c\\xc7q\\x8c\\xa0\\xa1\\xe2\\xbf\\xe48\\x8e\\xd3\\x...</td>\n",
       "      <td>b'\\xc8\\x16\\x8f\\xab\\xc6K\\xe2\\xbfB\\xb2d\\x8a\\xe2\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18558</td>\n",
       "      <td>15674</td>\n",
       "      <td>17140</td>\n",
       "      <td>B00408F3XC</td>\n",
       "      <td>A1RWB3QBIPL85Y</td>\n",
       "      <td>S. Largent \"Digital XTC\"</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>positive</td>\n",
       "      <td>1329264000</td>\n",
       "      <td>happy valentines day</td>\n",
       "      <td>delivered on time for v day, packaged very wel...</td>\n",
       "      <td>b'deliv time day packag well instruct fresh be...</td>\n",
       "      <td>b'deliv time day packag well instruct fresh be...</td>\n",
       "      <td>delivered time for day packaged very well with...</td>\n",
       "      <td>b'deliv time for day packag veri well with ins...</td>\n",
       "      <td>b'\\xab\\xaa\\xaaz\\xcc\\xb6\\xe0\\xbf\\xab\\xaa\\xea\\x8...</td>\n",
       "      <td>b'\\x00\\x8c0\\xb7\\x1d\\xbb\\xde\\xbfE\\\\\\x11VGM\\x95?...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   level_0  index     Id   ProductId          UserId  \\\n",
       "0    10618  16395  17899  B001EO653M  A21BMUL4UEY53N   \n",
       "1     5581  23132  25312  B000MDCDOG  A1E1AX7DE5S18X   \n",
       "2    10460  17867  19477  B001EO5PWO  A2YAHDOYQK3JJJ   \n",
       "3     6356   8644   9475  B000UBDKII  A2VRSADIZLOAU3   \n",
       "4    18558  15674  17140  B00408F3XC  A1RWB3QBIPL85Y   \n",
       "\n",
       "                     ProfileName  HelpfulnessNumerator  \\\n",
       "0                     chocoholic                     0   \n",
       "1                          Nirac                     0   \n",
       "2  Roger D. Lauzon \"coaterfreak\"                     0   \n",
       "3     Anne Cummings \"annebanana\"                     0   \n",
       "4       S. Largent \"Digital XTC\"                     0   \n",
       "\n",
       "   HelpfulnessDenominator     Score        Time  \\\n",
       "0                       0  positive  1294444800   \n",
       "1                       0  positive  1288224000   \n",
       "2                       0  positive  1285459200   \n",
       "3                       0  positive  1278115200   \n",
       "4                       0  positive  1329264000   \n",
       "\n",
       "                            Summary  \\\n",
       "0  Can't get much better than this.   \n",
       "1    Wonderful...Too Expensive Here   \n",
       "2                       great spice   \n",
       "3                               YUM   \n",
       "4              happy valentines day   \n",
       "\n",
       "                                                Text  \\\n",
       "0  This awesome mix is not your usual Hot Chocola...   \n",
       "1  I've never been a fan of tuna, however, after ...   \n",
       "2  these are the same spices thare used on the fo...   \n",
       "3  Great coffee.  We mix it with Godiva chocolate...   \n",
       "4  delivered on time for v day, packaged very wel...   \n",
       "\n",
       "                                   Text_not_included  \\\n",
       "0  b'awesom mix not usual hot chocol look not buy...   \n",
       "1  b'ive never fan tuna howev tri sundri tomato b...   \n",
       "2  b'spice thare use food network alway fresh gre...   \n",
       "3  b'great coffe mix godiva chocol coffe regular ...   \n",
       "4  b'deliv time day packag well instruct fresh be...   \n",
       "\n",
       "                                   Text_not_excluded  \\\n",
       "0  b'awesom mix usual hot chocol look buy product...   \n",
       "1  b'ive never fan tuna howev tri sundri tomato b...   \n",
       "2  b'spice thare use food network alway fresh gre...   \n",
       "3  b'great coffe mix godiva chocol coffe regular ...   \n",
       "4  b'deliv time day packag well instruct fresh be...   \n",
       "\n",
       "                                        clean_review  \\\n",
       "0  this awesome mix not your usual hot chocolate ...   \n",
       "1  ive never been fan tuna however after trying t...   \n",
       "2  these are the same spices thare used the food ...   \n",
       "3  great coffee mix with godiva chocolate coffee ...   \n",
       "4  delivered time for day packaged very well with...   \n",
       "\n",
       "                                      stemmed_review  \\\n",
       "0  b'this awesom mix not your usual hot chocol th...   \n",
       "1  b'ive never been fan tuna howev after tri this...   \n",
       "2  b'these are the same spice thare use the food ...   \n",
       "3  b'great coffe mix with godiva chocol coffe reg...   \n",
       "4  b'deliv time for day packag veri well with ins...   \n",
       "\n",
       "                                    average_Word2Vec  \\\n",
       "0  b'\\xfa\\x82\\xbe\\xf4)O\\xe0\\xbf\\xd0\\x17t\\xdeO\\x94...   \n",
       "1  b'\\xd1E\\x17\\x914\\x9f\\xdd\\xbft\\xd1E\\x85\\xd8\\xc5...   \n",
       "2  b'\\xdb\\xb6m\\xfb\\x9f\\xa5\\xe3\\xbf%I\\x92$/G\\xa1?I...   \n",
       "3  b\"\\x1c\\xc7q\\x8c\\xa0\\xa1\\xe2\\xbf\\xe48\\x8e\\xd3\\x...   \n",
       "4  b'\\xab\\xaa\\xaaz\\xcc\\xb6\\xe0\\xbf\\xab\\xaa\\xea\\x8...   \n",
       "\n",
       "                                      tfidf_Word2Vec  \n",
       "0  b'\\xb6\\xbdl\\xac~\\xb7\\xdd\\xbfXI\\xf0;\\x04&\\xa4?\\...  \n",
       "1  b'\\xfc\\xd1jVr)\\xda\\xbfK]9*0\\x89\\x95?\\x06$\\xc2[...  \n",
       "2  b\"\\xf75\\xd4\\x9b\\xf1m\\xe3\\xbfo\\xf0m\\xa5\\xb0l\\xa...  \n",
       "3  b'\\xc8\\x16\\x8f\\xab\\xc6K\\xe2\\xbfB\\xb2d\\x8a\\xe2\\...  \n",
       "4  b'\\x00\\x8c0\\xb7\\x1d\\xbb\\xde\\xbfE\\\\\\x11VGM\\x95?...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From previous exercise we know that excluding 'not' from set of stopwords improves separation between positive and negative reviews. At least for TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sno = nltk.stem.SnowballStemmer('english') #initialising the snowball stemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Stem 'clean_review'\n",
    "\n",
    "s=None\n",
    "\n",
    "stemmed_review=[]\n",
    "for sent in final['clean_review'].values:\n",
    "    \n",
    "    stems=[]\n",
    "    for w in sent.split():    \n",
    "                \n",
    "        stems.append(sno.stem(w).encode('utf8'))\n",
    "    \n",
    "    s=b' '.join(stems)\n",
    "    \n",
    "    \n",
    "    stemmed_review.append(s.decode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "final['stemmed_review']=stemmed_review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    this awesom mix not your usual hot chocol that...\n",
       "1    ive never been fan tuna howev after tri this s...\n",
       "2    these are the same spice thare use the food ne...\n",
       "3    great coffe mix with godiva chocol coffe regul...\n",
       "4    deliv time for day packag veri well with instr...\n",
       "Name: stemmed_review, dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final['stemmed_review'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    10618\n",
       "1     5581\n",
       "2    10460\n",
       "3     6356\n",
       "4    18558\n",
       "Name: level_0, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final['level_0'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db=sqlite3.connect('./clean_final.sqlite')\n",
    "cur=db.cursor()\n",
    "db.text_factory=str\n",
    "final.to_sql('reviews', db, if_exists='replace', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Randomly sample 2500 points\n",
    "import random\n",
    "final=final.iloc[random.sample(range(len(final)), 2500)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "time_based_split=final.sort_values('Time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "time_based_split.Text_not_included=[s.decode() for s in time_based_split.Text_not_included.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "time_based_split.Text_not_excluded=[s.decode() for s in time_based_split.Text_not_excluded.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "time_based_split=time_based_split.drop('level_0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_not_included</th>\n",
       "      <th>Text_not_excluded</th>\n",
       "      <th>clean_review</th>\n",
       "      <th>stemmed_review</th>\n",
       "      <th>average_Word2Vec</th>\n",
       "      <th>tfidf_Word2Vec</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1866</th>\n",
       "      <td>10992</td>\n",
       "      <td>11991</td>\n",
       "      <td>B0000T15M8</td>\n",
       "      <td>A2928LJN5IISB4</td>\n",
       "      <td>chatchi</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>positive</td>\n",
       "      <td>1067990400</td>\n",
       "      <td>The fruits of my labor</td>\n",
       "      <td>When chatchi's favorite afternoon treat became...</td>\n",
       "      <td>chatchi favorit afternoon treat becam mysteri ...</td>\n",
       "      <td>chatchi favorit afternoon treat becam mysteri ...</td>\n",
       "      <td>when chatchis favorite afternoon treat became ...</td>\n",
       "      <td>when chatchi favorit afternoon treat becam mys...</td>\n",
       "      <td>b'333v==\\xdb\\xbf\\xef\\xee\\xee\\xf9\\xf8\\xc4\\x9a?\\...</td>\n",
       "      <td>b'/\\x0b\\xfb+\\r\\xd4\\xce\\xbfk\";a1\\x1c\\x95?Uqc\\xd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1793</th>\n",
       "      <td>6790</td>\n",
       "      <td>7432</td>\n",
       "      <td>B0001E1IME</td>\n",
       "      <td>A2IKCTD1I73PLW</td>\n",
       "      <td>Adeba</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>positive</td>\n",
       "      <td>1083456000</td>\n",
       "      <td>amazon monopoly/ripoff</td>\n",
       "      <td>love the snack.  wanted to buy a bunch.&lt;p&gt;ship...</td>\n",
       "      <td>love snack want buy bunch ship amazon price sn...</td>\n",
       "      <td>love snack want buy bunch ship amazon price sn...</td>\n",
       "      <td>love the snack wanted buy bunch shipping amazo...</td>\n",
       "      <td>love the snack want buy bunch ship amazon more...</td>\n",
       "      <td>b'\\x9a\\x99\\x99M\\x1e\\n\\xe2\\xbf\\xa8\\x83:t\\xec\\x8...</td>\n",
       "      <td>b'^\\x15\\xf3\\xe1X\\x08\\xe2\\xbf\\x11\\xe8W\\x13\\xa9\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>1112</td>\n",
       "      <td>1206</td>\n",
       "      <td>B005O072PC</td>\n",
       "      <td>A3BD5B8Y8MY25X</td>\n",
       "      <td>J. L. K. \"special_k\"</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>positive</td>\n",
       "      <td>1088467200</td>\n",
       "      <td>The best twice baked potatoes you'll ever have!</td>\n",
       "      <td>Perfect year round with just about any meat di...</td>\n",
       "      <td>perfect year round meat dish omaha steak twice...</td>\n",
       "      <td>perfect year round meat dish omaha steak twice...</td>\n",
       "      <td>perfect year round with just about any meat di...</td>\n",
       "      <td>perfect year round with just about ani meat di...</td>\n",
       "      <td>b'F\\x17]\\xec\\x87G\\xdb\\xbf]t\\x11\\xd7\\xd3\\x87\\x9...</td>\n",
       "      <td>b'w^\\x1f\\xedP\\x16\\xd9\\xbf\\xe3\\xef\\xd6\\xe0\\x96\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>14096</td>\n",
       "      <td>15390</td>\n",
       "      <td>B0002IMRTC</td>\n",
       "      <td>ALKOHQH542KO</td>\n",
       "      <td>G. Elliott</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>positive</td>\n",
       "      <td>1090281600</td>\n",
       "      <td>Peet says it's good.</td>\n",
       "      <td>So it must be good.  He never lies to me.  Eve...</td>\n",
       "      <td>must good never lie even skizip like drink str...</td>\n",
       "      <td>must good never lie even skizip like drink str...</td>\n",
       "      <td>must good never lies even skizip likes and all...</td>\n",
       "      <td>must good never lie even skizip like and all d...</td>\n",
       "      <td>b'333\\xfbR\\xa9\\xdf\\xbf\\xcd\\xcc\\xcc\\x92\\xbc\\xbb...</td>\n",
       "      <td>b'\\x1fU\\x91\\x9d\\xda{\\xdc\\xbf\\xcc\\xc1\\xba\\x88\\x...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>659</th>\n",
       "      <td>5886</td>\n",
       "      <td>6374</td>\n",
       "      <td>B000084EKB</td>\n",
       "      <td>A1Z54EM24Y40LL</td>\n",
       "      <td>c2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>positive</td>\n",
       "      <td>1091059200</td>\n",
       "      <td>Great food!</td>\n",
       "      <td>This version is ground.  NOt my current cat's ...</td>\n",
       "      <td>version ground not current cat favorit version...</td>\n",
       "      <td>version ground current cat favorit version rea...</td>\n",
       "      <td>this version ground not current cats favorite ...</td>\n",
       "      <td>this version ground not current cat favorit ve...</td>\n",
       "      <td>b'\\xc9g\\xddl\\x90\\xf4\\xdf\\xbf\\xc2\\x14\\xf9\\x04\\x...</td>\n",
       "      <td>b'3\\xc2\\t\\x1c\\xa8\\xf6\\xdd\\xbfc9\\xf4\\x95\\xe1\\r\\...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      index     Id   ProductId          UserId           ProfileName  \\\n",
       "1866  10992  11991  B0000T15M8  A2928LJN5IISB4               chatchi   \n",
       "1793   6790   7432  B0001E1IME  A2IKCTD1I73PLW                 Adeba   \n",
       "330    1112   1206  B005O072PC  A3BD5B8Y8MY25X  J. L. K. \"special_k\"   \n",
       "365   14096  15390  B0002IMRTC    ALKOHQH542KO            G. Elliott   \n",
       "659    5886   6374  B000084EKB  A1Z54EM24Y40LL                    c2   \n",
       "\n",
       "      HelpfulnessNumerator  HelpfulnessDenominator     Score        Time  \\\n",
       "1866                     5                       5  positive  1067990400   \n",
       "1793                     2                       8  positive  1083456000   \n",
       "330                     13                      14  positive  1088467200   \n",
       "365                      5                      15  positive  1090281600   \n",
       "659                      0                       0  positive  1091059200   \n",
       "\n",
       "                                              Summary  \\\n",
       "1866                           The fruits of my labor   \n",
       "1793                           amazon monopoly/ripoff   \n",
       "330   The best twice baked potatoes you'll ever have!   \n",
       "365                              Peet says it's good.   \n",
       "659                                       Great food!   \n",
       "\n",
       "                                                   Text  \\\n",
       "1866  When chatchi's favorite afternoon treat became...   \n",
       "1793  love the snack.  wanted to buy a bunch.<p>ship...   \n",
       "330   Perfect year round with just about any meat di...   \n",
       "365   So it must be good.  He never lies to me.  Eve...   \n",
       "659   This version is ground.  NOt my current cat's ...   \n",
       "\n",
       "                                      Text_not_included  \\\n",
       "1866  chatchi favorit afternoon treat becam mysteri ...   \n",
       "1793  love snack want buy bunch ship amazon price sn...   \n",
       "330   perfect year round meat dish omaha steak twice...   \n",
       "365   must good never lie even skizip like drink str...   \n",
       "659   version ground not current cat favorit version...   \n",
       "\n",
       "                                      Text_not_excluded  \\\n",
       "1866  chatchi favorit afternoon treat becam mysteri ...   \n",
       "1793  love snack want buy bunch ship amazon price sn...   \n",
       "330   perfect year round meat dish omaha steak twice...   \n",
       "365   must good never lie even skizip like drink str...   \n",
       "659   version ground current cat favorit version rea...   \n",
       "\n",
       "                                           clean_review  \\\n",
       "1866  when chatchis favorite afternoon treat became ...   \n",
       "1793  love the snack wanted buy bunch shipping amazo...   \n",
       "330   perfect year round with just about any meat di...   \n",
       "365   must good never lies even skizip likes and all...   \n",
       "659   this version ground not current cats favorite ...   \n",
       "\n",
       "                                         stemmed_review  \\\n",
       "1866  when chatchi favorit afternoon treat becam mys...   \n",
       "1793  love the snack want buy bunch ship amazon more...   \n",
       "330   perfect year round with just about ani meat di...   \n",
       "365   must good never lie even skizip like and all d...   \n",
       "659   this version ground not current cat favorit ve...   \n",
       "\n",
       "                                       average_Word2Vec  \\\n",
       "1866  b'333v==\\xdb\\xbf\\xef\\xee\\xee\\xf9\\xf8\\xc4\\x9a?\\...   \n",
       "1793  b'\\x9a\\x99\\x99M\\x1e\\n\\xe2\\xbf\\xa8\\x83:t\\xec\\x8...   \n",
       "330   b'F\\x17]\\xec\\x87G\\xdb\\xbf]t\\x11\\xd7\\xd3\\x87\\x9...   \n",
       "365   b'333\\xfbR\\xa9\\xdf\\xbf\\xcd\\xcc\\xcc\\x92\\xbc\\xbb...   \n",
       "659   b'\\xc9g\\xddl\\x90\\xf4\\xdf\\xbf\\xc2\\x14\\xf9\\x04\\x...   \n",
       "\n",
       "                                         tfidf_Word2Vec  \n",
       "1866  b'/\\x0b\\xfb+\\r\\xd4\\xce\\xbfk\";a1\\x1c\\x95?Uqc\\xd...  \n",
       "1793  b'^\\x15\\xf3\\xe1X\\x08\\xe2\\xbf\\x11\\xe8W\\x13\\xa9\\...  \n",
       "330   b'w^\\x1f\\xedP\\x16\\xd9\\xbf\\xe3\\xef\\xd6\\xe0\\x96\\...  \n",
       "365   b'\\x1fU\\x91\\x9d\\xda{\\xdc\\xbf\\xcc\\xc1\\xba\\x88\\x...  \n",
       "659   b'3\\xc2\\t\\x1c\\xa8\\xf6\\xdd\\xbfc9\\xf4\\x95\\xe1\\r\\...  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_based_split.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db=sqlite3.connect('./clean_final.sqlite')\n",
    "cur=db.cursor()\n",
    "db.text_factory=str\n",
    "time_based_split.to_sql('reviews', db, if_exists='replace', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_idf = TfidfVectorizer(ngram_range=(1,2))\n",
    "bigrams_tf=tf_idf.fit_transform(time_based_split['Text_not_included'])\n",
    "bigrams_train_tf=bigrams_tf[:1750]\n",
    "bigrams_test_tf=bigrams_tf[1750:2500]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average word2Vec vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "i=0\n",
    "list_of_sent=[]\n",
    "for sent in time_based_split['Text_not_included'].values:\n",
    "    filtered_sentence=[]\n",
    "    for w in sent.split():\n",
    "        filtered_sentence.append(w)\n",
    "    list_of_sent.append(filtered_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2500,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(list_of_sent).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.models.base_any2vec:consider setting layer size to a multiple of 4 for greater performance\n"
     ]
    }
   ],
   "source": [
    "w2v_model=gensim.models.Word2Vec(list_of_sent,min_count=5,size=50, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# average Word2Vec\n",
    "# compute average word2vec for each review.\n",
    "sent_vectors = []; # the avg-w2v for each sentence/review is stored in this list\n",
    "for sent in list_of_sent: # for each review/sentence\n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length\n",
    "    cnt_words =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        try:\n",
    "            vec = w2v_model.wv[word]\n",
    "            sent_vec += vec\n",
    "            cnt_words += 1\n",
    "        except:\n",
    "            pass\n",
    "    sent_vec /= cnt_words\n",
    "    sent_vectors.append(sent_vec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf_feat = tf_idf.get_feature_names() # tfidf words/col-names\n",
    "# final_tf_idf is the sparse matrix with row= sentence, col=word and cell_val = tfidf\n",
    "\n",
    "tfidf_sent_vectors = []; # the tfidf-w2v for each sentence/review is stored in this list\n",
    "row=0\n",
    "for sent in list_of_sent: # for each review/sentence\n",
    "    sent_vec = np.zeros(50) # as word vectors are of zero length\n",
    "    weight_sum =0; # num of words with a valid vector in the sentence/review\n",
    "    for word in sent: # for each word in a review/sentence\n",
    "        try:\n",
    "            vec = w2v_model.wv[word]\n",
    "            # obtain the tf_idfidf of a word in a sentence/review\n",
    "            tfidf = tf_idf_vect[row, tfidf_feat.index(word)]\n",
    "            sent_vec += (vec * tfidf)\n",
    "            weight_sum += tfidf\n",
    "        except:\n",
    "            pass\n",
    "    sent_vec /= weight_sum\n",
    "    tfidf_sent_vectors.append(sent_vec)\n",
    "    row += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.7148837 ,  0.08005671, -0.03650757,  0.28071488,  0.27651436,\n",
       "       -0.11791959,  0.19611861, -0.3056778 ,  0.33357832, -0.38980113,\n",
       "        0.38191011,  0.04367037, -0.34364644,  0.40775464, -0.60267878,\n",
       "       -0.67429486, -0.07425891, -0.38215042,  0.5923246 , -0.17593618,\n",
       "        0.39752209,  0.19266622, -0.11856697, -0.19658204,  0.60234614,\n",
       "       -0.05947708, -0.02989638, -0.02909582,  0.23693911,  0.0119263 ,\n",
       "       -0.0016133 ,  0.13247287, -0.28407203,  0.11302154, -0.16785835,\n",
       "       -0.54972464, -0.29596767, -0.43512824, -0.26735326, -0.07841764,\n",
       "        0.15785508, -0.17558218, -0.27171402,  0.04191722,  0.05334444,\n",
       "       -0.11682048, -0.24933953,  0.20622117, -0.29056247, -0.20945565])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_vectors[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['positive', 'positive', 'positive', ..., 'positive', 'positive',\n",
       "       'negative'], dtype=object)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_based_split['Score'].as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.4256128 ,  0.02614202,  0.16407779,  0.08662151,  0.17497523,\n",
       "       -0.18193097,  0.0281729 ,  0.68341499,  0.17128933, -0.28271772,\n",
       "        0.50343395, -0.16118439,  0.25213345,  0.49005483,  0.25563263,\n",
       "        0.08423763,  0.40145238,  0.37542342, -0.31176805, -0.69918819,\n",
       "        0.52160808,  0.26135075, -0.45022561,  0.01511199, -0.088507  ,\n",
       "       -0.05023587,  0.0529531 , -0.12090148,  0.1561996 , -0.26825495,\n",
       "       -0.46496859,  0.15726482,  0.60949802, -0.12534812, -0.12280237,\n",
       "       -0.09772088, -0.51514824, -0.17853833,  0.07670793,  0.21406705,\n",
       "        0.0167552 ,  0.13813511,  0.41138902,  0.05027483,  0.33643198,\n",
       "       -0.09686825, -0.15950203, -0.1730536 ,  0.45132825,  0.25198714])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_based_split.average_Word2Vec.loc[378]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_avgw2v, X_test_avgw2v, y_train_avgw2v, y_test_avgw2v = train_test_split(sent_vectors, time_based_split['Score'].as_matrix(), test_size=0.3, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_tfidfw2v, X_test_tfidfw2v, y_train_tfidfw2v, y_test_tfidfw2v = train_test_split(tfidf_sent_vectors, time_based_split['Score'].as_matrix(), test_size=0.3, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BoW vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer(ngram_range=(1,2) ) #in scikit-learn\n",
    "bigrams=count_vect.fit_transform(time_based_split['Text_not_included'])\n",
    "bigrams_train=bigrams[:1750]\n",
    "bigrams_test=bigrams[1750:2500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using average word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=np.ndarray(shape=(1750,50),dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, d in enumerate(X_train_avgw2v):\n",
    "    a[i]=d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x=[]\n",
    "y=[]\n",
    "for k in range(1,1575,2):\n",
    "    model=KNeighborsClassifier(n_neighbors=k,weights='distance',algorithm='kd_tree')\n",
    "    avg_accuracy_w=np.mean(cross_val_score(model,a, y_train_avgw2v,cv=10, scoring='accuracy'))\n",
    "    x.append(k)\n",
    "    y.append(avg_accuracy_w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.72685807583221373, 0.84400671742051059)"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE7pJREFUeJzt3X+Q3PV93/HnSxKCRhiDLTklCCPR\nEMak0xp3S0xSJxoTDGYSaKb+Q4ozmNYtnTowLTVt0NjtOHQ6DE5bMkmIHdy6bj0OikqdRHHtEa4M\nnWlHdbSUXxZE9hm7cGDsszukhkwjJN79Y78H6+Ok25Pubvf4PB8zO/f9fL6f3X3vR7cvfe+zu99N\nVSFJasOacRcgSVo5hr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIevGXcBcGzdu\nrC1btsDMDDz55LjLkaRV4ZvAd6uy0LiJC/0tW7bQ7/fhiisMfUkaUW/EcRMX+i9761vh3ntfaa9f\nD6efPth+8UU45ZT5t2cda//JbHvb3va4b/u18Bi87WW57Re///3DjGAyQ3//fvjN34RkcLn5Zrj9\n9nFXJUkT65Hk0VHGjfRCbpIrkxxKMpXklnn2vznJfUkeTPJIkqvm2f98kptHqv7+++HwYagahP6Z\nZ450NUnS8S0Y+knWAncC7wYuAnYkuWjOsA8Du6vqYmA78Ntz9t8BfGHkqrZtg3XrBoG/bt2gLUk6\naaMc6V8CTFXVE1V1GNgFXDNnTAFndNuvB56Z3ZHkbwJPAAcXVdnsef49378kLZlRQv8c4Kmh9nTX\nN+wjwC8lmQY+D9wIkGQD8CvAry6qqvvvhyNHBoF/5MigLUk6aaOE/nzv+5x7+L0D+FRVbQauAj6d\nZA2DsL+jqp4/7h0k1yfpJ+nPzMzAG98IL7002PnSS4O2JOmkjfLunWng3KH2ZoaWbzrvB64EqKr9\nSU4DNgI/AbwnyUeBM4GXkvy/qvqt4StX1V3AXQC9Xq/43vdgzZpB4K9ZA9/73ok8NknSHKOE/gHg\ngiRbgacZvFD7i3PGPAlcBnwqyVuA04CZqnrH7IAkHwGenxv489q2DU49dfAOnvXrfSFXkpbIgqFf\nVUeS3ADsBdYCn6yqg0luBfpVtQf4IPCJJDcxWPq5rk7mG9cvvRT27Rus5W/bNmhLkk5aTiabl0Ov\n16t+vz/uMiRpVUnyQFUteDYGz7IpSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDJjP09++H\n224b/JQkLZnJ++asF16Ayy575RQM+/b5iVxJWiKTd6T//e8PAv/o0cFPT6ssSUtm8kL/da8bHOGv\nXevJ1iRpiU3e8s6GDZ5sTZKWyeSFPgyC3rCXpCU3ecs7kqRlY+hLUkMMfUlqiKEvSQ0x9CWpIYa+\nJDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNmczQ95uzJGlZTN5ZNv3mLElaNpN3pO83Z0nS\nshkp9JNcmeRQkqkkt8yz/81J7kvyYJJHklzV9V+e5IEkj3Y/37ngnfnNWZK0bBZc3kmyFrgTuByY\nBg4k2VNVjw0N+zCwu6o+luQi4PPAFuC7wM9X1TNJ/jKwFzjnuHfoN2dJ0rIZZU3/EmCqqp4ASLIL\nuAYYDv0Czui2Xw88A1BVDw6NOQicluTUqvrz496j35wlSctilNA/B3hqqD0N/MScMR8B7k1yI7AB\n+Nl5budvAQ8uGPiSpGUzypp+5umrOe0dwKeqajNwFfDpJC/fdpIfB24H/v68d5Bcn6SfpD8zMzNa\n5ZKkRRsl9KeBc4fam+mWb4a8H9gNUFX7gdOAjQBJNgO/D1xbVV+f7w6q6q6q6lVVb9OmTYt7BJKk\nkY0S+geAC5JsTbIe2A7smTPmSeAygCRvYRD6M0nOBP4LsLOq/sfSlS1JOhELhn5VHQFuYPDOm8cZ\nvEvnYJJbk1zdDfsg8PeSPAzcDVxXVdVd70eBf5bkoe7ypmV5JJKkBWWQzZOj1+tVv98fdxmStKok\neaCqeguNm7xP5EqSlo2hL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0\nJakhkxf6L7wAt90G+/ePuxJJes2ZvBOurVlT/TVrBl+Kvm+fX5soSSNYvSdcq4KjR+Hw4cGXo0uS\nlswo35G7shKYPdLftm3c1UjSa8rkhf6FF8K11w4C36UdSVpSkxf6GzbAzp3jrkKSXpMmb01fkrRs\nDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDRkp9JNcmeRQkqkk\nt8yz/81J7kvyYJJHklw1tG9nd71DSa5YyuIlSYuz4AnXkqwF7gQuB6aBA0n2VNVjQ8M+DOyuqo8l\nuQj4PLCl294O/DjwI8B/TfJjVXV0qR+IJGlhoxzpXwJMVdUTVXUY2AVcM2dMAWd0268Hnum2rwF2\nVdWfV9U3gKnu9iRJYzBK6J8DPDXUnu76hn0E+KUk0wyO8m9cxHUlSStklNDPPH1zv1h3B/CpqtoM\nXAV8OsmaEa9LkuuT9JP0Z2ZmRihJknQiRgn9aeDcofZmXlm+mfV+YDdAVe0HTgM2jnhdquququpV\nVW/Tpk2jVy9JWpRRQv8AcEGSrUnWM3hhds+cMU8ClwEkeQuD0J/pxm1PcmqSrcAFwB8vVfGSpMVZ\n8N07VXUkyQ3AXmAt8MmqOpjkVqBfVXuADwKfSHITg+Wb66qqgINJdgOPAUeAX/adO5I0Phlk8+To\n9XrV7/fHXYYkrSpJHqiq3kLj/ESuJDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGG\nviQ1ZPJC/4UX4LbbYP/+cVciSa85k3cahjVrqr9mDaxfD/v2waWXjrskSZp4q/c0DFVw9CgcPgz3\n3z/uaiTpNWXBs2yuuARmj/S3bRt3NZL0mjJ5oX/hhXDttYPAd2lHkpbU5IX+hg2wc+e4q5Ck16TJ\nW9OXJC0bQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqI\noS9JDTH0JakhI4V+kiuTHEoyleSWefbfkeSh7vLVJM8N7ftokoNJHk/yG0mylA9AkjS6Bc+nn2Qt\ncCdwOTANHEiyp6oemx1TVTcNjb8RuLjb/kngp4C/0u3+78DPAPcvUf2SpEUY5Uj/EmCqqp6oqsPA\nLuCa44zfAdzdbRdwGrAeOBU4Bfj2iZcrSToZo4T+OcBTQ+3pru9VkpwHbAW+BFBV+4H7gG91l71V\n9fg817s+ST9Jf2ZmZnGPQJI0slFCf741+DrG2O3APVV1FCDJjwJvATYz+I/inUl++lU3VnVXVfWq\nqrdp06bRKpckLdoooT8NnDvU3gw8c4yx23llaQfgF4D/WVXPV9XzwBeAt59IoZKkkzdK6B8ALkiy\nNcl6BsG+Z+6gJBcCZwH7h7qfBH4mybokpzB4EfdVyzuSpJWxYOhX1RHgBmAvg8DeXVUHk9ya5Oqh\noTuAXVU1vPRzD/B14FHgYeDhqvqjJatekrQo+cGMHr9er1f9fn/cZUjSqpLkgarqLTTOT+RKUkMM\nfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCX\npIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlq\niKEvSQ0x9CWpISOFfpIrkxxKMpXklnn235Hkoe7y1STPDe17c5J7kzye5LEkW5aufEnSYqxbaECS\ntcCdwOXANHAgyZ6qemx2TFXdNDT+RuDioZv4j8C/rKovJjkdeGmpipckLc4oR/qXAFNV9URVHQZ2\nAdccZ/wO4G6AJBcB66rqiwBV9XxV/dlJ1ixJOkGjhP45wFND7emu71WSnAdsBb7Udf0Y8FySzyZ5\nMMmvdX85zL3e9Un6SfozMzOLewSSpJGNEvqZp6+OMXY7cE9VHe3a64B3ADcDfx04H7juVTdWdVdV\n9aqqt2nTphFKkiSdiFFCfxo4d6i9GXjmGGO30y3tDF33wW5p6AjwB8DbTqRQSdLJGyX0DwAXJNma\nZD2DYN8zd1CSC4GzgP1zrntWktnD93cCj829riRpZSwY+t0R+g3AXuBxYHdVHUxya5Krh4buAHZV\nVQ1d9yiDpZ19SR5lsFT0iaV8AJKk0WUooydCr9erfr8/7jIkaVVJ8kBV9RYa5ydyJakhhr4kNcTQ\nl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1J\naoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SG\nGPqS1JCRQj/JlUkOJZlKcss8++9I8lB3+WqS5+bsPyPJ00l+a6kKlyQt3rqFBiRZC9wJXA5MAweS\n7Kmqx2bHVNVNQ+NvBC6eczP/AvhvS1KxJOmEjXKkfwkwVVVPVNVhYBdwzXHG7wDunm0k+WvADwP3\nnkyhkqSTN0ronwM8NdSe7vpeJcl5wFbgS117DfCvgX9yvDtIcn2SfpL+zMzMKHVLkk7AKKGfefrq\nGGO3A/dU1dGu/QHg81X11DHGD26s6q6q6lVVb9OmTSOUJEk6EQuu6TM4sj93qL0ZeOYYY7cDvzzU\nvhR4R5IPAKcD65M8X1WvejFYkrT8Rgn9A8AFSbYCTzMI9l+cOyjJhcBZwP7Zvqp679D+64CegS9J\n47Pg8k5VHQFuAPYCjwO7q+pgkluTXD00dAewq6qOtfQjSRqzTFpG93q96vf74y5DklaVJA9UVW+h\ncX4iV5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kN\nMfQlqSGGviQ1xNCXpIYY+pLUEENfkhoyeaH/7LOwf//C4yRJizZ5of/003DZZQa/JC2DifuO3I1J\nnQf1bXjmaXh23PUsYCPw3XEXMYLVUidY63JYLXXC6ql1Eus8r6o2LTRo4kI/SX+UL/edBKul1tVS\nJ1jrclgtdcLqqXW11DmfyVvekSQtG0NfkhoyiaF/17gLWITVUutqqROsdTmsljph9dS6Wup8lYlb\n05ckLZ9JPNKXJC2TiQr9JFcmOZRkKsktY67l3CT3JXk8ycEk/7Drf0OSLyb5WvfzrK4/SX6jq/2R\nJG9b4XrXJnkwyee69tYkX+7q/L0k67v+U7v2VLd/ywrXeWaSe5L8STe3l07wnN7U/dt/JcndSU6b\nlHlN8skk30nylaG+Rc9jkvd147+W5H0rVOevdf/+jyT5/SRnDu3b2dV5KMkVQ/3Lng3z1Tq07+Yk\nlWRj1x7bnJ60qpqIC7AW+DpwPrAeeBi4aIz1nA28rdt+HfBV4CLgo8AtXf8twO3d9lXAF4AAbwe+\nvML1/mPgd4HPde3dwPZu++PAP+i2PwB8vNveDvzeCtf5H4C/222vB86cxDkFzgG+AfyFofm8blLm\nFfhp4G3AV4b6FjWPwBuAJ7qfZ3XbZ61Ane8C1nXbtw/VeVH3vD8V2NrlwdqVyob5au36zwX2Av8b\n2DjuOT3pxznuAoYm9lJg71B7J7Bz3HUN1fOHwOXAIeDsru9s4FC3/TvAjqHxL49bgdo2A/uAdwKf\n634Rvzv0xHp5brtf3ku77XXduKxQnWd0QZo5/ZM4p+cAT3VP3nXdvF4xSfMKbJkTpouaR2AH8DtD\n/T8wbrnqnLPvF4DPdNs/8JyfndOVzIb5agXuAf4q8E1eCf2xzunJXCZpeWf2STZruusbu+5P9YuB\nLwM/XFXfAuh+vqkbNs76fx34p8BLXfuNwHNVdWSeWl6us9v/p934lXA+MAP8+24p6t8m2cAEzmlV\nPQ38K+BJ4FsM5ukBJnNeZy12HifhOfd3GBwxc5x6xlZnkquBp6vq4Tm7Jq7WUU1S6GeevrG/tSjJ\n6cB/Bv5RVf3f4w2dp2/Z60/yc8B3quqBEWsZ5zyvY/Dn88eq6mLgBQbLEMcytlq79fBrGCwz/Aiw\nAXj3ceqZyN/fzrFqG2vNST4EHAE+M9t1jHrG9dz6IeBDwD+fb/c8fWOf01FMUuhPM1g7m7UZeGZM\ntQCQ5BQGgf+Zqvps1/3tJGd3+88GvtP1j6v+nwKuTvJNYBeDJZ5fB85Msm6eWl6us9v/euD/rECd\ns/c9XVVf7tr3MPhPYNLmFOBngW9U1UxVvQh8FvhJJnNeZy12Hsc2v90LnD8HvLe6dZAJrPMvMfhP\n/+Hu+bUZ+F9J/uIE1jqySQr9A8AF3bsj1jN4MWzPuIpJEuDfAY9X1b8Z2rUHmH1F/n0M1vpn+6/t\nXtV/O/Cns39qL6eq2llVm6tqC4M5+1JVvRe4D3jPMeqcrf893fgVORKpqmeBp5Jc2HVdBjzGhM1p\n50ng7Ul+qPtdmK114uZ1yGLncS/wriRndX/ZvKvrW1ZJrgR+Bbi6qv5sTv3bu3dCbQUuAP6YMWVD\nVT1aVW+qqi3d82uawZs7nmXC5nRRxv2iwpwXTK5i8C6ZrwMfGnMtf4PBn2WPAA91l6sYrNPuA77W\n/XxDNz7AnV3tjwK9MdS8jVfevXM+gyfMFPCfgFO7/tO69lS3//wVrvGtQL+b1z9g8A6HiZxT4FeB\nPwG+AnyawbtKJmJegbsZvNbwIoMwev+JzCODNfWp7vK3V6jOKQbr3rPPq48Pjf9QV+ch4N1D/cue\nDfPVOmf/N3nlhdyxzenJXvxEriQ1ZJKWdyRJy8zQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x\n9CWpIf8fr3E0uU9g+ZUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0c586194a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot( x,y, 'r.')\n",
    "plt.xlim(np.min(x), np.max(x))\n",
    "plt.ylim(np.min(y), np.max(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[np.argmax(y)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.84400671742051059"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.72685807583221373"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using weighted tfidf_word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "b=np.ndarray(shape=(1750,50),dtype=float)\n",
    "for i, d in enumerate(X_train_tfidfw2v):\n",
    "    b[i]=d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_tf=[]\n",
    "y_tf=[]\n",
    "for k in range(1,1575,2):\n",
    "    model=KNeighborsClassifier(n_neighbors=k,weights='distance',algorithm='kd_tree')\n",
    "    avg_accuracy=np.mean(cross_val_score(model,b, y_train_avgw2v,cv=10, scoring='accuracy'))\n",
    "    x_tf.append(k)\n",
    "    y_tf.append(avg_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.75, 0.86)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE7FJREFUeJzt3X+QXWd93/H3R5JlBaX+QSxaR7Kx\nnAhjJ6W1uzEov9BgDEIT7GbgD6lJjVs37pTaU1wYsAbacTzTYTBtYJK4ENG6Dh5qxTUkVYkZAcbu\nTDsq0SrCNpIjWAyxFxOQaZ1iQ5Etf/vHPWtfr1fau5J2710979fMnT3nOc8993se7f3s2eecu0pV\nIUlqw5JhFyBJWjiGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwYK/SQbkxxIMpHkxhm2n5vkviR7kzyY\nZFPfttck2ZVkX5KHkqw4kQcgSRpcZrtPP8lS4GvA5cAksBvYUlX7+/psA/ZW1ceSXATcU1XnJVkG\n/DnwD6vqgSQ/BTxZVYfn6XgkSUcxyJn+pcBEVT1SVYeA7cCV0/oUcFq3fDrweLf8JuDBqnoAoKq+\nb+BL0vAsG6DPauCxvvVJ4LXT+twEfD7J9cBK4I1d+6uASrITWAVsr6pbpr9AkmuBawFWrlz59179\n6lfP5RgkqXl79ux5oqpWzdZvkNDPDG3T54S2ALdX1b9Lsh64I8nPd/v/ZeAXgB8C9ybZU1X3vmhn\nVduAbQBjY2M1Pj4+QFmSpClJ/nKQfoNM70wC5/Str+GF6Zsp1wB3AVTVLmAFcFb33P9eVU9U1Q+B\ne4BLBilMknTiDRL6u4F1SdYmWQ5sBnZM6/MocBlAkgvphf5BYCfwmiQv6y7qvh7YjyRpKGad3qmq\nZ5NcRy/AlwK3VdW+JDcD41W1A3g38IkkN9Cb+rm6ercF/Z8kv0PvB0fRu6vnT+frYCRJRzfrLZsL\nzTl9SZq77nrp2Gz9/ESuJDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCX\npIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlq\niKEvSQ0x9CWpIcuGXcBRbdsGH/0ofPe78MwzcMopvfajLU8ZpO9cl923+x72vk+GY3Df87Lv18Df\nZgCjG/rvex/ccsuwq5CkReEUWD5Iv9Gc3tm1Cz784WFXIUknndEM/U9+EqqGXYUknXRGd3qn3/Ll\ncOqpJ8W8m/t234viddz3otv3Mz/4wSEGMJqhf9VVcNttLxzU/ffD+vXDrkqSRtaDyUOD9BvN0F+/\nHn7v9+DTn4a3vc3Al6QTZKA5/SQbkxxIMpHkxhm2n5vkviR7kzyYZNMM259K8p6Bqtq2Da67Dr74\nRXjXu3oXdiVJx23W0E+yFLgVeAtwEbAlyUXTun0AuKuqLgY2A/9+2vaPAJ8bqKKnn4Z3vrM3tfPc\nc/DjH/emdyRJx22QM/1LgYmqeqSqDgHbgSun9SngtG75dODxqQ1J/j7wCLBvoIq+/304fPiF9QQ2\nbBjoqZKkoxsk9FcDj/WtT3Zt/W4CfjPJJHAPcD1AkpXA+4DfPtoLJLk2yXiS8R/96Ecv3vjWtzqn\nL0knyCChnxnapt9EvwW4varWAJuAO5IsoRf2H6mqp472AlW1rarGqmrsJ9as6d2emfS+vve9gxyH\nJGkAg9y9Mwmc07e+hr7pm841wEaAqtqVZAVwFvBa4O1JbgHOAJ5L8v+q6veP+GorV8J99/Xm8Tds\n8Cxfkk6gQUJ/N7AuyVrg2/Qu1P6DaX0eBS4Dbk9yIbACOFhVvzLVIclNwFNHDfwp69cb9pI0D2ad\n3qmqZ4HrgJ3Aw/Tu0tmX5OYkV3Td3g38VpIHgDuBq6v8OwqSNGoyatk8NjZW4+Pjwy5DkhaVJHuq\namy2fqP5B9ckSfPC0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLU\nEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x\n9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGjJ6of/00/DBD8KuXcOuRJJOOqmqYdfwImNLltT4kiWwfDnc\ney+sXz/skiRp5CXZU1Vjs/Ub6Ew/ycYkB5JMJLlxhu3nJrkvyd4kDybZ1LVfnmRPkoe6r2+Y9cWq\n4PBhOHQI7r9/kPIkSQNaNluHJEuBW4HLgUlgd5IdVbW/r9sHgLuq6mNJLgLuAc4DngDeWlWPJ/l5\nYCewepYXhKkz/Q0bjuGQJElHMmvoA5cCE1X1CECS7cCVQH/oF3Bat3w68DhAVe3t67MPWJHk1Kr6\n8RFf7YIL4KqreoHv1I4knVCDhP5q4LG+9UngtdP63AR8Psn1wErgjTPs523A3pkCP8m1wLUA5557\nLmzdOkBZkqS5GmROPzO0Tb/6uwW4varWAJuAO5I8v+8kPwd8CPinM71AVW2rqrGqGlu1atVglUuS\n5myQ0J8EzulbX0M3fdPnGuAugKraBawAzgJIsgb4Y+CqqvrG8RYsSTp2g4T+bmBdkrVJlgObgR3T\n+jwKXAaQ5EJ6oX8wyRnAnwJbq+p/nriyJUnHYtbQr6pngevo3XnzML27dPYluTnJFV23dwO/leQB\n4E7g6up9AOA64GeBf5XkK93jFfNyJJKkWY3eh7PGxmp8fHzYZUjSonJCP5wlSTo5GPqS1BBDX5Ia\nYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGG\nviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhL\nUkMMfUlqiKEvSQ0x9CWpIQOFfpKNSQ4kmUhy4wzbz01yX5K9SR5Msqlv29bueQeSvPlEFi9Jmptl\ns3VIshS4FbgcmAR2J9lRVfv7un0AuKuqPpbkIuAe4LxueTPwc8BPA19M8qqqOnyiD0SSNLtBzvQv\nBSaq6pGqOgRsB66c1qeA07rl04HHu+Urge1V9eOq+iYw0e1PkjQEg4T+auCxvvXJrq3fTcBvJpmk\nd5Z//RyeS5Jrk4wnGT948OCApUuS5mqQ0M8MbTVtfQtwe1WtATYBdyRZMuBzqaptVTVWVWOrVq0a\noCRJ0rGYdU6f3tn5OX3ra3hh+mbKNcBGgKralWQFcNaAz5UkLZBBzvR3A+uSrE2ynN6F2R3T+jwK\nXAaQ5EJgBXCw67c5yalJ1gLrgD87UcVLkuZm1jP9qno2yXXATmApcFtV7UtyMzBeVTuAdwOfSHID\nvembq6uqgH1J7gL2A88C/9w7dyRpeNLL5tExNjZW4+Pjwy5DkhaVJHuqamy2fqP3idynn4YPfhB2\n7Rp2JZJ00hm9M/0lS2p8yRJYvhzuvRfWrx92SZI08hbvmX4VHD4Mhw7B/fcPuxpJOqkMcsvmwkpg\n6kx/w4ZhVyNJJ5XRC/0LLoCrruoFvlM7knRCjV7or1wJW7cOuwpJOimN3py+JGneGPqS1BBDX5Ia\nYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGG\nviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNWSg0E+yMcmBJBNJbpxh+0eS\nfKV7fC3Jk33bbkmyL8nDSX43SU7kAUiSBrdstg5JlgK3ApcDk8DuJDuqav9Un6q6oa//9cDF3fIv\nAr8EvKbb/D+A1wP3n6D6JUlzMMiZ/qXARFU9UlWHgO3AlUfpvwW4s1suYAWwHDgVOAX47rGXK0k6\nHoOE/mrgsb71ya7tJZK8ElgLfAmgqnYB9wHf6R47q+rhGZ53bZLxJOMHDx6c2xFIkgY2SOjPNAdf\nR+i7Gbi7qg4DJPlZ4EJgDb0fFG9I8qsv2VnVtqoaq6qxVatWDVa5JGnOBgn9SeCcvvU1wONH6LuZ\nF6Z2AH4d+F9V9VRVPQV8DnjdsRQqSTp+g4T+bmBdkrVJltML9h3TOyW5ADgT2NXX/Cjw+iTLkpxC\n7yLuS6Z3JEkLY9bQr6pngeuAnfQC+66q2pfk5iRX9HXdAmyvqv6pn7uBbwAPAQ8AD1TVfzth1UuS\n5iQvzujhGxsbq/Hx8WGXIUmLSpI9VTU2Wz8/kStJDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhL\nUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1\nxNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMG\nCv0kG5McSDKR5MYZtn8kyVe6x9eSPNm37dwkn0/ycJL9Sc47ceVLkuZi2WwdkiwFbgUuByaB3Ul2\nVNX+qT5VdUNf/+uBi/t28Ung31TVF5L8JPDciSpekjQ3g5zpXwpMVNUjVXUI2A5ceZT+W4A7AZJc\nBCyrqi8AVNVTVfXD46xZknSMBgn91cBjfeuTXdtLJHklsBb4Utf0KuDJJJ9JsjfJh7vfHKY/79ok\n40nGDx48OLcjkCQNbJDQzwxtdYS+m4G7q+pwt74M+BXgPcAvAOcDV79kZ1XbqmqsqsZWrVo1QEmS\npGMxSOhPAuf0ra8BHj9C3810Uzt9z93bTQ09C/wJcMmxFCpJOn6DhP5uYF2StUmW0wv2HdM7JbkA\nOBPYNe25ZyaZOn1/A7B/+nMlSQtj1tDvztCvA3YCDwN3VdW+JDcnuaKv6xZge1VV33MP05vauTfJ\nQ/Smij5xIg9AkjS49GX0SBgbG6vx8fFhlyFJi0qSPVU1Nls/P5ErSQ0x9CWpIYa+JDXE0Jekhhj6\nktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhoycn9aOckPgAPDrmNA\nZwFPDLuIASyWOsFa58NiqRMWT62jWOcrq2rW/2922UJUMkcHBvmb0KMgyfhiqHWx1AnWOh8WS52w\neGpdLHXOxOkdSWqIoS9JDRnF0N827ALmYLHUuljqBGudD4ulTlg8tS6WOl9i5C7kSpLmzyie6UuS\n5omhL0kNGanQT7IxyYEkE0luHHIt5yS5L8nDSfYl+Rdd+8uTfCHJ17uvZ3btSfK7Xe0PJrlkgetd\nmmRvks9262uTfLmr84+SLO/aT+3WJ7rt5y1wnWckuTvJX3Rju36Ex/SG7t/+q0nuTLJiVMY1yW1J\nvpfkq31tcx7HJO/o+n89yTsWqM4Pd//+Dyb54yRn9G3b2tV5IMmb+9rnPRtmqrVv23uSVJKzuvWh\njelxq6qReABLgW8A5wPLgQeAi4ZYz9nAJd3y3wC+BlwE3ALc2LXfCHyoW94EfA4I8Drgywtc778E\n/jPw2W79LmBzt/xx4J91y+8EPt4tbwb+aIHr/EPgn3TLy4EzRnFMgdXAN4Gf6BvPq0dlXIFfBS4B\nvtrXNqdxBF4OPNJ9PbNbPnMB6nwTsKxb/lBfnRd17/tTgbVdHixdqGyYqdau/RxgJ/CXwFnDHtPj\nPs5hF9A3sOuBnX3rW4Gtw66rr57/ClxO79PCZ3dtZ9P7MBnAHwBb+vo/328BalsD3Au8Afhs9434\nRN8b6/mx7b5513fLy7p+WaA6T+uCNNPaR3FMVwOPdW/eZd24vnmUxhU4b1qYzmkcgS3AH/S1v6jf\nfNU5bduvA5/qll/0np8a04XMhplqBe4G/g7wLV4I/aGO6fE8Rml6Z+pNNmWyaxu67lf1i4EvA3+z\nqr4D0H19RddtmPV/FHgv8Fy3/lPAk1X17Ay1PF9nt/2vu/4L4XzgIPCfuqmo/5BkJSM4plX1beDf\nAo8C36E3TnsYzXGdMtdxHIX33D+md8bMUeoZWp1JrgC+XVUPTNs0crUOapRCPzO0Df1+0iQ/CXwa\neFdV/d+jdZ2hbd7rT/JrwPeqas+AtQxznJfR+/X5Y1V1MfA0vWmIIxlard18+JX0phl+GlgJvOUo\n9Yzk92/nSLUNteYk7weeBT411XSEeob13noZ8H7gX8+0eYa2oY/pIEYp9CfpzZ1NWQM8PqRaAEhy\nCr3A/1RVfaZr/m6Ss7vtZwPf69qHVf8vAVck+Rawnd4Uz0eBM5JM/W2l/lqer7PbfjrwvxegzqnX\nnqyqL3frd9P7ITBqYwrwRuCbVXWwqp4BPgP8IqM5rlPmOo5DG9/uAuevAb9R3TzICNb5M/R+6D/Q\nvb/WAH+e5G+NYK0DG6XQ3w2s6+6OWE7vYtiOYRWTJMB/BB6uqt/p27QDmLoi/w56c/1T7Vd1V/Vf\nB/z11K/a86mqtlbVmqo6j96YfamqfgO4D3j7Eeqcqv/tXf8FOROpqr8CHktyQdd0GbCfERvTzqPA\n65K8rPtemKp15Ma1z1zHcSfwpiRndr/ZvKlrm1dJNgLvA66oqh9Oq39zdyfUWmAd8GcMKRuq6qGq\nekVVnde9vybp3dzxV4zYmM7JsC8qTLtgsoneXTLfAN4/5Fp+md6vZQ8CX+kem+jN094LfL37+vKu\nf4Bbu9ofAsaGUPMGXrh753x6b5gJ4L8Ap3btK7r1iW77+Qtc498Fxrtx/RN6dziM5JgCvw38BfBV\n4A56d5WMxLgCd9K71vAMvTC65ljGkd6c+kT3+EcLVOcEvXnvqffVx/v6v7+r8wDwlr72ec+GmWqd\ntv1bvHAhd2hjerwP/wyDJDVklKZ3JEnzzNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDfn/L/zr\n/JvW3JoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0c4c86c748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot( x_tf,y_tf, 'r.')\n",
    "plt.xlim(np.min(x_tf), np.max(x_tf))\n",
    "plt.ylim(np.min(y_tf), np.max(y_tf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(y_tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_tf[np.argmax(y_tf)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.84400671742051059"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(y_tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.74229403642334679"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(y_tf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using BoW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neighbors=[]\n",
    "accuracy=[]\n",
    "for k in range(1, 1574, 2):\n",
    "    model=KNeighborsClassifier(n_neighbors=k,algorithm='kd_tree', weights='distance')\n",
    "    avg_accuracy_bow=np.mean(cross_val_score(model, bigrams_train, y_train_avgw2v,cv=10))\n",
    "    neighbors.append(k)\n",
    "    accuracy.append(avg_accuracy_bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.83037050306015825, 0.84400671742051059)"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFpNJREFUeJzt3X+QXWWd5/H3J4kxY5QfmszKJIHE\nLYohUrWDew0y6hRlxIkZC7SKP8gMo8yyUq4FqyyUJiW1hUzVTo07K+7WIIo/himGhcmi7kYLl7EY\n+C+TTUcIEGLWTGShA4xNWaOC5YQM3/3jPo3XtklOJ925fan3q+pWn/Oc5577PU+67yfnOed2p6qQ\nJGnBsAuQJM0PBoIkCTAQJEmNgSBJAgwESVJjIEiSgI6BkGRDkn1J9ifZPM3205Pcn+TBJA8n2TjN\n9ueSXDelfWF7zreO7zAkScfrqIGQZCFwM/BeYC2wKcnaKd2uB7ZW1bnApcDnp2y/Cfj2NLv/GLB3\npkVLkmZflzOEdcD+qjpQVYeAu4CLp/Qp4KS2fDLw1OSGJO8HDgB7Bp+QZCXwe8CXj610SdJsWtSh\nzwrgyYH1ceC8KX1uAP4mydXAUuDdAEmWAp8ELgSum/KczwGfAF53pBdPciVwJcAb4F+v7lCwJKnv\nceDZqnTp2yUQptvR1N93sQm4rar+S5LzgduTnAN8Gripqp5LfrGbJO8DflhVu5JccKQXr6pbgVsB\nekmNdShYktTXm0HfLoEwDqwaWF/JwJRQcwWwAaCqtidZAiyjfyZxSZLPAKcALyb5Of2zjovaxecl\nwElJ/qqqLjtiJb/5m/3H3/0d/NM/wbJl8NOfwvPPw6te1e/zwgu/vDzp5bYfz7L7dt/ue/7t+5Vw\nDLO478M/+tFhOuoSCDuBM5OsAQ7Sv2j8+1P6PAGsB25Lcjb9N/mJqnrnZIckNwDPVdWft6Ytrf0C\n4LqjhgHA0qXwjW90KFmSBLA72d2171EvKlfVYeAq4F76dwRtrao9SW5MclHrdi3w4fRf+E7g8vLX\nqErSSMkovW/3er0aG/MqgiR1lWRXVXW6lOAnlSVJwCgHwvbt8Cd/0v8qSTpuXS4qzz/bt8P69XDo\nECxeDPfdB+efP+yqJGmkjeYZwgMP9MPgn/+5//WBB4ZdkSSNvNEMhAsu6J8ZLFzY/3rBBcOuSJJG\n3mhOGZ1/fn+a6IEH+mHgdJEkHbfRDAToh4BBIEmzZjSnjCRJs85AkCQBBoIkqTEQJEmAgSBJagwE\nSRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqRisQnn/eP5spSXMkVTXsGjrrLVhQYwsW\n+GczJamjJLuqqtel72idIVT5ZzMlaY50CoQkG5LsS7I/yeZptp+e5P4kDyZ5OMnGabY/l+S6tr6q\n9d+bZE+Sj3WqNvHPZkrSHDnqX0xLshC4GbgQGAd2JtlWVY8NdLse2FpVtyRZC9wDrB7YfhPw7YH1\nw8C1VfXdJK8DdiX5zpR9/qqzzoIPftA/mylJc6DLn9BcB+yvqgMASe4CLgYG37wLOKktnww8Nbkh\nyfuBA8DzL3Wuehp4ui3/NMleYMWUff6qpUthy5YOJUuSZqrLlNEK4MmB9fHWNugG4LIk4/TPDq4G\nSLIU+CTw6ZfbeZLVwLnAjpfZfmWSsSRjExMTHcqVJB2LLoGQadqm3pq0CbitqlYCG4HbkyygHwQ3\nVdVz0+44eS3wNeDjVfWT6fpU1a1V1auq3vLlyzuUK0k6Fl2mjMaBVQPrKxmYEmquADYAVNX2JEuA\nZcB5wCVJPgOcAryY5OdV9edJXkU/DO6oqq8f53FIko5TlzOEncCZSdYkWQxcCmyb0ucJYD1AkrOB\nJcBEVb2zqlZX1Wrgc8B/amEQ4CvA3qr6bOdqn3nGD6VJ0hw5aiBU1WHgKuBeYC/9u4n2JLkxyUWt\n27XAh5PsBu4ELq8jf+Lt7cAfAu9K8lB7bDxC/76DB2H9ekNBkubAaH1SOamxhQvhj//Yu40kqYNX\n7ieVwQ+lSdIcGa1AWLHC32EkSXNktALhjW80DCRpjoxWIEiS5oyBIEkCDARJUmMgSJIAA0GS1BgI\nkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwE\nSVJjIEiSgI6BkGRDkn1J9ifZPM3205Pcn+TBJA8n2TjN9ueSXNd1n5KkE+uogZBkIXAz8F5gLbAp\nydop3a4HtlbVucClwOenbL8J+PYM9ylJOoG6nCGsA/ZX1YGqOgTcBVw8pU8BJ7Xlk4GnJjckeT9w\nANgzw31Kkk6gLoGwAnhyYH28tQ26AbgsyThwD3A1QJKlwCeBTx/DPmn7uDLJWJKxiYmJDuVKko5F\nl0DING01ZX0TcFtVrQQ2ArcnWUA/CG6qqueOYZ/9xqpbq6pXVb3ly5d3KFeSdCwWdegzDqwaWF/J\nwJRQcwWwAaCqtidZAiwDzgMuSfIZ4BTgxSQ/B3Z12Kck6QTqEgg7gTOTrAEO0r9o/PtT+jwBrAdu\nS3I2sASYqKp3TnZIcgPwXFX9eZJFHfYpSTqBjhoIVXU4yVXAvcBC4KtVtSfJjcBYVW0DrgW+lOQa\n+lM/l1fVtFNAR9rnLByPJOkY5Qjv2/NOr9ersbGxYZchSSMjya6q6nXp6yeVJUmAgSBJagwESRJg\nIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkx\nECRJgIEgSWoMBEkSMGqB8MwzsH37sKuQpFek0QqEgwdh/XpDQZLmwGgFAsChQ/DAA8OuQpJecUYv\nEBYvhgsuGHYVkvSK0ykQkmxIsi/J/iSbp9l+epL7kzyY5OEkG1v7uiQPtcfuJB8YeM41SfYkeTTJ\nnUmWHLWQFSvgvvvg/PNncIiSpC6OGghJFgI3A+8F1gKbkqyd0u16YGtVnQtcCny+tT8K9Krqt4AN\nwBeTLEqyAvj3bds5wML2vCN74xsNA0maI13OENYB+6vqQFUdAu4CLp7Sp4CT2vLJwFMAVfWzqjrc\n2pe0fpMWAb+WZBHwmsnnSJKGo0sgrACeHFgfb22DbgAuSzIO3ANcPbkhyXlJ9gCPAB+pqsNVdRD4\nM+AJ4Gngx1X1N9O9eJIrk4wlGZuYmOh4WJKkmeoSCJmmraasbwJuq6qVwEbg9iQLAKpqR1W9GXgr\nsCXJkiSn0j/LWAP8BrA0yWXTvXhV3VpVvarqLV++vNtRSZJmrEsgjAOrBtZX8qvTO1cAWwGqajv9\n6aFlgx2qai/wPHAO8G7gB1U1UVUvAF8HfvtYDkCSNDu6BMJO4Mwka5Ispn/xd9uUPk8A6wGSnE0/\nECbacxa19jOAs4DHW/+3JXlNkrTn7p2F45EkHaNFR+tQVYeTXAXcS/9uoK9W1Z4kNwJjVbUNuBb4\nUpJr6E8nXV5VleQdwOYkLwAvAh+tqmeBZ5PcDXwXOAw8CNw6FwcoSeomVVMvB8xfvV6vxsbGhl2G\nJI2MJLuqqtel7+h9UlmSNCcMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJg\nIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKnp\nFAhJNiTZl2R/ks3TbD89yf1JHkzycJKNrX1dkofaY3eSDww855Qkdyf5XpK9Sc6fvcOSJM3UoqN1\nSLIQuBm4EBgHdibZVlWPDXS7HthaVbckWQvcA6wGHgV6VXU4yWnA7iTfrKrDwH8F/ndVXZJkMfCa\nWT0ySdKMdDlDWAfsr6oDVXUIuAu4eEqfAk5qyycDTwFU1c/amz/AktaPJCcBvwN8pfU7VFX/eDwH\nIkk6Pl0CYQXw5MD6eGsbdANwWZJx+mcHV09uSHJekj3AI8BHWkC8CZgA/qJNM305ydLpXjzJlUnG\nkoxNTEx0PS5J0gx1CYRM01ZT1jcBt1XVSmAjcHuSBQBVtaOq3gy8FdiSZAn9qaq3ALdU1bnA88Cv\nXJtoz7+1qnpV1Vu+fHmng5IkzVyXQBgHVg2sr6RNCQ24AtgKUFXb6U8PLRvsUFV76b/xn9P2OV5V\nO9rmu+kHhCRpSLoEwk7gzCRr2sXfS4FtU/o8AawHSHI2/UCYaM9Z1NrPAM4CHq+qZ4Ank5zVnr8e\neAxJ0tAc9S6jdofQVcC9wELgq1W1J8mNwFhVbQOuBb6U5Br600mXV1UleQewOckLwIvAR6vq2bbr\nq4E7WsgcAP5o1o9OktRZqqZeDpi/er1ejY2NDbsMSRoZSXZVVa9LXz+pLEkCDARJUmMgSJIAA0GS\n1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJ\nAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJ6BgISTYk2Zdkf5LN02w/Pcn9SR5M8nCSja19XZKH\n2mN3kg9Med7C9pxvzc7hSJKO1aKjdUiyELgZuBAYB3Ym2VZVjw10ux7YWlW3JFkL3AOsBh4FelV1\nOMlpwO4k36yqw+15HwP2AifN2hFJko5JlzOEdcD+qjpQVYeAu4CLp/QpfvGmfjLwFEBV/WzgzX9J\n6wdAkpXA7wFfPvbyJUmzpUsgrACeHFgfb22DbgAuSzJO/+zg6skNSc5Lsgd4BPjIQEB8DvgE8OKR\nXjzJlUnGkoxNTEx0KFeSdCy6BEKmaasp65uA26pqJbARuD3JAoCq2lFVbwbeCmxJsiTJ+4AfVtWu\no714Vd1aVb2q6i1fvrxDuZKkY9ElEMaBVQPrK2lTQgOuALYCVNV2+tNDywY7VNVe4HngHODtwEVJ\nHqc/BfWuJH91DPVLkmZJl0DYCZyZZE2SxcClwLYpfZ4A1gMkOZt+IEy05yxq7WcAZwGPV9WWqlpZ\nVavb/v62qi6blSOSJB2To95l1O4Qugq4F1gIfLWq9iS5ERirqm3AtcCXklxDfzrp8qqqJO8ANid5\ngf61go9W1bNzdjSSpGOWqqmXA+avXq9XY2Njwy5DkkZGkl1V1evS108qS5IAA0GS1BgIkiTAQJAk\nNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiS\nAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCOgZCkg1J9iXZn2TzNNtPT3J/kgeTPJxkY2tfl+Sh\n9tid5AOtfVXrvzfJniQfm93DkiTN1KKjdUiyELgZuBAYB3Ym2VZVjw10ux7YWlW3JFkL3AOsBh4F\nelV1OMlpwO4k3wQOA9dW1XeTvA7YleQ7U/YpSTqBupwhrAP2V9WBqjoE3AVcPKVPASe15ZOBpwCq\n6mdVdbi1L2n9qKqnq+q7bfmnwF5gxfEciCTp+HQJhBXAkwPr4/zqm/cNwGVJxumfHVw9uSHJeUn2\nAI8AHxkIiMntq4FzgR3TvXiSK5OMJRmbmJjoUK4k6Vh0CYRM01ZT1jcBt1XVSmAjcHuSBQBVtaOq\n3gy8FdiSZMlLO05eC3wN+HhV/WS6F6+qW6uqV1W95cuXdyhXknQsugTCOLBqYH0lbUpowBXAVoCq\n2k5/emjZYIeq2gs8D5wDkORV9MPgjqr6+rEUL0maPV0CYSdwZpI1SRYDlwLbpvR5AlgPkORs+oEw\n0Z6zqLWfAZwFPJ4kwFeAvVX12dk5FEnS8ThqILQ5/6uAe+lf/N1aVXuS3JjkotbtWuDDSXYDdwKX\nV1UB76B/Z9FDwDeAj1bVs8DbgT8E3jVwW+rGWT86SVJn6b9vj4Zer1djY2PDLkOSRkaSXVXV69LX\nTypLkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJ\nMBAkSY2BIEkCDARJUmMgSJIAA0GS1IxWIDzzDGzfPuwqJOkVabQC4eBBWL/eUJCkOZCqGnYNnSS5\n8g3wxTOg/gGeOgjPDLumI1gGPDvsIjoalVpHpU4YnVpHpU6w1uNxRlUt79JxlAJhrKp6w66jC2ud\nfaNSJ4xOraNSJ1jriTJaU0aSpDljIEiSgNEKhFuHXcAMWOvsG5U6YXRqHZU6wVpPiJG5hiBJmluj\ndIYgSZpDBoIkCRiRQEiyIcm+JPuTbB5yLauS3J9kb5I9ST7W2l+f5DtJvt++ntrak+S/tdofTvKW\nIdS8MMmDSb7V1tck2dFq/eski1v7q9v6/rZ99Qms8ZQkdyf5Xhvb8+frmCa5pv3bP5rkziRL5suY\nJvlqkh8meXSgbcbjmORDrf/3k3zoBNX5n9u//8NJvpHklIFtW1qd+5L87kD7nL83TFfrwLbrklSS\nZW19aGM6K6pqXj+AhcDfA28CFgO7gbVDrOc04C1t+XXA/wXWAp8BNrf2zcCftuWNwLeBAG8Ddgyh\n5v8A/HfgW219K3BpW/4C8O/a8keBL7TlS4G/PoE1/iXwb9vyYuCU+TimwArgB8CvDYzl5fNlTIHf\nAd4CPDrQNqNxBF4PHGhfT23Lp56AOt8DLGrLfzpQ59r2c/9qYE17P1h4ot4bpqu1ta8C7gX+H7Bs\n2GM6K8c67AI6/GOcD9w7sL4F2DLsugbq+V/AhcA+4LTWdhqwry1/Edg00P+lfieovpXAfcC7gG+1\nb9RnB37wXhrf9s19flte1PrlBNR4UnuTzZT2eTem9APhyfaDvaiN6e/OpzEFVk95o53ROAKbgC8O\ntP9Sv7mqc8q2DwB3tOVf+pmfHNMT+d4wXa3A3cC/Ah7nF4Ew1DE93scoTBlN/gBOGm9tQ9dO/88F\ndgD/oqqeBmhff711G3b9nwM+AbzY1t8A/GNVHZ6mnpdqbdt/3PrPtTcBE8BftKmtLydZyjwc06o6\nCPwZ8ATwNP0x2sX8G9NBMx3HYX/PAvwb+v/T5gj1DK3OJBcBB6tq95RN867WmRiFQMg0bUO/VzbJ\na4GvAR+vqp8cqes0bSek/iTvA35YVbs61jOsWhfRPyW/parOBZ6nP7XxcoY5pqcCF9OfuvgNYCnw\n3iPUMy+/f5uXq22oNSf5FHAYuGOy6WXqGUqdSV4DfAr4j9NtnqZt6GPa1SgEwjj9ubpJK4GnhlQL\nAEleRT8M7qiqr7fmf0hyWtt+GvDD1j7M+t8OXJTkceAu+tNGnwNOSbJomnpeqrVtPxn40QmocxwY\nr6odbf1u+gExH8f03cAPqmqiql4Avg78NvNvTAfNdByHNr7tYuv7gD+oNrcyD+v8l/T/Q7C7/Wyt\nBL6b5I3zsNYZGYVA2Amc2e7iWEz/wty2YRWTJMBXgL1V9dmBTduAyTsHPkT/2sJk+wfb3QdvA348\nefo+16pqS1WtrKrV9Mftb6vqD4D7gUteptbJY7ik9Z/z/8VU1TPAk0nOak3rgceYh2NKf6robUle\n074XJmudV2M6xUzH8V7gPUlObWdE72ltcyrJBuCTwEVV9bMp9V/a7thaA5wJ/B+G9N5QVY9U1a9X\n1er2szVO/0aTZ5hnYzpjw76I0fGCzkb6d/P8PfCpIdfyDvqneg8DD7XHRvrzwvcB329fX9/6B7i5\n1f4I0BtS3Rfwi7uM3kT/B2o/8D+AV7f2JW19f9v+phNY328BY21c/yf9OzHm5ZgCnwa+BzwK3E7/\n7pd5MabAnfSvbbxA/43qimMZR/pz+Pvb449OUJ376c+zT/5cfWGg/6danfuA9w60z/l7w3S1Ttn+\nOL+4qDy0MZ2Nh7+6QpIEjMaUkSTpBDAQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKk5v8DmQrVUC+R\n9EUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0c2c6d5908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot( neighbors,accuracy, 'r.')\n",
    "plt.xlim(np.min(neighbors), np.max(neighbors))\n",
    "plt.ylim(np.min(accuracy), np.max(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neighbors[np.argmax(accuracy)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.84400671742051059"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83037050306015825"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using TF-IDF - TSNE best performer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neighbors_tf=[]\n",
    "accuracy_tf=[]\n",
    "for k in range(1, 1574, 2):\n",
    "    model=KNeighborsClassifier(n_neighbors=k,algorithm='kd_tree', weights='distance',n_jobs=1)\n",
    "    avg_accuracy_tf=np.mean(cross_val_score(model, bigrams_train_tf, y_train_avgw2v,cv=10,n_jobs=1))\n",
    "    neighbors_tf.append(k)\n",
    "    accuracy_tf.append(avg_accuracy_tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.79150264964920136, 0.85258811016569636)"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFPJJREFUeJzt3X+QXeV93/H3R5IFtcwP21rXIIEl\nWoVB9kxjzxZbTZxoomCDJgZn7D8kh8G02OCmMLWDx4Yx8RDyhwe303g6ITiidnEYAlHt2NXYZGhL\nkZN2FKJVZX4IonjBBBYZe4FpPMbFQvDtH+csXJaFvSvt6t7lvF8zd/ac5zz33u99tPejs885595U\nFZKkblgy6AIkSUePoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdciyQRcw3coV\nK2rNz372YsOqVfDWtw6uIElaBPbs2fNEVY3M1m/oQn/NypWMPfLIiw2f/zxcfPHgCpKkRSDJ3/fT\nb/imd557DpJmOYEnnxxsPZL0GjJ8ob90KUx9CFwVvPnNg61Hkl5Dhi/0n3sOlrRlLVninr4kzaPh\nC/3jjoNjjmkCf8kS9/QlaR71FfpJzk6yP8l4kitm2H5qkjuT7E1yT5LNbfuaJP8vyffa25dnfbIV\nK+BLX2qmeZ5/Hj75Sdi1a84vTJL0crOGfpKlwHXAOcB6YGuS9dO6XQVsr6p3AluAP+rZ9mBV/WJ7\n+0RfVT35ZBP4zz8PzzwDF1wA27b1dVdJ0ivrZ0//TGC8qh6qqoPArcB50/oUcHy7fAJw4Iiq2rgR\nli9vzt6pgvFxuOQSg1+SjlA/ob8KeLRnfaJt63U1cH6SCeA24LKebWvbaZ/vJnlvX1Vt2NBM8Rx3\n3Evbv/GNvu4uSZpZP6GfGdqmf7HuVuDGqloNbAZuSrIE+CFwajvt8zvAnyY5ftp9SXJxkrEkY5OT\nk80c/mWXwU9+8tKOH/pQH+VKkl5JP6E/AZzSs76al0/fXARsB6iqXcCxwMqq+nlVPdm27wEeBH5h\n+hNU1baqGq2q0ZGREdi5E5599qWdPvhBr8yVpCPUT+jvBtYlWZtkOc2B2h3T+jwCbAJIcgZN6E8m\nGWkPBJPkNGAd8NCsz7hxY3P2Tq9zzumjVEnSq5k19KvqEHApcDvwAM1ZOvuSXJPk3Lbb5cDHk9wN\n3AJcWFUF/ApwT9v+deATVfXUrFVt2AAf+1hPlV6kJUnzIVXTp+cHa3R0tMbGxpp5/U2b4ODB5kye\nO+5o/jOQJL1Mkj1VNTpbv6H7lM0XbNjQBP2f/MmgK5Gk14zh+xiG6b72Nbjhhmav3ytzJemIDHfo\n79zZTO8891zzc+fOQVckSYvacIf+1JW5S5c2PzduHHRFkrSoDXfoT83rf+ADsGYNfPGLTvFI0hEY\n3gO5U+69F771rWb5gQfgO9+B737XM3kk6TAM954+vPzzdp591rl9STpMwx/60z9v53Wvc25fkg7T\n8E/vTH3ezle+AiefDJ/5jFM7knSYhj/0oQl+P2xNko7Y8E/vTNm1C371V+GUU+Cznx10NZK0KC2O\nPf1du+C9720u0oLm1E2Aa68dXE2StAgtjj39nTtfDPwpN988kFIkaTFbHKE/0+fr/+hHXqglSXO0\nOKZ3NmyAv/or+MhH4OGHm7ZDh5r/DN7whmb92Web0zmnlqf0ts3Xso99dB/7tfAafGwfe4Ef+x3w\ndvqwOEIfmuC/8kq45JIX2w4ehKdm/04WSXqtO6b5xsJZLY7pnSl+e5YkHZHFFfobN8KyxfPHiSQN\nm8WVoBs2wF/+ZXPK5l//NTz99KKbd/Oxh/h5fGwfexE/9s+feuoZ+rC4Qh+a4P/mNwddhSQNlfuS\nff30W1zTO5KkI2LoS1KHGPqS1CGGviR1yPCF/tNPwxe+4EcsSNICSFUNuoaXGF2ypMaWLIHly5sv\nRfcLUyRpVkn2VNXobP2Gb0+/qvlEzYMH/S5cSZpnfYV+krOT7E8ynuSKGbafmuTOJHuT3JNk8wzb\nf5rk0308WfOJmsuX+124kjTPZr04K8lS4DrgLGAC2J1kR1Xd39PtKmB7VV2fZD1wG7CmZ/sfAH/R\nV0Wnnw4XXNAEvlM7kjSv+rki90xgvKoeAkhyK3Ae0Bv6BRzfLp8AHJjakOSDwEPA031VtGJF82ma\nkqR518/0zirg0Z71ibat19XA+UkmaPbyLwNIsgL4LPB7r/YESS5OMpZkbHJyss/SJUlz1U/oZ4a2\n6af8bAVurKrVwGbgpiRLaML+D6rqp6/2BFW1rapGq2p0ZGSkn7olSYehn+mdCeCUnvXV9EzftC4C\nzgaoql1JjgVWAu8GPpzki8CJwPNJnqmqPzziyiVJc9ZP6O8G1iVZCzwGbAE+Mq3PI8Am4MYkZ9B8\ng8tkVb13qkOSq4GfGviSNDizTu9U1SHgUuB24AGas3T2Jbkmybltt8uBjye5G7gFuLCG7aovSdIQ\nXpE7OlpjY2ODLkOSFpXFe0WuJGnBGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhL\nUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhL\nUocY+pLUIcMX+k8/DV/4AuzaNehKJOk1J1U16BpeYnTJkhpbsgSWL4c77oANGwZdkiQNvSR7qmp0\ntn7Dt6dfBc89BwcPws6dg65Gkl5Tlg26gJdJYGpPf+PGQVcjSa8pfe3pJzk7yf4k40mumGH7qUnu\nTLI3yT1JNrftZyb5Xnu7O8lvzvpkp58Ov//7Tu1I0gKYdU4/yVLg74CzgAlgN7C1qu7v6bMN2FtV\n1ydZD9xWVWuSvB44WFWHkpwE3A2cXFWHXun5RkdHa2xs7IhfmCR1yXzO6Z8JjFfVQ1V1ELgVOG9a\nnwKOb5dPAA4AVNXPegL+2LafJGlA+gn9VcCjPesTbVuvq4Hzk0wAtwGXTW1I8u4k+4B7gU+82l6+\nJGlh9RP6maFt+h77VuDGqloNbAZuSrIEoKruqqq3A/8cuDLJsS97guTiJGNJxiYnJ+f2CiRJfesn\n9CeAU3rWV9NO3/S4CNgOUFW7aKZyVvZ2qKoHgKeBd0x/gqraVlWjVTU6MjLSf/WSpDnpJ/R3A+uS\nrE2yHNgC7JjW5xFgE0CSM2hCf7K9z7K2/W3A6cDD81S7JGmOZj1Pvz3z5lLgdmAp8NWq2pfkGmCs\nqnYAlwM3JPkUzdTPhVVVSX4ZuCLJs8DzwG9X1RML9mokSa9q+D6GwVM2JWnOFu/HMEiSFoyhL0kd\nYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kd\nYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kd\nYuhLUof0FfpJzk6yP8l4kitm2H5qkjuT7E1yT5LNbftZSfYkubf9+Wvz/QIkSf1bNluHJEuB64Cz\ngAlgd5IdVXV/T7ergO1VdX2S9cBtwBrgCeADVXUgyTuA24FV8/waJEl96mdP/0xgvKoeqqqDwK3A\nedP6FHB8u3wCcACgqvZW1YG2fR9wbJJjjrxsSdLh6Cf0VwGP9qxP8PK99auB85NM0OzlXzbD43wI\n2FtVP5++IcnFScaSjE1OTvZVuCRp7voJ/czQVtPWtwI3VtVqYDNwU5IXHjvJ24FrgUtmeoKq2lZV\no1U1OjIy0l/lkqQ56yf0J4BTetZX007f9LgI2A5QVbuAY4GVAElWA98ELqiqB4+0YEnS4esn9HcD\n65KsTbIc2ALsmNbnEWATQJIzaEJ/MsmJwHeAK6vqf89f2ZKkwzFr6FfVIeBSmjNvHqA5S2dfkmuS\nnNt2uxz4eJK7gVuAC6uq2vv9U+B3k3yvvb1lQV6JJGlWabJ5eIyOjtbY2Nigy5CkRSXJnqoana2f\nV+RKUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kd\nYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kd\nYuhLUocY+pLUIX2FfpKzk+xPMp7kihm2n5rkziR7k9yTZHPb/ua2/adJ/nC+i5ckzc2soZ9kKXAd\ncA6wHtiaZP20blcB26vqncAW4I/a9meA3wU+PW8VS5IOWz97+mcC41X1UFUdBG4FzpvWp4Dj2+UT\ngAMAVfV0Vf0vmvCXJA3Ysj76rAIe7VmfAN49rc/VwH9LchmwAvj1ealOkjSv+tnTzwxtNW19K3Bj\nVa0GNgM3Jen7IHGSi5OMJRmbnJzs926SpDnqJ5gngFN61lfTTt/0uAjYDlBVu4BjgZX9FlFV26pq\ntKpGR0ZG+r2bJGmO+gn93cC6JGuTLKc5ULtjWp9HgE0ASc6gCX132SVpyMw6p19Vh5JcCtwOLAW+\nWlX7klwDjFXVDuBy4IYkn6KZ+rmwqgogycM0B3mXJ/kg8L6qun9hXo4k6dX0cyCXqroNuG1a2+d7\nlu8HfukV7rvmCOqTJM0jr8iVpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0Jek\nDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0Jek\nDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeqQvkI/ydlJ9icZT3LFDNtPTXJnkr1J7kmyuWfb\nle399id5/3wWL0mam2WzdUiyFLgOOAuYAHYn2VFV9/d0uwrYXlXXJ1kP3AasaZe3AG8HTgb+R5Jf\nqKrn5vuFSJJm18+e/pnAeFU9VFUHgVuB86b1KeD4dvkE4EC7fB5wa1X9vKp+AIy3jydJGoB+Qn8V\n8GjP+kTb1utq4PwkEzR7+ZfN4b6SpKOkn9DPDG01bX0rcGNVrQY2AzclWdLnfUlycZKxJGOTk5N9\nlCRJOhz9hP4EcErP+mpenL6ZchGwHaCqdgHHAiv7vC9Vta2qRqtqdGRkpP/qJUlz0k/o7wbWJVmb\nZDnNgdkd0/o8AmwCSHIGTehPtv22JDkmyVpgHfA381W8JGluZj17p6oOJbkUuB1YCny1qvYluQYY\nq6odwOXADUk+RTN9c2FVFbAvyXbgfuAQ8G88c0eSBidNNg+P0dHRGhsbG3QZkrSoJNlTVaOz9fOK\nXEnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMM\nfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOGb7Qf/xx\n2LVr0FVI0mvS8IX+Y4/Bpk0GvyQtgFTVoGt4iZVJvQ3qR3DgMXh80PXMYiXwxKCL6MNiqRMWT62L\npU6w1oUwjHW+rapGZus0dKGfZKyqRgddRz8WS62LpU5YPLUuljrBWhfCYqlzJsM3vSNJWjCGviR1\nyDCG/rZBFzAHi6XWxVInLJ5aF0udYK0LYbHU+TJDN6cvSVo4w7inL0laIEMV+knOTrI/yXiSKwZc\nyylJ7kzyQJJ9Sf5t2/6mJP89yffbn29s25PkP7a135PkXUe53qVJ9ib5dru+NsldbZ1/lmR5235M\nuz7ebl9zlOs8McnXk/xtO7YbhnhMP9X+29+X5JYkxw7LuCb5apIfJ7mvp23O45jko23/7yf56FGq\n89+1//73JPlmkhN7tl3Z1rk/yft72hc8G2aqtWfbp5NUkpXt+sDG9IhV1VDcgKXAg8BpwHLgbmD9\nAOs5CXhXu3wc8HfAeuCLwBVt+xXAte3yZuAvgADvAe46yvX+DvCnwLfb9e3Alnb5y8C/bpd/G/hy\nu7wF+LOjXOfXgI+1y8uBE4dxTIFVwA+Af9QznhcOy7gCvwK8C7ivp21O4wi8CXio/fnGdvmNR6HO\n9wHL2uVre+pc377vjwHWtnmw9Ghlw0y1tu2nALcDfw+sHPSYHvHrHHQBPQO7Abi9Z/1K4MpB19VT\nz38FzgL2Aye1bScB+9vlPwa29vR/od9RqG01cAfwa8C321/EJ3reWC+MbfvLu6FdXtb2y1Gq8/g2\nSDOtfRjHdBXwaPvmXdaO6/uHaVyBNdPCdE7jCGwF/rin/SX9FqrOadt+E7i5XX7Je35qTI9mNsxU\nK/B14J8BD/Ni6A90TI/kNkzTO1NvsikTbdvAtX+qvxO4C/jHVfVDgPbnW9pug6z/S8BngOfb9TcD\n/7eqDs1Qywt1ttv/oe1/NJwGTAL/uZ2K+k9JVjCEY1pVjwH/HngE+CHNOO1hOMd1ylzHcRjec/+K\nZo+ZV6lnYHUmORd4rKrunrZp6Grt1zCFfmZoG/ipRUneAHwD+GRV/eTVus7QtuD1J/kN4MdVtafP\nWgY5zsto/ny+vqreCTxNMw3xSgZWazsffh7NNMPJwArgnFepZyh/f1uvVNtAa07yOeAQcPNU0yvU\nM6j31uuBzwGfn2nzDG0DH9N+DFPoT9DMnU1ZDRwYUC0AJHkdTeDfXFV/3jb/KMlJ7faTgB+37YOq\n/5eAc5M8DNxKM8XzJeDEJMtmqOWFOtvtJwBPHYU6p557oqruate/TvOfwLCNKcCvAz+oqsmqehb4\nc+BfMJzjOmWu4ziw8W0PcP4G8FvVzoMMYZ3/hOY//bvb99dq4P8keesQ1tq3YQr93cC69uyI5TQH\nw3YMqpgkAb4CPFBV/6Fn0w5g6oj8R2nm+qfaL2iP6r8H+IepP7UXUlVdWVWrq2oNzZj9z6r6LeBO\n4MOvUOdU/R9u+x+VPZGqehx4NMnpbdMm4H6GbExbjwDvSfL69ndhqtahG9cecx3H24H3JXlj+5fN\n+9q2BZXkbOCzwLlV9bNp9W9pz4RaC6wD/oYBZUNV3VtVb6mqNe37a4Lm5I7HGbIxnZNBH1SYdsBk\nM81ZMg8CnxtwLb9M82fZPcD32ttmmnnaO4Dvtz/f1PYPcF1b+73A6ABq3siLZ++cRvOGGQf+C3BM\n235suz7ebj/tKNf4i8BYO67fojnDYSjHFPg94G+B+4CbaM4qGYpxBW6hOdbwLE0YXXQ440gzpz7e\n3v7lUapznGbee+p99eWe/p9r69wPnNPTvuDZMFOt07Y/zIsHcgc2pkd684pcSeqQYZrekSQtMENf\nkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQ/4/52xb49Zet1sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0c5836aba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot( neighbors_tf,accuracy_tf, 'r.')\n",
    "plt.xlim(np.min(neighbors_tf), np.max(neighbors_tf))\n",
    "plt.ylim(np.min(accuracy_tf), np.max(accuracy_tf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(accuracy_tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neighbors_tf[np.argmax(accuracy_tf)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85258811016569636"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(accuracy_tf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tabulating the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results=pd.DataFrame(columns=['vectorization_method', \n",
    "                             'max_accuracy_on_cross_validation_set', 'optimal_k'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results=results.append(\n",
    "    [\n",
    "        {\n",
    "            'vectorization_method' : 'bag_of_words',\n",
    "            \n",
    "            'max_accuracy_on_cross_validation_set' : 0.84,\n",
    "            'optimal_k': 9\n",
    "        },\n",
    "        \n",
    "        {\n",
    "            'vectorization_method' : 'tf_idf',\n",
    "            \n",
    "            'max_accuracy_on_cross_validation_set' : 0.85,\n",
    "            'optimal_k': 9\n",
    "        },\n",
    "        \n",
    "        {\n",
    "            'vectorization_method' : 'average_word2Vec',\n",
    "            \n",
    "            'max_accuracy_on_cross_validation_set' : 0.84,\n",
    "            'optimal_k': 21\n",
    "        },\n",
    "        \n",
    "        {\n",
    "            'vectorization_method' : 'tf_idf_weighted_word2Vec',\n",
    "            \n",
    "            'max_accuracy_on_cross_validation_set' : 0.84,\n",
    "            'optimal_k' : 17\n",
    "        }\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vectorization_method</th>\n",
       "      <th>max_accuracy_on_cross_validation_set</th>\n",
       "      <th>optimal_k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bag_of_words</td>\n",
       "      <td>0.84</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tf_idf</td>\n",
       "      <td>0.85</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>average_word2Vec</td>\n",
       "      <td>0.84</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tf_idf_weighted_word2Vec</td>\n",
       "      <td>0.84</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       vectorization_method  max_accuracy_on_cross_validation_set optimal_k\n",
       "0              bag_of_words                                  0.84         9\n",
       "1                    tf_idf                                  0.85         9\n",
       "2          average_word2Vec                                  0.84        21\n",
       "3  tf_idf_weighted_word2Vec                                  0.84        17"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_set_accuracy=[]\n",
    "\n",
    "# bag_of_words\n",
    "model_bow=KNeighborsClassifier(n_neighbors=int(results[results.vectorization_method=='bag_of_words'].optimal_k),\n",
    "                              weights='distance', algorithm='brute')\n",
    "model_bow.fit(bigrams_train, y_train_avgw2v)\n",
    "test_set_accuracy.append(accuracy_score(y_test_avgw2v, model_bow.predict(bigrams_test)))\n",
    "\n",
    "# tf_idf\n",
    "model_tfidf=KNeighborsClassifier(n_neighbors=int(results[results.vectorization_method=='tf_idf'].optimal_k),\n",
    "                              weights='distance', algorithm='brute')\n",
    "model_tfidf.fit(bigrams_train_tf, y_train_tfidfw2v)\n",
    "test_set_accuracy.append(accuracy_score(y_test_avgw2v ,model_tfidf.predict(bigrams_test_tf)))\n",
    "\n",
    "# average_word2Vec\n",
    "model_avg_word2Vec = KNeighborsClassifier(n_neighbors=int(results[results.vectorization_method=='average_word2Vec'].optimal_k),\n",
    "                              weights='distance', algorithm='kd_tree')\n",
    "model_avg_word2Vec.fit(X_train_avgw2v, y_train_avgw2v)\n",
    "test_set_accuracy.append(accuracy_score(y_test_avgw2v ,model_avg_word2Vec.predict(X_test_avgw2v)))\n",
    "\n",
    "# tfidf weighted word2Vec\n",
    "model_tfidf_word2Vec = KNeighborsClassifier(n_neighbors=int(results[results.vectorization_method=='average_word2Vec'].optimal_k),\n",
    "                              weights='distance', algorithm='kd_tree')\n",
    "model_tfidf_word2Vec.fit(X_train_tfidfw2v, y_train_tfidfw2v)\n",
    "test_set_accuracy.append(accuracy_score(y_test_tfidfw2v ,model_tfidf_word2Vec.predict(X_test_tfidfw2v)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results['test_set_outcome']=test_set_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vectorization_method</th>\n",
       "      <th>max_accuracy_on_cross_validation_set</th>\n",
       "      <th>optimal_k</th>\n",
       "      <th>test_set_outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bag_of_words</td>\n",
       "      <td>0.84</td>\n",
       "      <td>9</td>\n",
       "      <td>0.822667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tf_idf</td>\n",
       "      <td>0.85</td>\n",
       "      <td>9</td>\n",
       "      <td>0.836000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>average_word2Vec</td>\n",
       "      <td>0.84</td>\n",
       "      <td>21</td>\n",
       "      <td>0.822667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tf_idf_weighted_word2Vec</td>\n",
       "      <td>0.84</td>\n",
       "      <td>17</td>\n",
       "      <td>0.822667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       vectorization_method  max_accuracy_on_cross_validation_set optimal_k  \\\n",
       "0              bag_of_words                                  0.84         9   \n",
       "1                    tf_idf                                  0.85         9   \n",
       "2          average_word2Vec                                  0.84        21   \n",
       "3  tf_idf_weighted_word2Vec                                  0.84        17   \n",
       "\n",
       "   test_set_outcome  \n",
       "0          0.822667  \n",
       "1          0.836000  \n",
       "2          0.822667  \n",
       "3          0.822667  "
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "TF-IDF produces the most accurate vector representations of text reviews"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}